youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

all mat kernel func use new task pack convension

Browse Source
This commit is contained in:
Youmu Liu
2026-06-11 17:27:44 +08:00
parent 09c1cace48
commit 6b449a341e
13 changed files with 192 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/CMakePresets.json
+51
View File
@@ -0,0 +1,51 @@
{
"version": 6,
"configurePresets": [
{
"name": "clang-debug",
"displayName": "Clang Debug",
"generator": "Ninja",
"binaryDir": "${sourceDir}/build",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Debug",
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
"CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++"
}
},
{
"name": "clang-release",
"displayName": "Clang Release",
"generator": "Ninja",
"binaryDir": "${sourceDir}/build",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Release",
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
"CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++"
}
},
{
"name": "使用工具链文件配置预设",
"displayName": "使用工具链文件配置预设",
"description": "设置 Ninja 生成器、版本和安装目录",
"generator": "Ninja",
"binaryDir": "${sourceDir}/out/build/${presetName}",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Debug",
"CMAKE_TOOLCHAIN_FILE": "",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}"
}
}
],
"buildPresets": [
{
"name": "clang-debug",
"configurePreset": "clang-debug"
},
{
"name": "clang-release",
"configurePreset": "clang-release"
}
]
}
StgSharp.Native/ExternalLibs.cmake
-16
View File
@@ -7,14 +7,6 @@ function(sn_normalize_os_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
set(_sn_os "windows")
elseif(_sn_system_name STREQUAL "linux")
set(_sn_os "linux")
elseif(_sn_system_name STREQUAL "darwin" OR _sn_system_name STREQUAL "macos")
set(_sn_os "macos")
elseif(_sn_system_name STREQUAL "ios")
set(_sn_os "ios")
elseif(_sn_system_name STREQUAL "android")
set(_sn_os "android")
else()
set(_sn_os "unknown")
endif()
endif()
@@ -22,14 +14,6 @@ function(sn_normalize_os_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
set(_sn_define SN_OS_WINDOWS)
elseif(_sn_os STREQUAL "linux")
set(_sn_define SN_OS_LINUX)
elseif(_sn_os STREQUAL "macos")
set(_sn_define SN_OS_MACOS)
elseif(_sn_os STREQUAL "ios")
set(_sn_define SN_OS_IOS)
elseif(_sn_os STREQUAL "android")
set(_sn_define SN_OS_ANDROID)
elseif(_sn_os STREQUAL "unknown")
set(_sn_define SN_OS_UNKNOWN)
else()
message(FATAL_ERROR "Unsupported SN_TARGET_OS='${INPUT_VALUE}'.")
endif()
StgSharp.Native/sn-head/sn_intrinsic.matkernel.h
+1 -9
View File
@@ -19,15 +19,7 @@
__m512 z[ZMMCOUNT(T)]; \
T m[4][4]; \
}
/*
#define SN_DEFINE_MAT_KERNEL_TYPE(name, T) \
typedef union name SN_MAT_KERNEL_MEMBER(T) name
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_float, float);
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_double, double);
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_uint64_t, uint64_t);
/* */
#define SN_MAT_KERNEL_DEFINE(T) \
typedef union SN_MAT_KERNEL_TYPE_##T SN_MAT_KERNEL_MEMBER(T) SN_MAT_KERNEL_TYPE_##T
StgSharp.Native/sn-head/sn_intrinsic.std.h
+3 -3
View File
@@ -67,7 +67,7 @@
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
scalar_pack *restrict scalar = (scalar_pack *restrict)task->scalar;\
M128 scalar = task->scalar_pack;\
_SN_MK_EXPAND_BUFFER_SIZE(1);\
_SN_MK_EXPAND_BUFFER_SIZE(2);
@@ -75,14 +75,14 @@
SN_MAT_KERNEL_LOCAL const *left = (SN_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
scalar_pack *restrict scalar = (scalar_pack *restrict)task->scalar;\
M128 scalar = task->scalar_pack;\
_SN_MK_EXPAND_BUFFER_SIZE(0);\
_SN_MK_EXPAND_BUFFER_SIZE(1);\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_MK_EXPAND_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
scalar_pack *restrict scalar = (scalar_pack *restrict)&(task->padding);\
M128 scalar = task->scalar_pack;\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_PARAM_STYLE(T) _SN_MATPROC_##T
StgSharp.Native/sn-src/dllmain.c
+2 -7
View File
@@ -5,13 +5,8 @@
#ifdef _WIN32
/* Define and initialize a global zero kernel aligned to 64 bytes for universal ISA usage */
#ifdef __cplusplus
extern "C" {
#endif
__declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
#ifdef __cplusplus
}
#endif
__declspec(align(64)) MAT_KERNEL(u64) SN_ZERO_KERNEL;
static void init_zero_kernel(void)
{
StgSharp.Native/sn-src/load_glfw.c
-1
View File
@@ -1,6 +1,5 @@
#include "StgSharpNative.h"
#include "glfw_function.h"
#include <string.h>
SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table)
{
StgSharp.Native/sn-src/load_intrinsic.x86.c
-7
View File
@@ -7,13 +7,6 @@
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
#include <isa_availability.h>
#else
#include <cpuid.h>
#endif // _MSC_VER
PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
StgSharp.Native/sn-src/m.bufop.x86.c
+61 -61
View File
@@ -1,17 +1,17 @@
#include "sn_target.h"
#include <immintrin.h>
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include <immintrin.h>
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
SN_MK_PROC_DECL_STD(float, sse, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
__mk_param_std(ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(scalar.f32[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].x[0] = vec;
ans[i].x[1] = vec;
ans[i].x[2] = vec;
@@ -21,9 +21,9 @@ SN_MK_PROC_DECL_STD(float, sse, , fill)
SN_MK_PROC_DECL_STD(float, avx, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
__mk_param_std(ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(&scalar.f32[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].y[0] = vec;
ans[i].y[1] = vec;
}
@@ -31,19 +31,19 @@ SN_MK_PROC_DECL_STD(float, avx, , fill)
SN_MK_PROC_DECL_STD(float, 512, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
__mk_param_std(ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(scalar.m128);
for (size_t i = 0; i < count_2; i++) {
ans[i].z[0] = vec;
}
}
SN_MK_PROC_DECL_STD(double, sse, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m128d vec_d = _mm_set1_pd(((double *)(scalar->data))[0]);
__mk_param_std(ANS_SCALAR, double);
register __m128d vec_d = _mm_set1_pd(scalar.f64[0]);
register __m128 vec = _mm_castpd_ps(vec_d);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_2; i++) {
ans[i].x[0] = vec;
ans[i].x[1] = vec;
ans[i].x[2] = vec;
@@ -57,9 +57,9 @@ SN_MK_PROC_DECL_STD(double, sse, , fill)
SN_MK_PROC_DECL_STD(double, avx, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
__mk_param_std(ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(&scalar.f64[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].y[0] = _mm256_castpd_ps(vec);
ans[i].y[1] = _mm256_castpd_ps( vec);
ans[i].y[2] = _mm256_castpd_ps( vec);
@@ -69,9 +69,9 @@ SN_MK_PROC_DECL_STD(double, avx, , fill)
SN_MK_PROC_DECL_STD(double, 512, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(((double const *)scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
__mk_param_std(ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(scalar.f64[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].z[0] = _mm512_castpd_ps(vec);
ans[i].z[1] = _mm512_castpd_ps(vec);
}
@@ -79,9 +79,9 @@ SN_MK_PROC_DECL_STD(double, 512, , fill)
SN_MK_PROC_DECL_STD(float, sse, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m128 c0 = _mm_add_ps(left[i].x[0], right[i].x[0]);
register __m128 c1 = _mm_add_ps(left[i].x[1], right[i].x[1]);
register __m128 c2 = _mm_add_ps(left[i].x[2], right[i].x[2]);
@@ -96,9 +96,9 @@ SN_MK_PROC_DECL_STD(float, sse, , add)
SN_MK_PROC_DECL_STD(float, avx, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256 c0 = _mm256_add_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_add_ps(left[i].y[1], right[i].y[1]);
@@ -109,18 +109,18 @@ SN_MK_PROC_DECL_STD(float, avx, , add)
SN_MK_PROC_DECL_STD(float, 512, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m512 c0 = _mm512_add_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
SN_MK_PROC_DECL_STD(double, sse, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].x[0]),
_mm_castps_pd(right[i].x[0]));
register __m128d c1 = _mm_add_pd(_mm_castps_pd(left[i].x[1]),
@@ -151,9 +151,9 @@ SN_MK_PROC_DECL_STD(double, sse, , add)
SN_MK_PROC_DECL_STD(double, avx, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256d c0 = _mm256_add_pd(left[i].y[0], right[i].y[0]);
register __m256d c1 = _mm256_add_pd(left[i].y[1], right[i].y[1]);
@@ -168,9 +168,9 @@ SN_MK_PROC_DECL_STD(double, avx, , add)
SN_MK_PROC_DECL_STD(double, 512, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m512d c0 = _mm512_add_pd(_mm512_castps_pd( left[i].z[0]), _mm512_castps_pd( right[i].z[0]));
register __m512d c1 = _mm512_add_pd(_mm512_castps_pd( left[i].z[1]), _mm512_castps_pd( right[i].z[1]));
ans[i].z[0] = c0;
@@ -180,9 +180,9 @@ SN_MK_PROC_DECL_STD(double, 512, , add)
SN_MK_PROC_DECL_STD(float, sse, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std( LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m128 c0 = _mm_sub_ps(left[i].x[0], right[i].x[0]);
register __m128 c1 = _mm_sub_ps(left[i].x[1], right[i].x[1]);
register __m128 c2 = _mm_sub_ps(left[i].x[2], right[i].x[2]);
@@ -197,9 +197,9 @@ SN_MK_PROC_DECL_STD(float, sse, , sub)
SN_MK_PROC_DECL_STD(float, avx, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std( LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
@@ -209,9 +209,9 @@ SN_MK_PROC_DECL_STD(float, avx, , sub)
SN_MK_PROC_DECL_STD(float, 512, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std( LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
@@ -219,9 +219,9 @@ SN_MK_PROC_DECL_STD(float, 512, , sub)
SN_MK_PROC_DECL_STD(double, sse, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m128d c0 = _mm_sub_pd(left[i].x[0], right[i].x[0]);
register __m128d c1 = _mm_sub_pd(left[i].x[1], right[i].x[1]);
register __m128d c2 = _mm_sub_pd(left[i].x[2], right[i].x[2]);
@@ -244,9 +244,9 @@ SN_MK_PROC_DECL_STD(double, sse, , sub)
SN_MK_PROC_DECL_STD(double, avx, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
@@ -256,9 +256,9 @@ SN_MK_PROC_DECL_STD(double, avx, , sub)
SN_MK_PROC_DECL_STD(double, 512, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
for (size_t i = 0; i < count_0; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
@@ -266,10 +266,10 @@ SN_MK_PROC_DECL_STD(double, 512, , sub)
SN_MK_PROC_DECL_STD(float, sse, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, float);
__mk_param_std( RIGHT_ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m128 vec = _mm_set_ps1(scalar.f32[0]);
for (size_t i = 0; i < count_1; i++) {
register __m128 c0 = _mm_mul_ps(right[i].x[0], vec);
register __m128 c1 = _mm_mul_ps(right[i].x[1], vec);
register __m128 c2 = _mm_mul_ps(right[i].x[2], vec);
@@ -284,10 +284,10 @@ SN_MK_PROC_DECL_STD(float, sse, , scalar_mul)
SN_MK_PROC_DECL_STD(float, avx, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, float);
__mk_param_std( RIGHT_ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
register __m256 vec = _mm256_broadcast_ss(&scalar.f32[0]);
for (size_t i = 0; i < count_1; i++) {
register __m256 c0 = _mm256_mul_ps(right[i].y[0], vec);
register __m256 c1 = _mm256_mul_ps(right[i].y[1], vec);
@@ -298,10 +298,10 @@ SN_MK_PROC_DECL_STD(float, avx, , scalar_mul)
SN_MK_PROC_DECL_STD(float, 512, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, float);
__mk_param_std( RIGHT_ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m512 vec = _mm512_broadcastss_ps(scalar.m128);
for (size_t i = 0; i < count_1; i++) {
register __m512 c0 = _mm512_mul_ps(right[i].z[0], vec);
ans[i].z[0] = c0;
@@ -310,10 +310,10 @@ SN_MK_PROC_DECL_STD(float, 512, , scalar_mul)
SN_MK_PROC_DECL_STD(double, sse, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, double);
__mk_param_std( RIGHT_ANS_SCALAR, double);
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m128d vec = _mm_set1_pd(scalar.f64[0]);
for (size_t i = 0; i < count_1; i++) {
register __m128d c0 = _mm_mul_pd(right[i].x[0], vec);
register __m128d c1 = _mm_mul_pd(right[i].x[1], vec);
register __m128d c2 = _mm_mul_pd(right[i].x[2], vec);
@@ -336,10 +336,10 @@ SN_MK_PROC_DECL_STD(double, sse, , scalar_mul)
SN_MK_PROC_DECL_STD(double, avx, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, double);
__mk_param_std( RIGHT_ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
register __m256d vec = _mm256_broadcast_sd(&scalar.f64[0]);
for (size_t i = 0; i < count_1; i++) {
register __m256d c0 = _mm256_mul_pd(right[i].y[0], vec);
register __m256d c1 = _mm256_mul_pd(right[i].y[1], vec);
register __m256d c2 = _mm256_mul_pd(right[i].y[2], vec);
@@ -354,10 +354,10 @@ SN_MK_PROC_DECL_STD(double, avx, , scalar_mul)
SN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, double);
__mk_param_std( RIGHT_ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(((double const *)scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
register __m512d vec = _mm512_set1_pd(scalar.f64[0]);
for (size_t i = 0; i < count_1; i++) {
register __m512d c0 = _mm512_mul_pd(right[i].z[0], vec);
register __m512d c1 = _mm512_mul_pd(right[i].z[1], vec);
StgSharp.Native/sn-src/m.solve.x86.c
+4 -4
View File
@@ -11,14 +11,14 @@ static const float quality_threshold = 1e-6f;
SN_MK_PROC_DECL_STD(float, sse, , try_plu)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS_SCALAR, float);
__mk_param_std(LEFT_RIGHT_ANS_SCALAR, float);
/* Aliases */
MAT_KERNEL(float) *upper = left; /* U */
MAT_KERNEL(float) *lower = right; /* L (unit diag) */
MAT_KERNEL(float) *source = ans; /* A */
unsigned char *pivot = (unsigned char *)&scalar->data[0]; /* length = 4 */
int *success = (int *)&scalar->data[1];
unsigned char *pivot = (unsigned char *)&scalar.i64[0]; /* length = 4 */
int *success = (int *)&scalar.i64[1];
/* Init permutation and outputs */
pivot[0] = 0;
@@ -117,7 +117,7 @@ SN_MK_PROC_DECL_STD(float, sse, , try_plu)
*/
SN_MK_PROC_DECL_STD(float, sse, , quality)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
__mk_param_std(RIGHT_ANS, float);
register __m128 tiny = _mm_set1_ps(1e-6f);
StgSharp.Native/sn-src/m.transpose.x86.c
+3 -3
View File
@@ -9,7 +9,7 @@
// Matrix transpose implementations
SN_MK_PROC_DECL_STD(float, sse, , transpose)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
__mk_param_std(RIGHT_ANS, float);
register __m128 col0 = right->x[0];
register __m128 col1 = right->x[1];
@@ -28,7 +28,7 @@ SN_MK_PROC_DECL_STD(float, sse, , transpose)
SN_MK_PROC_DECL_STD(float, avx, , transpose)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
__mk_param_std(RIGHT_ANS, float);
register __m256 col_low = right->y[0];
register __m256 col_high = right->y[1];
@@ -44,7 +44,7 @@ SN_MK_PROC_DECL_STD(float, avx, , transpose)
SN_MK_PROC_DECL_STD(float, 512, , transpose)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
__mk_param_std(RIGHT_ANS, float);
register __m512 matrix = right->z[0];
const __m512i transpose_indices =
StgSharp.Native/sn-src/mk.fma.x86.c
+65 -67
View File
@@ -19,29 +19,28 @@ SN_MK_PROC_DECL_STD(float, sse, , fma_pack)
{
__mk_param_std(RIGHT_ANS, float);
memcpy(ans, right, sizeof(MAT_KERNEL(float)) * common_y * common_z);
memcpy(ans, right+offset_1, sizeof(MAT_KERNEL(float)) * count_1);
}
SN_MK_PROC_DECL_STD(float, avx, , fma_pack)
{
__mk_param_std(RIGHT_ANS, float);
memcpy(ans, right, sizeof(MAT_KERNEL(float)) * common_y * common_z);
//memcpy(ans, right, sizeof(MAT_KERNEL(float)) * common_y * common_z);
}
SN_MK_PROC_DECL_STD(float, 512, , fma_pack)
{
__mk_param_std(RIGHT_ANS, float);
memcpy(ans, right, sizeof(MAT_KERNEL(float)) * common_y * common_z);
//memcpy(ans, right, sizeof(MAT_KERNEL(float)) * common_y * common_z);
}
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m128 c0 = _mm_load_ps((float *)&ans_cur->x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->x[1]);
@@ -49,38 +48,38 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
register __m128 c3 = _mm_load_ps((float *)&ans_cur->x[3]);
register __m128 bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *right_base = right;
SN_MAT_KERNEL_LOCAL const *left_base = left;
const __m128 a0 = _mm_load_ps((float *)&left_cur->x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_cur->x[1]);
const __m128 a2 = _mm_load_ps((float *)&left_cur->x[2]);
const __m128 a3 = _mm_load_ps((float *)&left_cur->x[3]);
const __m128 a0 = _mm_load_ps((float *)&left_base->x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_base->x[1]);
const __m128 a2 = _mm_load_ps((float *)&left_base->x[2]);
const __m128 a3 = _mm_load_ps((float *)&left_base->x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right_cur->x[0]);
bcol = _mm_load_ps((float *)&right_base->x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right_cur->x[1]);
bcol = _mm_load_ps((float *)&right_base->x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right_cur->x[2]);
bcol = _mm_load_ps((float *)&right_base->x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right_cur->x[3]);
bcol = _mm_load_ps((float *)&right_base->x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
@@ -96,15 +95,15 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
SN_MK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < common_z; z+=2) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z;
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *b_base = right;
SN_MAT_KERNEL_LOCAL const *a_base = left;
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
@@ -131,13 +130,13 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < common_z; z+=2) {
for (size_t z = 0; z < count_0; z+=2) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z;
@@ -167,13 +166,13 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m512 c0 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c1 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]);
for (size_t z = 0; z < common_z; z+=4) {
for (size_t z = 0; z < count_0; z+=4) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z;
@@ -204,8 +203,7 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m128d c0_lo = ans_cur->x[0];
register __m128d c0_hi = ans_cur->x[1];
@@ -216,11 +214,11 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
register __m128d c3_lo = ans_cur->x[6];
register __m128d c3_hi = ans_cur->x[7];
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
const double *right_values = &right_cur->m[0][0];
const double *left_values = &left_cur->m[0][0];
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *right_base = right;
SN_MAT_KERNEL_LOCAL const *left_base = left;
const double *right_values = &right_base->m[0][0];
const double *left_values = &left_base->m[0][0];
SN_MK_FMA_F64_SSE_PHASE(0);
@@ -245,7 +243,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // tKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
@@ -253,17 +251,17 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]);
register __m256d bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *right_base = right;
SN_MAT_KERNEL_LOCAL const *left_base = left;
const __m256d a0 = _mm256_load_pd((double *)&left_cur->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->y[3]);
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_base->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_base->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->y[0]);
bcol = _mm256_load_pd((double *)&right_base->y[0]);
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c0 = _mm256_add_pd(c0,
@@ -274,7 +272,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_cur->y[1]);
bcol = _mm256_load_pd((double *)&right_base->y[1]);
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c1 = _mm256_add_pd(c1,
@@ -285,7 +283,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_cur->y[2]);
bcol = _mm256_load_pd((double *)&right_base->y[2]);
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c2 = _mm256_add_pd(c2,
@@ -296,7 +294,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_cur->y[3]);
bcol = _mm256_load_pd((double *)&right_base->y[3]);
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c3 = _mm256_add_pd(c3,
@@ -317,7 +315,7 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
@@ -325,38 +323,38 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]);
register __m256d bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *right_base = right;
SN_MAT_KERNEL_LOCAL const *left_base = left;
const __m256d a0 = _mm256_load_pd((double *)&left_cur->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->y[3]);
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_base->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_base->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->y[0]);
bcol = _mm256_load_pd((double *)&right_base->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_cur->y[1]);
bcol = _mm256_load_pd((double *)&right_base->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_cur->y[2]);
bcol = _mm256_load_pd((double *)&right_base->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_cur->y[3]);
bcol = _mm256_load_pd((double *)&right_base->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
@@ -372,26 +370,26 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m512d c0 = _mm512_load_pd((double *)&ans_cur->z[0]);
register __m512d c1 = _mm512_load_pd((double *)&ans_cur->z[1]);
register __m512d c2 = _mm512_load_pd((double *)&ans_cur->z[2]);
register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]);
for (size_t z = 0; z < common_z; z+=2) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + z;
SN_MAT_KERNEL_LOCAL const *left_cur = left + z;
for (size_t z = 0; z < count_0; z+=2) {
SN_MAT_KERNEL_LOCAL const *right_base = right + z;
SN_MAT_KERNEL_LOCAL const *left_base = left + z;
register __m512d a0 = _mm512_load_pd((double *)&left_cur->z[0]);
register __m512d a1 = _mm512_load_pd((double *)&left_cur->z[1]);
register __m512d a2 = _mm512_load_pd((double *)&left_cur->z[2]);
register __m512d a3 = _mm512_load_pd((double *)&left_cur->z[3]);
register __m512d a0 = _mm512_load_pd((double *)&left_base->z[0]);
register __m512d a1 = _mm512_load_pd((double *)&left_base->z[1]);
register __m512d a2 = _mm512_load_pd((double *)&left_base->z[2]);
register __m512d a3 = _mm512_load_pd((double *)&left_base->z[3]);
register __m512d b0 = _mm512_load_pd((double *)&right_cur->z[0]);
register __m512d b1 = _mm512_load_pd((double *)&right_cur->z[1]);
register __m512d b2 = _mm512_load_pd((double *)&right_cur->z[2]);
register __m512d b3 = _mm512_load_pd((double *)&right_cur->z[3]);
register __m512d b0 = _mm512_load_pd((double *)&right_base->z[0]);
register __m512d b1 = _mm512_load_pd((double *)&right_base->z[1]);
register __m512d b2 = _mm512_load_pd((double *)&right_base->z[2]);
register __m512d b3 = _mm512_load_pd((double *)&right_base->z[3]);
SN_MK_FMA_F64_AVX2_CYCLE(0);
SN_MK_FMA_F64_AVX2_CYCLE(1);
StgSharp.Native/sn-src/mk.p_scan.x86.c
+1 -2
View File
@@ -5,7 +5,6 @@
#include "sn_internal.h"
#include "sn_intrinsic.context.x86.h"
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.matkernel.h"
#include <emmintrin.h>
@@ -14,7 +13,7 @@
SN_MK_PROC_DECL_STD(float, sse, , k_quality)
{
__mk_param_std(TILE, RIGHT_ANS_SCALAR, float);
__mk_param_std(RIGHT_ANS_SCALAR, float);
}
INTERNAL SN_STD_TARGET_ATTR(sse, ) void kp_check_f32(mat_kernel *kernel)
StgSharp.Native/sn-src/vector.c
+1 -1
View File
@@ -1,7 +1,7 @@
#include "sn_intrinsic.h"
#include "sn_internal.h"
INTERNAL void SN_DECL f32_normalize(sn_vec_f32 *restrict source, sn_vec_f32 *restrict target)
INTERNAL void SN_DECL f32_normalize(VEC_SEGMENT(float) *restrict source, VEC_SEGMENT(float) *restrict target)
{
register __m128 s = source->f32_x[0];
register __m128 a_length_sq = _mm_mul_ps(s, s);