youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

switch C code prefix to NIF

Browse Source
This commit is contained in:
Youmu Liu
2026-09-03 20:04:20 +08:00
parent 7c7101ee48
commit 41ca31fd9a
51 changed files with 1047 additions and 1030 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Nitload.Native/nif-src/dllmain.c
+42
View File
@@ -0,0 +1,42 @@
#include "NitloadFrameWork.h"
#include "nif_intrinsic.matkernel.h"
#include <stdint.h>
#ifdef _WIN32
/* Define and initialize a global zero kernel aligned to 64 bytes for universal ISA usage */
__declspec(align(64)) MAT_KERNEL(u64) NIF_ZERO_KERNEL;
static void init_zero_kernel(void)
{
for (int c = 0; c < 4; ++c) {
for (int r = 0; r < 4; ++r) {
NIF_ZERO_KERNEL.m[r][c] = 0;
}
}
}
BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
{
switch (ul_reason_for_call) {
case DLL_PROCESS_ATTACH:
init_zero_kernel();
return 1;
case DLL_THREAD_ATTACH:
return 1;
case DLL_THREAD_DETACH:
return 1;
case DLL_PROCESS_DETACH:
return 1;
break;
}
return TRUE;
}
#else
attribute((constructor)) void DllMain()
{
init_zero_kernel();
}
#endif // _WIN32
src/Nitload.Native/nif-src/load_glfw.c
+121
View File
@@ -0,0 +1,121 @@
#include "NitloadFrameWork.h"
NIF_API void NIF_DECL nif_load_glfw_functions(GLFWFunctionTable *table)
{
table->glfwCreateCursor = glfwCreateCursor;
table->glfwCreateStandardCursor = glfwCreateStandardCursor;
table->glfwCreateWindow = glfwCreateWindow;
table->glfwDefaultWindowHints = glfwDefaultWindowHints;
table->glfwDestroyCursor = glfwDestroyCursor;
table->glfwDestroyWindow = glfwDestroyWindow;
table->glfwExtensionSupported = glfwExtensionSupported;
table->glfwFocusWindow = glfwFocusWindow;
table->glfwGetClipboardString = glfwGetClipboardString;
table->glfwGetCurrentContext = glfwGetCurrentContext;
table->glfwGetCursorPos = glfwGetCursorPos;
table->glfwGetError = glfwGetError;
table->glfwGetFrameBufferSize = glfwGetFramebufferSize;
table->glfwGetGamepadName = glfwGetGamepadName;
table->glfwGetGamepadState = glfwGetGamepadState;
table->glfwGetGammaRamp = glfwGetGammaRamp;
table->glfwGetInputMode = glfwGetInputMode;
table->glfwGetJoystickAxes = glfwGetJoystickAxes;
table->glfwGetJoystickButtons = glfwGetJoystickButtons;
table->glfwGetJoystickGUID = glfwGetJoystickGUID;
table->glfwGetJoystickHats = glfwGetJoystickHats;
table->glfwGetJoystickName = glfwGetJoystickName;
table->glfwGetJoystickUserPointer = glfwGetJoystickUserPointer;
table->glfwGetKey = glfwGetKey;
table->glfwGetKeyName = glfwGetKeyName;
table->glfwGetKeyScancode = glfwGetKeyScancode;
table->glfwGetMonitorContentScale = glfwGetMonitorContentScale;
table->glfwGetMonitorName = glfwGetMonitorName;
table->glfwGetMonitorPhysicalSize = glfwGetMonitorPhysicalSize;
table->glfwGetMonitorPos = glfwGetMonitorPos;
table->glfwGetMonitors = glfwGetMonitors;
table->glfwGetMonitorUserPointer = glfwGetMonitorUserPointer;
table->glfwGetMonitorWorkarea = glfwGetMonitorWorkarea;
table->glfwGetMouseButton = glfwGetMouseButton;
table->glfwGetPrimaryMonitor = glfwGetPrimaryMonitor;
table->glfwGetProcAddress = glfwGetProcAddress;
table->glfwGetRequiredInstanceExtensions = glfwGetRequiredInstanceExtensions;
table->glfwGetTime = glfwGetTime;
table->glfwGetTimerFrequency = glfwGetTimerFrequency;
table->glfwGetTimerValue = glfwGetTimerValue;
table->glfwGetVersion = glfwGetVersion;
table->glfwGetVersionString = glfwGetVersionString;
table->glfwGetVideoMode = glfwGetVideoMode;
table->glfwGetVideoModes = glfwGetVideoModes;
table->glfwGetWindowAttrib = glfwGetWindowAttrib;
table->glfwGetWindowContentScale = glfwGetWindowContentScale;
table->glfwGetWindowFrameSize = glfwGetWindowFrameSize;
table->glfwGetWindowMonitor = glfwGetWindowMonitor;
table->glfwGetWindowOpacity = glfwGetWindowOpacity;
table->glfwGetWindowPos = glfwGetWindowPos;
table->glfwGetWindowSize = glfwGetWindowSize;
table->glfwGetWindowUserPointer = glfwGetWindowUserPointer;
table->glfwHideWindow = glfwHideWindow;
table->glfwIconifyWindow = glfwIconifyWindow;
table->glfwInit = glfwInit;
table->glfwInitHint = glfwInitHint;
table->glfwJoystickIsGamePad = glfwJoystickIsGamepad;
table->glfwJoystickPresent = glfwJoystickPresent;
table->glfwMakeContextCurrent = glfwMakeContextCurrent;
table->glfwMaximizeWindow = glfwMaximizeWindow;
table->glfwPollEvents = glfwPollEvents;
table->glfwPostEmptyEvent = glfwPostEmptyEvent;
table->glfwRawMouseMotionSupported = glfwRawMouseMotionSupported;
table->glfwRequestWindowAttention = glfwRequestWindowAttention;
table->glfwRestoreWindow = glfwRestoreWindow;
table->glfwSetCharCallback = glfwSetCharCallback;
table->glfwSetCharModsCallback = glfwSetCharModsCallback;
table->glfwSetClipboardString = glfwSetClipboardString;
table->glfwSetCursor = glfwSetCursor;
table->glfwSetCursorEnterCallback = glfwSetCursorEnterCallback;
table->glfwSetCursorPos = glfwSetCursorPos;
table->glfwSetCursorPosCallback = glfwSetCursorPosCallback;
table->glfwSetDropCallback = glfwSetDropCallback;
table->glfwSetErrorCallback = glfwSetErrorCallback;
table->glfwSetFramebufferSizeCallback = glfwSetFramebufferSizeCallback;
table->glfwSetGamma = glfwSetGamma;
table->glfwSetGammaRamp = glfwSetGammaRamp;
table->glfwSetInputMode = glfwSetInputMode;
table->glfwSetJoystickCallback = glfwSetJoystickCallback;
table->glfwSetJoystickUserPointer = glfwSetJoystickUserPointer;
table->glfwSetKeyCallback = glfwSetKeyCallback;
table->glfwSetMonitorCallback = glfwSetMonitorCallback;
table->glfwSetMonitorUserPointer = glfwSetMonitorUserPointer;
table->glfwSetMouseButtonCallback = glfwSetMouseButtonCallback;
table->glfwSetScrollCallback = glfwSetScrollCallback;
table->glfwSetTime = glfwSetTime;
table->glfwSetWindowAspectRatio = glfwSetWindowAspectRatio;
table->glfwSetWindowAttrib = glfwSetWindowAttrib;
table->glfwSetWindowCloseCallback = glfwSetWindowCloseCallback;
table->glfwSetWindowContentScaleCallback = glfwSetWindowContentScaleCallback;
table->glfwSetWindowFocusCallback = glfwSetWindowFocusCallback;
table->glfwSetWindowIcon = glfwSetWindowIcon;
table->glfwSetWindowIconifyCallback = glfwSetWindowIconifyCallback;
table->glfwSetWindowMaximizeCallback = glfwSetWindowMaximizeCallback;
table->glfwSetWindowMonitor = glfwSetWindowMonitor;
table->glfwSetWindowOpacity = glfwSetWindowOpacity;
table->glfwSetWindowPos = glfwSetWindowPos;
table->glfwSetWindowPosCallback = glfwSetWindowPosCallback;
table->glfwSetWindowRefreshCallback = glfwSetWindowRefreshCallback;
table->glfwSetWindowShouldClose = glfwSetWindowShouldClose;
table->glfwSetWindowSize = glfwSetWindowSize;
table->glfwSetWindowSizeCallback = glfwSetWindowSizeCallback;
table->glfwSetWindowSizeLimits = glfwSetWindowSizeLimits;
table->glfwSetWindowTitle = glfwSetWindowTitle;
table->glfwSetWindowUserPointer = glfwSetWindowUserPointer;
table->glfwShowWindow = glfwShowWindow;
table->glfwSwapBuffers = glfwSwapBuffers;
table->glfwSwapInterval = glfwSwapInterval;
table->glfwTerminate = glfwTerminate;
table->glfwUpdateGamePadMappings = glfwUpdateGamepadMappings;
table->glfwVulkanSupported = glfwVulkanSupported;
table->glfwWaitEvents = glfwWaitEvents;
table->glfwWaitEventsTimeout = glfwWaitEventsTimeout;
table->glfwWindowHint = glfwWindowHint;
table->glfwWindowHintString = glfwWindowHintString;
table->glfwWindowShouldClose = glfwWindowShouldClose;
}
src/Nitload.Native/nif-src/load_intrinsic.x86.c
+118
View File
@@ -0,0 +1,118 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "NitloadFrameWork.h"
#include "nif_intrinsic.h"
#include "nif_intrinsic.std.h"
#include "nif_intrinsic.context.x86.h"
PRIVATE void load_intrinsic_sse(nif_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx(nif_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512(nif_intrinsic *context, SIMDID id);
NIF_API void NIF_DECL nif_load_intrinsic_function(void *intrinsic_context, uint64_t id)
{
nif_null_assert(intrinsic_context);
SIMDID simdid;
if (id) {
simdid.mask = id;
} else {
simdid = nif_get_simd_level_global();
}
nif_intrinsic *context = (nif_intrinsic *)intrinsic_context;
switch (simdid.make_byte[1]) {
case SIMDID_MAIN_LVL_SSE:
load_intrinsic_sse(context, simdid);
break;
case SIMDID_MAIN_LVL_AVX2:
load_intrinsic_avx(context, simdid);
break;
case SIMDID_MAIN_LVL_AVX512:
case SIMDID_MAIN_LVL_AVX10:
load_intrinsic_512(context, simdid);
break;
}
}
void load_intrinsic_sse(nif_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, sse, , add);
context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, sse, , fill);
context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, sse, , scalar_mul);
context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, sse, , sub);
context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, sse, , transpose);
context->mat_mk[F32].proc_array[Fma] = GET_MK_PROC_STD(float, sse, , fma);
context->mat_mk[F32].proc_array[Pivot] = NULL;
context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, sse, , add);
context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, sse, , fill);
context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, sse, , scalar_mul);
context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, sse, , sub);
context->mat_mk[F64].proc_array[Transpose] = NULL;
context->mat_mk[F64].proc_array[Fma] = GET_MK_PROC_STD(double, sse, , fma);
context->mat_mk[F64].proc_array[Pivot] = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
void load_intrinsic_avx(nif_intrinsic *context, SIMDID id)
{
char avx_seg = id.make_byte[4];
char is_fma = (avx_seg & SIMDID_AVX_FMA) != 0;
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, avx, , add);
context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, avx, , fill);
context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, avx, , scalar_mul);
context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, avx, , sub);
context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, avx, , transpose);
context->mat_mk[F32].proc_array[Fma] = is_fma ? GET_MK_PROC_STD(float, avx, _fma, fma) :
GET_MK_PROC_STD(float, avx, , fma);
context->mat_mk[F32].proc_array[Pivot] = NULL;
context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, avx, , add);
context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, avx, , fill);
context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, avx, , scalar_mul);
context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, avx, , sub);
context->mat_mk[F64].proc_array[Transpose] = NULL;
context->mat_mk[F64].proc_array[Fma] = is_fma ? GET_MK_PROC_STD(double, avx, _fma, fma) :
GET_MK_PROC_STD(double, avx, , fma);
context->mat_mk[F64].proc_array[Pivot] = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
void load_intrinsic_512(nif_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, 512, , add);
context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, 512, , fill);
context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, 512, , scalar_mul);
context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, 512, , sub);
context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, 512, , transpose);
context->mat_mk[F32].proc_array[Fma] = GET_MK_PROC_STD(float, 512, , fma);
context->mat_mk[F32].proc_array[Pivot] = NULL;
context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, 512, , add);
context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, 512, , fill);
context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, 512, , scalar_mul);
context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, 512, , sub);
context->mat_mk[F64].proc_array[Transpose] = NULL;
context->mat_mk[F64].proc_array[Fma] = GET_MK_PROC_STD(double, 512, , fma);
context->mat_mk[F64].proc_array[Pivot] = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
#endif
src/Nitload.Native/nif-src/m.bufop.x86.c
+369
View File
@@ -0,0 +1,369 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include <immintrin.h>
#include "nif_intrinsic.h"
#include "nif_intrinsic.std.h"
#include "nif_intrinsic.context.x86.h"
NIF_MK_PROC_DECL_STD(float, sse, , fill)
{
__mk_param_std(ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(scalar.f32[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].x[0] = vec;
ans[i].x[1] = vec;
ans[i].x[2] = vec;
ans[i].x[3] = vec;
}
}
NIF_MK_PROC_DECL_STD(float, avx, , fill)
{
__mk_param_std(ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(&scalar.f32[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].y[0] = vec;
ans[i].y[1] = vec;
}
}
NIF_MK_PROC_DECL_STD(float, 512, , fill)
{
__mk_param_std(ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(scalar.m128);
for (size_t i = 0; i < count_2; i++) {
ans[i].z[0] = vec;
}
}
NIF_MK_PROC_DECL_STD(double, sse, , fill)
{
__mk_param_std(ANS_SCALAR, double);
register __m128d vec_d = _mm_set1_pd(scalar.f64[0]);
register __m128 vec = _mm_castpd_ps(vec_d);
for (size_t i = 0; i < count_2; i++) {
ans[i].x[0] = vec;
ans[i].x[1] = vec;
ans[i].x[2] = vec;
ans[i].x[3] = vec;
ans[i].x[4] = vec;
ans[i].x[5] = vec;
ans[i].x[6] = vec;
ans[i].x[7] = vec;
}
}
NIF_MK_PROC_DECL_STD(double, avx, , fill)
{
__mk_param_std(ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(&scalar.f64[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].y[0] = _mm256_castpd_ps(vec);
ans[i].y[1] = _mm256_castpd_ps( vec);
ans[i].y[2] = _mm256_castpd_ps( vec);
ans[i].y[3] = _mm256_castpd_ps( vec);
}
}
NIF_MK_PROC_DECL_STD(double, 512, , fill)
{
__mk_param_std(ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(scalar.f64[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].z[0] = _mm512_castpd_ps(vec);
ans[i].z[1] = _mm512_castpd_ps(vec);
}
}
NIF_MK_PROC_DECL_STD(float, sse, , add)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m128 c0 = _mm_add_ps(left[i].x[0], right[i].x[0]);
register __m128 c1 = _mm_add_ps(left[i].x[1], right[i].x[1]);
register __m128 c2 = _mm_add_ps(left[i].x[2], right[i].x[2]);
register __m128 c3 = _mm_add_ps(left[i].x[3], right[i].x[3]);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
}
}
NIF_MK_PROC_DECL_STD(float, avx, , add)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256 c0 = _mm256_add_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_add_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
NIF_MK_PROC_DECL_STD(float, 512, , add)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m512 c0 = _mm512_add_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
NIF_MK_PROC_DECL_STD(double, sse, , add)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].x[0]),
_mm_castps_pd(right[i].x[0]));
register __m128d c1 = _mm_add_pd(_mm_castps_pd(left[i].x[1]),
_mm_castps_pd(right[i].x[1]));
register __m128d c2 = _mm_add_pd(_mm_castps_pd(left[i].x[2]),
_mm_castps_pd(right[i].x[2]));
register __m128d c3 = _mm_add_pd(_mm_castps_pd(left[i].x[3]),
_mm_castps_pd(right[i].x[3]));
register __m128d c4 = _mm_add_pd(_mm_castps_pd(left[i].x[4]),
_mm_castps_pd(right[i].x[4]));
register __m128d c5 = _mm_add_pd(_mm_castps_pd(left[i].x[5]),
_mm_castps_pd(right[i].x[5]));
register __m128d c6 = _mm_add_pd(_mm_castps_pd(left[i].x[6]),
_mm_castps_pd(right[i].x[6]));
register __m128d c7 = _mm_add_pd(_mm_castps_pd(left[i].x[7]),
_mm_castps_pd(right[i].x[7]));
ans[i].x[0] = _mm_castpd_ps(c0);
ans[i].x[1] = _mm_castpd_ps(c1);
ans[i].x[2] = _mm_castpd_ps(c2);
ans[i].x[3] = _mm_castpd_ps(c3);
ans[i].x[4] = _mm_castpd_ps(c4);
ans[i].x[5] = _mm_castpd_ps(c5);
ans[i].x[6] = _mm_castpd_ps(c6);
ans[i].x[7] = _mm_castpd_ps(c7);
}
}
NIF_MK_PROC_DECL_STD(double, avx, , add)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256d c0 = _mm256_add_pd(left[i].y[0], right[i].y[0]);
register __m256d c1 = _mm256_add_pd(left[i].y[1], right[i].y[1]);
register __m256d c2 = _mm256_add_pd(left[i].y[0], right[i].y[0]);
register __m256d c3 = _mm256_add_pd(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
ans[i].y[2] = c2;
ans[i].y[3] = c3;
}
}
NIF_MK_PROC_DECL_STD(double, 512, , add)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m512d c0 = _mm512_add_pd(_mm512_castps_pd( left[i].z[0]), _mm512_castps_pd( right[i].z[0]));
register __m512d c1 = _mm512_add_pd(_mm512_castps_pd( left[i].z[1]), _mm512_castps_pd( right[i].z[1]));
ans[i].z[0] = c0;
ans[i].z[1] = c1;
}
}
NIF_MK_PROC_DECL_STD(float, sse, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m128 c0 = _mm_sub_ps(left[i].x[0], right[i].x[0]);
register __m128 c1 = _mm_sub_ps(left[i].x[1], right[i].x[1]);
register __m128 c2 = _mm_sub_ps(left[i].x[2], right[i].x[2]);
register __m128 c3 = _mm_sub_ps(left[i].x[3], right[i].x[3]);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
}
}
NIF_MK_PROC_DECL_STD(float, avx, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
NIF_MK_PROC_DECL_STD(float, 512, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
NIF_MK_PROC_DECL_STD(double, sse, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m128d c0 = _mm_sub_pd(left[i].x[0], right[i].x[0]);
register __m128d c1 = _mm_sub_pd(left[i].x[1], right[i].x[1]);
register __m128d c2 = _mm_sub_pd(left[i].x[2], right[i].x[2]);
register __m128d c3 = _mm_sub_pd(left[i].x[3], right[i].x[3]);
register __m128d c4 = _mm_sub_pd(left[i].x[4], right[i].x[4]);
register __m128d c5 = _mm_sub_pd(left[i].x[5], right[i].x[5]);
register __m128d c6 = _mm_sub_pd(left[i].x[6], right[i].x[6]);
register __m128d c7 = _mm_sub_pd(left[i].x[7], right[i].x[7]);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
ans[i].x[4] = c4;
ans[i].x[5] = c5;
ans[i].x[6] = c6;
ans[i].x[7] = c7;
}
}
NIF_MK_PROC_DECL_STD(double, avx, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
NIF_MK_PROC_DECL_STD(double, 512, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
NIF_MK_PROC_DECL_STD(float, sse, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(scalar.f32[0]);
for (size_t i = 0; i < count_1; i++) {
register __m128 c0 = _mm_mul_ps(right[i].x[0], vec);
register __m128 c1 = _mm_mul_ps(right[i].x[1], vec);
register __m128 c2 = _mm_mul_ps(right[i].x[2], vec);
register __m128 c3 = _mm_mul_ps(right[i].x[3], vec);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
}
}
NIF_MK_PROC_DECL_STD(float, avx, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(&scalar.f32[0]);
for (size_t i = 0; i < count_1; i++) {
register __m256 c0 = _mm256_mul_ps(right[i].y[0], vec);
register __m256 c1 = _mm256_mul_ps(right[i].y[1], vec);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
NIF_MK_PROC_DECL_STD(float, 512, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(scalar.m128);
for (size_t i = 0; i < count_1; i++) {
register __m512 c0 = _mm512_mul_ps(right[i].z[0], vec);
ans[i].z[0] = c0;
}
}
NIF_MK_PROC_DECL_STD(double, sse, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, double);
register __m128d vec = _mm_set1_pd(scalar.f64[0]);
for (size_t i = 0; i < count_1; i++) {
register __m128d c0 = _mm_mul_pd(right[i].x[0], vec);
register __m128d c1 = _mm_mul_pd(right[i].x[1], vec);
register __m128d c2 = _mm_mul_pd(right[i].x[2], vec);
register __m128d c3 = _mm_mul_pd(right[i].x[3], vec);
register __m128d c4 = _mm_mul_pd(right[i].x[4], vec);
register __m128d c5 = _mm_mul_pd(right[i].x[5], vec);
register __m128d c6 = _mm_mul_pd(right[i].x[6], vec);
register __m128d c7 = _mm_mul_pd(right[i].x[7], vec);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
ans[i].x[4] = c4;
ans[i].x[5] = c5;
ans[i].x[6] = c6;
ans[i].x[7] = c7;
}
}
NIF_MK_PROC_DECL_STD(double, avx, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(&scalar.f64[0]);
for (size_t i = 0; i < count_1; i++) {
register __m256d c0 = _mm256_mul_pd(right[i].y[0], vec);
register __m256d c1 = _mm256_mul_pd(right[i].y[1], vec);
register __m256d c2 = _mm256_mul_pd(right[i].y[2], vec);
register __m256d c3 = _mm256_mul_pd(right[i].y[3], vec);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
ans[i].y[2] = c2;
ans[i].y[3] = c3;
}
}
NIF_MK_PROC_DECL_STD(double, 512, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(scalar.f64[0]);
for (size_t i = 0; i < count_1; i++) {
register __m512d c0 = _mm512_mul_pd(right[i].z[0], vec);
register __m512d c1 = _mm512_mul_pd(right[i].z[1], vec);
ans[i].z[0] = c0;
ans[i].z[1] = c1;
}
}
#endif
src/Nitload.Native/nif-src/m.solve.x86.c
+184
View File
@@ -0,0 +1,184 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "nif_intrinsic.h"
#include "nif_intrinsic.std.h"
#include "nif_intrinsic.context.x86.h"
#include <math.h>
static const float quality_threshold = 1e-6f;
NIF_MK_PROC_DECL_STD(float, sse, , try_plu)
{
__mk_param_std(LEFT_RIGHT_ANS_SCALAR, float);
/* Aliases */
MAT_KERNEL(float) *upper = left; /* U */
MAT_KERNEL(float) *lower = right; /* L (unit diag) */
MAT_KERNEL(float) *source = ans; /* A */
unsigned char *pivot = (unsigned char *)&scalar.i64[0]; /* length = 4 */
int *success = (int *)&scalar.i64[1];
/* Init permutation and outputs */
pivot[0] = 0;
pivot[1] = 1;
pivot[2] = 2;
pivot[3] = 3;
*success = 1;
/* Working copy of A in column-major a[col][row], load via SIMD view */
MAT_KERNEL(float) a;
const __m128 z = _mm_setzero_ps();
for (int c = 0; c < 4; ++c) {
__m128 col = _mm_load_ps((float *)&source->x[c]);
_mm_storeu_ps((float *)&a.x[c], col);
_mm_store_ps((float *)&upper->x[c], z);
_mm_store_ps((float *)&lower->x[c], z);
}
const float eps = 1e-12f;
for (int k = 0; k < 4; ++k) {
/* Pivot search on column k, rows k..3 */
int p = k;
float maxabs = fabsf(a.m[k][p]);
for (int r = k + 1; r < 4; ++r) {
float v = fabsf(a.m[k][r]);
if (v > maxabs) {
maxabs = v;
p = r;
}
}
if (maxabs < eps) {
*success = 0;
/* leave outputs zeroed */
return;
}
/* Swap rows if needed */
if (p != k) {
for (int c = 0; c < 4; ++c) {
float tmp = a.m[c][k];
a.m[c][k] = a.m[c][p];
a.m[c][p] = tmp;
}
unsigned char tmpi = pivot[k];
pivot[k] = pivot[p];
pivot[p] = tmpi;
}
float pivot_val = a.m[k][k];
/* Eliminate below pivot: store multipliers in L column k */
for (int r = k + 1; r < 4; ++r) {
float factor = a.m[k][r] / pivot_val;
lower->m[k][r] = factor;
for (int c = k; c < 4; ++c) {
a.m[c][r] -= factor * a.m[c][k];
}
}
/* Set U row k (columns k..3) and L diagonal */
lower->m[k][k] = 1.0f;
for (int c = k; c < 4; ++c) {
upper->m[c][k] = a.m[c][k];
}
}
/* Ensure remaining diagonal ones for L */
for (int d = 0; d < 4; ++d) {
if (lower->m[d][d] == 0.0f)
lower->m[d][d] = 1.0f;
}
}
#define DOT_F32_SSE(a, b, ans) \
do { \
__m128 _p = _mm_mul_ps((a), (b)); /* p0 p1 p2 p3 */ \
__m128 _t = _mm_shuffle_ps(_p, _p, _MM_SHUFFLE(2, 3, 0, 1)); \
_p = _mm_add_ps(_p, _t); \
_t = _mm_movehl_ps(_t, _p); \
ans = _mm_cvtss_f32(_mm_add_ss(_p, _t)); \
} while (0)
#define FIND_MAX_F32_SSE(a, b) \
do { \
__m128 _t = _mm_movehl_ps(a, a); \
a = _mm_max_ps(a, _t); \
_t = _mm_shuffle_ps(a, a, _MM_SHUFFLE(2, 3, 0, 1)); \
b = _mm_max_ps(a, _t); \
} while (0)
/*
* layout of scalarpack:
*
*/
NIF_MK_PROC_DECL_STD(float, sse, , quality)
{
__mk_param_std(RIGHT_ANS, float);
register __m128 tiny = _mm_set1_ps(1e-6f);
register __m128 col0 = _mm_load_ps((float *)&right->x[0]);
register __m128 col1 = _mm_load_ps((float *)&right->x[1]);
register __m128 col2 = _mm_load_ps((float *)&right->x[2]);
register __m128 col3 = _mm_load_ps((float *)&right->x[3]);
NIF_ALIGNAS(16) float self_dot_src[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
// try in reigister mix later
DOT_F32_SSE(col0, col0, self_dot_src[0]);
DOT_F32_SSE(col1, col1, self_dot_src[1]);
DOT_F32_SSE(col2, col2, self_dot_src[2]);
DOT_F32_SSE(col3, col3, self_dot_src[3]);
register __m128 self_dot = _mm_load_ps(self_dot_src);
register __m128 max;
FIND_MAX_F32_SSE(self_dot, max);
//float norm = _mm_cvtss_f32(max);
// min on cross
register __m128 col_shuff_1 = _mm_shuffle_ps(col1, col1, _MM_SHUFFLE(0, 3, 2, 1));
register __m128 col_shuff_2 = _mm_shuffle_ps(col2, col2, _MM_SHUFFLE(1, 0, 3, 2));
register __m128 col_shuff_3 = _mm_shuffle_ps(col3, col3, _MM_SHUFFLE(2, 1, 0, 3));
register __m128 min = _mm_min_ps(col0, col_shuff_1);
register __m128 min0 = _mm_min_ps(col_shuff_2, col_shuff_3);
min = _mm_min_ps(min, min0);
float quality = _mm_cvtss_f32(_mm_div_ps(min, _mm_add_ps(max, tiny)));
ans->m[0][0] = quality;
if (quality < quality_threshold) {
return;
}
NIF_ALIGNAS(16) float cross_dot_src[2][4] = { 0 };
DOT_F32_SSE(col0, col1, cross_dot_src[0][0]);
DOT_F32_SSE(col0, col2, cross_dot_src[0][1]);
DOT_F32_SSE(col0, col3, cross_dot_src[0][2]);
DOT_F32_SSE(col1, col2, cross_dot_src[0][3]);
DOT_F32_SSE(col1, col3, cross_dot_src[1][0]);
DOT_F32_SSE(col2, col3, cross_dot_src[1][1]);
register __m128 cross_dot_0 = _mm_load_ps(cross_dot_src[0]);
register __m128 cross_dot_1 = _mm_load_ps(cross_dot_src[1]);
register __m128 base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(1, 0, 0, 0));
register __m128 base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 2, 1));
register __m128 relation_0 =
_mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(2, 3, 2, 1));
base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 3, 3));
register __m128 relation_1 =
_mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
relation_0 = _mm_max_ps(relation_0, relation_1);
FIND_MAX_F32_SSE(relation_0, relation_0);
ans->m[0][1] = _mm_cvtss_f32(relation_0);
}
#endif
src/Nitload.Native/nif-src/m.transpose.x86.c
+56
View File
@@ -0,0 +1,56 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "nif_intrinsic.h"
#include "nif_intrinsic.context.x86.h"
#include "nif_intrinsic.std.h"
// Matrix transpose implementations
NIF_MK_PROC_DECL_STD(float, sse, , transpose)
{
__mk_param_std(RIGHT_ANS, float);
register __m128 col0 = right->x[0];
register __m128 col1 = right->x[1];
register __m128 col2 = right->x[2];
register __m128 col3 = right->x[3];
register __m128 tmp0 = _mm_unpacklo_ps(col0, col1);
register __m128 tmp1 = _mm_unpackhi_ps(col0, col1);
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
ans->x[0] = _mm_movelh_ps(tmp0, tmp2);
ans->x[1] = _mm_movehl_ps(tmp2, tmp0);
ans->x[2] = _mm_movelh_ps(tmp1, tmp3);
ans->x[3] = _mm_movehl_ps(tmp3, tmp1);
}
NIF_MK_PROC_DECL_STD(float, avx, , transpose)
{
__mk_param_std(RIGHT_ANS, float);
register __m256 col_low = right->y[0];
register __m256 col_high = right->y[1];
register __m256 unpack_low = _mm256_unpacklo_ps(col_low, col_high);
register __m256 unpack_high = _mm256_unpackhi_ps(col_low, col_high);
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
ans->y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
ans->y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
NIF_MK_PROC_DECL_STD(float, 512, , transpose)
{
__mk_param_std(RIGHT_ANS, float);
register __m512 matrix = right->z[0];
const __m512i transpose_indices =
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
ans->z[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
#endif
src/Nitload.Native/nif-src/mk.fma.pack.c
+288
View File
@@ -0,0 +1,288 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "nif_intrinsic.context.x86.h"
/*
* pack left matrix data to L4 buffer
*
* CONVENSIONS:
*
* buffer_1 is pointer to original colum,
* offset_1 is the offset to the original column,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the column
*
* scalar pack carries column length and offset data:
* [column length, empty, empty, empty]
*/
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
// for SSE branch, no need for rounding, a direct copy is sufficient as long as the buffer is large enough to hold the packed data
memcpy(ans, right + offset_1, sizeof(MAT_KERNEL(float)) * count_2);
}
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i = 0;
for (; i < (count_1 & (~0x1)); i += 2) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 ker_1_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_1_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if (count_1 & 0x1) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i = 0;
register __m512 ker_0;
register __m512 ker_1;
register __m512 ker_2;
register __m512 ker_3;
register __m512 col_0;
register __m512 col_1;
register __m512 col_2;
register __m512 col_3;
for (; i < (count_1 & (~0x3)); i += 4) {
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
ker_2 = _mm512_load_ps((float *)&right[offset_1 + i].z[2]);
ker_3 = _mm512_load_ps((float *)&right[offset_1 + i].z[3]);
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
}
switch (count_1 & 0x3) {
case 1:
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
col_0 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 0));
col_1 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 1));
col_2 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 2));
col_3 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 3));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 2:
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
col_0 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 0));
col_1 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 1));
col_2 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 0));
col_3 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 1));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 3:
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
ker_2 = _mm512_load_ps((float *)&right[offset_1 + i].z[2]);
ker_3 = _mm512_setzero_ps();
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
default:
break;
}
}
/*
* pack right matrix data to L4 buffer
*
* CONVENSIONS:
*
* buffer_1 is pointer to original row,
* offset_1 is the offset to the original row,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the row
*
* scalar pack carries column length and offset data:
* [column length, empty, empty, empty]
*/
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_right)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
int col_length = scalar.i32[0];
for (size_t i = 0; i < count_1; i++) {
ans[i] = right[(offset_1 + i) * col_length];
}
}
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_right)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
int col_length = scalar.i32[0];
size_t i = 0;
for (; i < (count_1 & (~0x1)); i += 2) {
register __m256 ker_0_left =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[0]);
register __m256 ker_0_right =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[1]);
register __m256 ker_1_left =
_mm256_load_ps((float *)&right[(offset_1 + i + 1) * col_length].y[0]);
register __m256 ker_1_right =
_mm256_load_ps((float *)&right[(offset_1 + i + 1) * col_length].y[1]);
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if (count_1 & 0x1) {
register __m256 ker_0_left =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[0]);
register __m256 ker_0_right =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_right)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
int col_length = scalar.i32[0];
size_t i = 0;
register __m512 ker_0;
register __m512 ker_1;
register __m512 ker_2;
register __m512 ker_3;
register __m512 col_0;
register __m512 col_1;
register __m512 col_2;
register __m512 col_3;
for (; i < (count_1 & (~0x3)); i += 4) {
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]);
ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[0]);
ker_2 = _mm512_load_ps((float *)&right[(offset_1 + i + 2) * col_length].z[0]);
ker_3 = _mm512_load_ps((float *)&right[(offset_1 + i + 3) * col_length].z[0]);
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
}
switch (count_1 & 0x3) {
case 1:
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i) * col_length].z[0]);
col_0 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 0));
col_1 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 1));
col_2 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 2));
col_3 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 3));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 2:
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]);
ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[1]);
col_0 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 0));
col_1 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 1));
col_2 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 0));
col_3 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 1));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 3:
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]);
ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[1]);
ker_2 = _mm512_load_ps((float *)&right[(offset_1 + i + 2) * col_length].z[2]);
ker_3 = _mm512_setzero_ps();
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
default:
break;
}
}
#endif
src/Nitload.Native/nif-src/mk.fma.x86.c
+375
View File
@@ -0,0 +1,375 @@
#include "nif_target.h"
#include <immintrin.h>
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
// #include "nif_intrinsic.h"
#include "nif_template.fma.x86.h"
#include "nif_intrinsic.std.h"
#include "nif_intrinsic.context.x86.h"
#include <stdio.h>
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
NIF_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m128 c0 = _mm_load_ps((float *)&ans_cur->x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans_cur->x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans_cur->x[3]);
register __m128 bcol;
for (size_t z = 0; z < count_0; z++) {
NIF_MAT_KERNEL_LOCAL const *right_base = right;
NIF_MAT_KERNEL_LOCAL const *left_base = left;
const __m128 a0 = _mm_load_ps((float *)&left_base->x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_base->x[1]);
const __m128 a2 = _mm_load_ps((float *)&left_base->x[2]);
const __m128 a3 = _mm_load_ps((float *)&left_base->x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right_base->x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right_base->x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right_base->x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right_base->x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
}
_mm_store_ps((float *)&ans_cur->x[0], c0);
_mm_store_ps((float *)&ans_cur->x[1], c1);
_mm_store_ps((float *)&ans_cur->x[2], c2);
_mm_store_ps((float *)&ans_cur->x[3], c3);
}
NIF_MK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < count_0; z++) {
NIF_MAT_KERNEL_LOCAL const *b_base = right;
NIF_MAT_KERNEL_LOCAL const *a_base = left;
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
register __m256 b2 = _mm256_load_ps((float *)&b_base->y[2]);
register __m256 b3 = _mm256_load_ps((float *)&b_base->y[3]);
register __m256 a0 = _mm256_load_ps((float *)&a_base->y[0]);
register __m256 a1 = _mm256_load_ps((float *)&a_base->y[1]);
register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
NIF_MK_FMA_F32_AVX2_CYCLE(0);
NIF_MK_FMA_F32_AVX2_CYCLE(1);
NIF_MK_FMA_F32_AVX2_CYCLE(2);
NIF_MK_FMA_F32_AVX2_CYCLE(3);
}
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
_mm256_store_ps((float *)&(ans_cur->y[2]), c2);
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
}
NIF_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < count_0; z += 2) {
const NIF_MAT_KERNEL_LOCAL *a_base = left + z;
const NIF_MAT_KERNEL_LOCAL *b_base = right + z;
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
register __m256 b2 = _mm256_load_ps((float *)&b_base->y[2]);
register __m256 b3 = _mm256_load_ps((float *)&b_base->y[3]);
register __m256 a0 = _mm256_load_ps((float *)&a_base->y[0]);
register __m256 a1 = _mm256_load_ps((float *)&a_base->y[1]);
register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
NIF_MK_FMA_F32_AVX2_CYCLE(0);
NIF_MK_FMA_F32_AVX2_CYCLE(1);
NIF_MK_FMA_F32_AVX2_CYCLE(2);
NIF_MK_FMA_F32_AVX2_CYCLE(3);
}
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
_mm256_store_ps((float *)&(ans_cur->y[2]), c2);
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
}
NIF_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m512 c0 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c1 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]);
for (size_t z = 0; z < count_0; z += 4) {
const NIF_MAT_KERNEL_LOCAL *a_base = left + z;
const NIF_MAT_KERNEL_LOCAL *b_base = right + z;
register __m512 b0 = _mm512_load_ps((float *)&b_base->z[0]);
register __m512 b1 = _mm512_load_ps((float *)&b_base->z[1]);
register __m512 b2 = _mm512_load_ps((float *)&b_base->z[2]);
register __m512 b3 = _mm512_load_ps((float *)&b_base->z[3]);
register __m512 a0 = _mm512_load_ps((float *)&a_base->z[0]);
register __m512 a1 = _mm512_load_ps((float *)&a_base->z[1]);
register __m512 a2 = _mm512_load_ps((float *)&a_base->z[2]);
register __m512 a3 = _mm512_load_ps((float *)&a_base->z[3]);
NIF_MK_FMA_F32_512_CYCLE(0);
NIF_MK_FMA_F32_512_CYCLE(1);
NIF_MK_FMA_F32_512_CYCLE(2);
NIF_MK_FMA_F32_512_CYCLE(3);
}
_mm512_store_ps((float *)&ans_cur->z[0], c0);
_mm512_store_ps((float *)&ans_cur->z[1], c1);
_mm512_store_ps((float *)&ans_cur->z[2], c2);
_mm512_store_ps((float *)&ans_cur->z[3], c3);
}
NIF_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m128d c0_lo = ans_cur->x[0];
register __m128d c0_hi = ans_cur->x[1];
register __m128d c1_lo = ans_cur->x[2];
register __m128d c1_hi = ans_cur->x[3];
register __m128d c2_lo = ans_cur->x[4];
register __m128d c2_hi = ans_cur->x[5];
register __m128d c3_lo = ans_cur->x[6];
register __m128d c3_hi = ans_cur->x[7];
for (size_t z = 0; z < count_0; z++) {
NIF_MAT_KERNEL_LOCAL const *right_base = right;
NIF_MAT_KERNEL_LOCAL const *left_base = left;
const double *right_values = &right_base->m[0][0];
const double *left_values = &left_base->m[0][0];
NIF_MK_FMA_F64_SSE_PHASE(0);
NIF_MK_FMA_F64_SSE_PHASE(1);
NIF_MK_FMA_F64_SSE_PHASE(2);
NIF_MK_FMA_F64_SSE_PHASE(3);
}
ans_cur->x[0] = c0_lo;
ans_cur->x[1] = c0_hi;
ans_cur->x[2] = c1_lo;
ans_cur->x[3] = c1_hi;
ans_cur->x[4] = c2_lo;
ans_cur->x[5] = c2_hi;
ans_cur->x[6] = c3_lo;
ans_cur->x[7] = c3_hi;
}
NIF_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]);
register __m256d bcol;
for (size_t z = 0; z < count_0; z++) {
NIF_MAT_KERNEL_LOCAL const *right_base = right;
NIF_MAT_KERNEL_LOCAL const *left_base = left;
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_base->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_base->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_base->y[0]);
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_base->y[1]);
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_base->y[2]);
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_base->y[3]);
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
}
_mm256_store_pd((double *)&ans_cur->y[0], c0);
_mm256_store_pd((double *)&ans_cur->y[1], c1);
_mm256_store_pd((double *)&ans_cur->y[2], c2);
_mm256_store_pd((double *)&ans_cur->y[3], c3);
}
NIF_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]);
register __m256d bcol;
for (size_t z = 0; z < count_0; z++) {
NIF_MAT_KERNEL_LOCAL const *right_base = right;
NIF_MAT_KERNEL_LOCAL const *left_base = left;
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_base->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_base->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_base->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_base->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_base->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_base->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
}
_mm256_store_pd((double *)&ans_cur->y[0], c0);
_mm256_store_pd((double *)&ans_cur->y[1], c1);
_mm256_store_pd((double *)&ans_cur->y[2], c2);
_mm256_store_pd((double *)&ans_cur->y[3], c3);
}
NIF_MK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m512d c0 = _mm512_load_pd((double *)&ans_cur->z[0]);
register __m512d c1 = _mm512_load_pd((double *)&ans_cur->z[1]);
register __m512d c2 = _mm512_load_pd((double *)&ans_cur->z[2]);
register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]);
for (size_t z = 0; z < count_0; z += 2) {
NIF_MAT_KERNEL_LOCAL const *right_base = right + z;
NIF_MAT_KERNEL_LOCAL const *left_base = left + z;
register __m512d a0 = _mm512_load_pd((double *)&left_base->z[0]);
register __m512d a1 = _mm512_load_pd((double *)&left_base->z[1]);
register __m512d a2 = _mm512_load_pd((double *)&left_base->z[2]);
register __m512d a3 = _mm512_load_pd((double *)&left_base->z[3]);
register __m512d b0 = _mm512_load_pd((double *)&right_base->z[0]);
register __m512d b1 = _mm512_load_pd((double *)&right_base->z[1]);
register __m512d b2 = _mm512_load_pd((double *)&right_base->z[2]);
register __m512d b3 = _mm512_load_pd((double *)&right_base->z[3]);
NIF_MK_FMA_F64_AVX2_CYCLE(0);
NIF_MK_FMA_F64_AVX2_CYCLE(1);
NIF_MK_FMA_F64_AVX2_CYCLE(2);
NIF_MK_FMA_F64_AVX2_CYCLE(3);
}
_mm512_store_pd((double *)&ans_cur->z[0], c0);
_mm512_store_pd((double *)&ans_cur->z[1], c1);
_mm512_store_pd((double *)&ans_cur->z[2], c2);
_mm512_store_pd((double *)&ans_cur->z[3], c3);
}
#endif
src/Nitload.Native/nif-src/mk.p_scan.x86.c
+50
View File
@@ -0,0 +1,50 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "nif_internal.h"
#include "nif_intrinsic.context.x86.h"
#include "nif_intrinsic.std.h"
#include "nif_intrinsic.matkernel.h"
#include <emmintrin.h>
#include <immintrin.h>
#include <pmmintrin.h>
NIF_MK_PROC_DECL_STD(float, sse, , k_quality)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
}
INTERNAL NIF_STD_TARGET_ATTR(sse, ) void kp_check_f32(mat_kernel *kernel)
{
typedef MAT_KERNEL(float) NIF_MAT;
NIF_MAT const *k = (NIF_MAT const *)kernel;
float cross[4];
cross[0] = k->m[0][0];
cross[1] = k->m[1][1];
cross[2] = k->m[2][2];
cross[3] = k->m[3][3];
float evaluation[4];
register __m128 cross_vec = _mm_loadu_ps((float *)cross);
register __m128 abs_mask = _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff));
cross_vec = _mm_and_ps(cross_vec, abs_mask);
register __m128 average_vec = _mm_hadd_ps(cross_vec, cross_vec);
average_vec = _mm_hadd_ps(average_vec, average_vec);
register __m128 quarter_vec = _mm_set1_ps(0.25f);
average_vec = _mm_mul_ps(average_vec, quarter_vec);
register __m128 stddev_vec = _mm_sub_ps(cross_vec, average_vec);
stddev_vec = _mm_mul_ps(stddev_vec, stddev_vec);
stddev_vec = _mm_hadd_ps(stddev_vec, stddev_vec);
stddev_vec = _mm_hadd_ps(stddev_vec, stddev_vec);
stddev_vec = _mm_mul_ps(stddev_vec, quarter_vec);
stddev_vec = _mm_sqrt_ps(stddev_vec);
evaluation[0] = _mm_cvtss_f32(average_vec);
evaluation[1] = _mm_cvtss_f32(stddev_vec);
}
#endif
src/Nitload.Native/nif-src/s.factorial.x86.c
+33
View File
@@ -0,0 +1,33 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "nif_internal.h"
#include "nif_intrinsic.h"
static __m128i facIncrement;
static __m128i facBase;
INTERNAL uint64_t NIF_DECL factorial_simd_sse(int n)
{
facIncrement = _mm_set1_epi32(4);
facBase = _mm_setr_epi32(1, 2, 3, 4);
__m128i baseResult = facBase;
__m128i facBaseLocal = facBase;
const int cycle = (n - 3) >> 2;
const uint32_t rest = (cycle << 2) + 1;
for (size_t i = 0; i < cycle; i++) {
facBaseLocal = _mm_add_epi32(facBaseLocal, facIncrement);
baseResult = _mm_mullo_epi32(baseResult, facBaseLocal);
}
NIF_ALIGNAS(16) int32_t lanes[4];
_mm_store_si128((__m128i *)lanes, baseResult);
uint64_t result = (uint64_t)lanes[0] * (uint64_t)lanes[1] * (uint64_t)lanes[2] *
(uint64_t)lanes[3];
for (uint64_t i = rest; i < n; i++) {
result *= i;
}
return result;
}
#endif
src/Nitload.Native/nif-src/s.hash.x86.c
+50
View File
@@ -0,0 +1,50 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "NitloadFrameWork.h"
INTERNAL int NIF_DECL city_hash_simplify_sse(byte const *str, int const length)
{
uint32_t hash = length, size = length << 1;
byte const *end = str + size;
byte const *ptr = str;
if (size < 4) {
for (; ptr < end; ptr += 2) {
hash = _mm_crc32_u16(hash, *ptr);
}
} else if (size < 16) {
for (; ptr < end; ptr += 4) {
hash = _mm_crc32_u32(hash, ((uint32_t const *)ptr)[0]);
}
for (; ptr < end; ptr++) {
hash = _mm_crc32_u16(hash, *ptr);
}
} else {
byte const *last_128 = end - 16;
byte const *last_32 = end - 4;
uint64_t hash64 = size;
for (; ptr < last_128; ptr += 16) {
__m128i vec = _mm_loadu_si128((__m128i const *)ptr);
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 0));
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 1));
}
hash = _mm_crc32_u8(hash, hash64);
hash = _mm_crc32_u8(hash, hash64 >> 32);
for (; ptr < last_32; ptr += 4) {
hash = _mm_crc32_u32(hash, ((uint32_t const *)ptr)[0]);
}
for (; ptr < end; ptr += 2) {
hash = _mm_crc32_u8(hash, *ptr);
}
}
hash ^= hash >> 16;
hash *= 0x85ebca6b;
hash ^= hash >> 13;
hash *= 0xc2b2ae35;
hash ^= hash >> 16;
return hash;
}
#endif
src/Nitload.Native/nif-src/simdid.x86.c
+329
View File
@@ -0,0 +1,329 @@
#include "nif_target.h"
/* clang-format off */
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "nif_intrinsic.h"
#include "NitloadFrameWork.h"
static SIMDID global_simdid = { 0 };
#include "nif_intrinsic.context.x86.h"
#include <immintrin.h>
#include <string.h>
#if defined(_MSC_VER)
#include <intrin.h>
#else
#include <cpuid.h>
#endif
static void get_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx)
{
#if defined(_MSC_VER)
int regs[4];
__cpuidex(regs, leaf, subleaf);
*eax = regs[0];
*ebx = regs[1];
*ecx = regs[2];
*edx = regs[3];
#else
unsigned int a, b, c, d;
__cpuid_count((unsigned int)leaf, (unsigned int)subleaf, a, b, c, d);
*eax = (int)a;
*ebx = (int)b;
*ecx = (int)c;
*edx = (int)d;
#endif
}
/* XCR0 is per logical processor. Reading it after thread affinity is applied
* verifies that the OS saves/restores the register state needed by this core. */
static unsigned long long read_xcr0(void)
{
#if defined(_MSC_VER)
return _xgetbv(0);
#else
return __builtin_ia32_xgetbv(0);
#endif
}
static int os_has_xsave_state(unsigned long long required)
{
return (read_xcr0() & required) == required;
}
#define XCR0_X87 (1ull << 0)
#define XCR0_XMM (1ull << 1)
#define XCR0_YMM (1ull << 2)
#define XCR0_OPMASK (1ull << 5)
#define XCR0_ZMM_HI256 (1ull << 6)
#define XCR0_HI16_ZMM (1ull << 7)
#define XCR0_AVX_STATE (XCR0_X87 | XCR0_XMM | XCR0_YMM)
#define XCR0_AVX512_STATE \
(XCR0_AVX_STATE | XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)
static void decode_family_model(int cpuid1_eax, int *family, int *model)
{
int family_value = 0;
int model_value = 0;
const int base_fam = (cpuid1_eax >> 8) & 0xF;
const int ext_fam = (cpuid1_eax >> 20) & 0xFF;
const int base_model = (cpuid1_eax >> 4) & 0xF;
const int ext_model = (cpuid1_eax >> 16) & 0xF;
if (base_fam == 0xF) {
family_value = base_fam + ext_fam;
} else {
family_value = base_fam;
}
if ((base_fam == 0x6) || (base_fam == 0xF)) {
model_value = (ext_model << 4) | base_model;
} else {
model_value = base_model;
}
*family = family_value;
*model = model_value;
}
/* AVX-512 execution-width classification for AMD.
* Family 19h (Zen4) uses a 256-bit dual-pumped path.
* Family 1Ah models 00h-0Fh are documented for the full-width EPYC 9005
* implementation. Models 10h-1Fh are intentionally UNKNOWN: that CPUID
* range is used by both full-width EPYC 9005 Zen5c and dual-pumped EPYC
* 8005 products, and CPUID has no architectural execution-width bit. */
static uint64_t amd_avx512_impl(int family, int model)
{
if (family == 0x19) {
return SIMDID_AVX512_IMPL_DUAL;
}
if (family == 0x1A) {
if (model >= 0x00 && model <= 0x0F) {
return SIMDID_AVX512_IMPL_NATIVE;
}
}
return SIMDID_AVX512_IMPL_UNKNOWN;
}
/* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units).
* - AMD Zen1 (family 0x17, model < 0x30) and Zen+ (0x18-0x2F) use a split/glued path.
* - Hygon Dhyana is architecturally Zen1-based; always split.
* - AMD Zen2+ (family 0x17 model >= 0x30, family 0x19+) have native 256-bit execution. */
static int is_amd_glue_ymm(const char *vendor, int cpuid1_eax)
{
int family = 0;
int model = 0;
if (strcmp(vendor, "HygonGenuine") == 0) {
return 1;
}
if (strcmp(vendor, "AuthenticAMD") != 0) {
return 0;
}
decode_family_model(cpuid1_eax, &family, &model);
return (family == 0x17) && (model < 0x30);
}
NIF_API SIMDID NIF_DECL nif_get_simd_level_local()
{
SIMDID simdid = { 0 };
int eax = 0;
int ebx = 0;
int ecx = 0;
int edx = 0;
int family = 0;
int model = 0;
get_cpuid(0, 0, &eax, &ebx, &ecx, &edx);
const int max_basic_leaf = eax;
char vendor[13];
memcpy(vendor + 0, &ebx, 4);
memcpy(vendor + 4, &edx, 4);
memcpy(vendor + 8, &ecx, 4);
vendor[12] = '\0';
const int is_intel = (strcmp(vendor, "GenuineIntel") == 0);
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0);
const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0);
const int is_centaur = (strcmp(vendor, "CentaurHauls") == 0);
const int is_zhaoxin = (strcmp(vendor, " Shanghai ") == 0);
const int is_via = is_centaur || is_zhaoxin;
get_cpuid(1, 0, &eax, &ebx, &ecx, &edx);
const int cpuid1_eax = eax;
decode_family_model(cpuid1_eax, &family, &model);
(void)model;
const int has_sse42 = (ecx & (1 << 20)) != 0;
const int has_avx = (ecx & (1 << 28)) != 0;
const int has_osxsave = (ecx & (1 << 27)) != 0;
const int has_fma = (ecx & (1 << 12)) != 0;
const int has_f16c = (ecx & (1 << 29)) != 0;
const uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
uint64_t manu_code = SIMDID_MANU_UNKNOWN;
if (is_intel) {
manu_code = SIMDID_MANU_INTEL;
} else if (is_amd) {
manu_code = SIMDID_MANU_AMD;
} else if (is_hygon) {
manu_code = SIMDID_MANU_HYGON;
} else if (is_via) {
manu_code = SIMDID_MANU_VIA;
}
const uint64_t manu = SIMDID_PACK_MANU(manu_code);
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
uint64_t avx_bits = 0;
uint64_t avx512_bits = 0;
uint64_t avx10_bits = 0;
uint64_t uarch_bits = 0;
int has_hybrid = 0;
if (has_sse42) {
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE);
}
if (has_avx && has_osxsave && os_has_xsave_state(XCR0_AVX_STATE)) {
avx_bits |= SIMDID_AVX;
if (is_amd_glue_ymm(vendor, cpuid1_eax) || is_via) {
avx_bits |= SIMDID_AVX_DUAL;
}
if (max_basic_leaf >= 7) {
get_cpuid(7, 0, &eax, &ebx, &ecx, &edx);
const int has_avx2 = (ebx & (1 << 5)) != 0;
const int has_avx512f = (ebx & (1 << 16)) != 0;
const int has_avx512dq = (ebx & (1 << 17)) != 0;
const int has_avx512cd = (ebx & (1 << 28)) != 0;
const int has_avx512bw = (ebx & (1 << 30)) != 0;
const int has_avx512vl = (ebx & (1u << 31)) != 0;
const int has_avx512_vnni = (ecx & (1 << 11)) != 0;
has_hybrid = (edx & (1 << 15)) != 0;
int has_avx10 = 0;
if (eax >= 1) {
get_cpuid(7, 1, &eax, &ebx, &ecx, &edx);
has_avx10 = (edx & (1 << 19)) != 0;
}
if (has_avx2) {
avx_bits |= SIMDID_AVX2;
if (has_fma) {
avx_bits |= SIMDID_AVX_FMA;
}
if (has_f16c) {
avx_bits |= SIMDID_AVX_F16C;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2);
}
if (has_avx512f && has_avx512cd && has_avx512dq && has_avx512bw &&
has_avx512vl && os_has_xsave_state(XCR0_AVX512_STATE)) {
avx512_bits |= SIMDID_AVX512_BASE;
if (has_avx512_vnni) {
avx512_bits |= SIMDID_AVX512_VNNI;
}
if (is_intel) {
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
} else if (is_amd) {
avx512_bits |= amd_avx512_impl(family, model);
} else {
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);
if (has_avx10 && (max_basic_leaf >= 0x24)) {
int a10_eax = 0;
int a10_ebx = 0;
int a10_ecx = 0;
int a10_edx = 0;
int avx10_ver = 0;
get_cpuid(0x24, 0, &a10_eax, &a10_ebx, &a10_ecx, &a10_edx);
avx10_ver = a10_ebx & 0xFF;
if (avx10_ver >= 1) {
unsigned max_avx10_subleaf = (unsigned)a10_eax;
unsigned stored_version = (unsigned)avx10_ver;
if (stored_version > SIMDID_AVX10_VERSION_MASK) {
stored_version = SIMDID_AVX10_VERSION_MASK;
}
avx10_bits = stored_version;
if (max_avx10_subleaf >= 1) {
get_cpuid(0x24, 1, &a10_eax, &a10_ebx,
&a10_ecx, &a10_edx);
if ((a10_ecx & (1 << 2)) != 0) {
avx10_bits |= SIMDID_AVX10_VNNI_INT;
}
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10);
}
}
}
}
}
if (has_hybrid && (max_basic_leaf >= 0x1A)) {
int h_eax = 0;
int h_ebx = 0;
int h_ecx = 0;
int h_edx = 0;
get_cpuid(0x1A, 0, &h_eax, &h_ebx, &h_ecx, &h_edx);
{
const unsigned core_type = (unsigned)(h_eax >> 24) & 0xFFU;
if (core_type == 0x40U) {
uarch_bits |= SIMDID_UARCH_HYBRID_P;
} else if (core_type == 0x20U) {
uarch_bits |= SIMDID_UARCH_HYBRID_E;
}
}
} else {
uarch_bits |= SIMDID_UARCH_HYBRID_P | SIMDID_UARCH_HYBRID_E;
}
{
const int has_avx_dual = (avx_bits & SIMDID_AVX_DUAL) != 0;
const uint64_t avx512_impl = avx512_bits & SIMDID_AVX512_IMPL_MASK;
const int has_avx512_dual = avx512_impl == SIMDID_AVX512_IMPL_DUAL;
if (has_avx_dual || has_avx512_dual) {
uarch_bits |= SIMDID_UARCH_SIMD_DUAL;
}
}
{
uint64_t final_mask = root;
final_mask |= manu;
final_mask |= main_lvl;
final_mask |= SIMDID_PACK_AVX512(avx512_bits);
final_mask |= SIMDID_PACK_AVX(avx_bits);
final_mask |= SIMDID_PACK_AVX10(avx10_bits);
final_mask |= SIMDID_PACK_UARCH(uarch_bits);
simdid.mask = final_mask;
}
return simdid;
}
NIF_API SIMDID NIF_DECL nif_get_simd_level_global()
{
if (global_simdid.mask) {
return global_simdid;
}
global_simdid = nif_get_simd_level_local();
return global_simdid;
}
#endif
src/Nitload.Native/nif-src/simdid_compare.x86.c
+222
View File
@@ -0,0 +1,222 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "NitloadFrameWork.h"
#include "nif_intrinsic.context.x86.h"
/* -----------------------------------------------------------------------
* Field extraction helpers.
*
* SIMDID layout (64 bits, little-endian byte order in make_byte[]):
* [3..0] Root ISA make_byte[0] low nibble
* [7..4] Manufacture make_byte[0] high nibble
* [15..8] MainLevel make_byte[1]
* [31..16] AVX512 Feature make_byte[2..3] (16 bits)
* [39..32] AVX Feature make_byte[4]
* [47..40] AVX10 Feature make_byte[5]
* [55..48] uArchHi make_byte[6]
* [63..56] Reserved make_byte[7]
* ----------------------------------------------------------------------- */
#define FIELD_MAIN(id) ((unsigned char)(id).make_byte[1])
#define FIELD_AVX512(id) \
((uint16_t)((unsigned char)(id).make_byte[2] | ((unsigned char)(id).make_byte[3] << 8)))
#define FIELD_AVX(id) ((unsigned char)(id).make_byte[4])
#define FIELD_AVX10(id) ((unsigned char)(id).make_byte[5])
#define FIELD_UARCH(id) ((unsigned char)(id).make_byte[6])
/* Portable popcount for a 16-bit value. */
static int popcount16(uint16_t v)
{
int c = 0;
while (v) {
v &= v - 1;
c++;
}
return c;
}
/* Map AVX512_IMPL 2-bit field to a strength ordinal for tie-breaking.
* Index = impl value (0=UNKNOWN, 1=DUAL, 2=NATIVE, 3=reserved).
* Strength: DUAL(0) < UNKNOWN(1) < NATIVE(2). */
static const int impl_strength_table[4] = { 1, 0, 2, 1 };
static int impl_strength(uint16_t avx512)
{
unsigned impl = (avx512 >> SIMDID_AVX512_IMPL_SHIFT) & 0x3u;
return impl_strength_table[impl];
}
/* -----------------------------------------------------------------------
* nif_get_unite_simd
*
* Produce the conservative intersection of two SIMDIDs — the strongest
* capability set that is safe to use on BOTH cores:
* - MainLevel : min (only assume what both cores support)
* - AVX512 features : bitwise AND (IMPL field: pick the weaker one)
* - AVX features : bitwise AND
* - AVX10 version : min; discrete features: bitwise AND
* - Root / Manu : keep left (must be same arch in practice)
* - uArchHi : bitwise OR (HYBRID_P/E union; SIMD_DUAL set if either core is dual)
* ----------------------------------------------------------------------- */
NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right)
{
SIMDID out;
out.mask = 0;
/* Root + Manufacture (byte 0): prefer left (same arch assumed) */
unsigned char lb0 = (unsigned char)left.make_byte[0];
unsigned char rb0 = (unsigned char)right.make_byte[0];
out.make_byte[0] = (char)(lb0 ? lb0 : rb0);
/* MainLevel (byte 1): min */
unsigned char lm = FIELD_MAIN(left);
unsigned char rm = FIELD_MAIN(right);
out.make_byte[1] = (char)(lm < rm ? lm : rm);
/* AVX512 features (bytes 2-3): AND all bits, then fix IMPL to weaker */
uint16_t l512 = FIELD_AVX512(left);
uint16_t r512 = FIELD_AVX512(right);
uint16_t merged = l512 & r512;
/* Clear the IMPL field, then set the weaker one */
int ls = impl_strength(l512);
int rs = impl_strength(r512);
int s = (ls <= rs) ? l512 : r512; /* pick the one with weaker strength ordinal */
unsigned weak_impl = (s >> SIMDID_AVX512_IMPL_SHIFT) & 0x3u;
merged &= ~(uint16_t)SIMDID_AVX512_IMPL_MASK;
merged |= (uint16_t)(weak_impl << SIMDID_AVX512_IMPL_SHIFT);
out.make_byte[2] = (char)(merged & 0xFF);
out.make_byte[3] = (char)((merged >> 8) & 0xFF);
/* AVX (byte 4): AND */
out.make_byte[4] = left.make_byte[4] & right.make_byte[4];
/* AVX10 (byte 5): minimum version and common discrete features. */
unsigned char lavx10 = FIELD_AVX10(left);
unsigned char ravx10 = FIELD_AVX10(right);
unsigned char lver = lavx10 & SIMDID_AVX10_VERSION_MASK;
unsigned char rver = ravx10 & SIMDID_AVX10_VERSION_MASK;
unsigned char common = (lavx10 & ravx10) & SIMDID_AVX10_VNNI_INT;
out.make_byte[5] = (char)((lver < rver ? lver : rver) | common);
/* uArchHi (byte 6):
* topology bits [1..0]: AND — conservative intersection (weaker topology wins)
* SIMD_DUAL bit [2]: OR — dual if either core is dual */
unsigned char lu = FIELD_UARCH(left);
unsigned char ru = FIELD_UARCH(right);
unsigned char topo = (lu & 0x3u) & (ru & 0x3u);
unsigned char dual = (lu | ru) & (unsigned char)SIMDID_UARCH_SIMD_DUAL;
out.make_byte[6] = (char)(topo | dual);
/* Reserved tail byte. */
out.make_byte[7] = 0;
return out;
}
/* -----------------------------------------------------------------------
* nif_compare_simd
*
* Compare two SIMDIDs by ISA strength. Returns:
* +1 if left is strictly stronger than right
* -1 if left is strictly weaker than right
* 0 if equal
*
* Comparison priority (most significant first):
*
* 1. MainLevel ([15..8])
* Higher value = stronger. This is the primary discriminator:
* NONE < SSE < AVX2 < AVX512 < AVX10
*
* 2. AVX10 version and discrete features ([47..40])
* Only meaningful when both MainLevel == AVX10.
* Higher versions are stronger; VNNI_INT is a tie-breaker.
*
* 3. AVX512 BASE bit (bit 4 of [31..16])
* Only meaningful when both MainLevel >= AVX512.
* BASE=1 (F+CD+VL+DQ+BW all present) > BASE=0.
*
* 4. AVX512 extension popcount
* VNNI is currently the only tracked AVX512 extension.
* The IMPL and BASE bits are masked out before counting so they don't
* inflate the extension count.
*
* 5. AVX512 IMPL strength (tie-break, performance hint)
* DUAL(weakest) < UNKNOWN < NATIVE(strongest).
* This is not ISA capability but execution-width risk.
*
* 6. uArchHi topology bits [1..0]
* Numeric value: unified(11=3) > P-core(10=2) > E-core(01=1) > unknown(00=0).
* Higher value = stronger / higher-capability core form.
*
* 7. uArchHi SIMD_DUAL (final tie-break)
* Non-dual (SIMD_DUAL=0) > dual (SIMD_DUAL=1).
* ----------------------------------------------------------------------- */
NIF_API int NIF_DECL nif_compare_simd(SIMDID left, SIMDID right)
{
/* 1. MainLevel */
unsigned lm = FIELD_MAIN(left);
unsigned rm = FIELD_MAIN(right);
if (lm != rm)
return lm > rm ? 1 : -1;
/* 2. AVX10 version and discrete features (only at AVX10). */
if (lm >= SIMDID_MAIN_LVL_AVX10) {
unsigned la = FIELD_AVX10(left);
unsigned ra = FIELD_AVX10(right);
unsigned lv = la & SIMDID_AVX10_VERSION_MASK;
unsigned rv = ra & SIMDID_AVX10_VERSION_MASK;
if (lv != rv)
return lv > rv ? 1 : -1;
int lvnni = (la & SIMDID_AVX10_VNNI_INT) != 0;
int rvnni = (ra & SIMDID_AVX10_VNNI_INT) != 0;
if (lvnni != rvnni)
return lvnni ? 1 : -1;
}
/* 3. AVX512 BASE (only when MainLevel >= AVX512) */
if (lm >= SIMDID_MAIN_LVL_AVX512) {
uint16_t l512 = FIELD_AVX512(left);
uint16_t r512 = FIELD_AVX512(right);
int lb = (l512 & SIMDID_AVX512_BASE) != 0;
int rb = (r512 & SIMDID_AVX512_BASE) != 0;
if (lb != rb)
return lb > rb ? 1 : -1;
// 4. Extension popcount (mask out BASE + IMPL + reserved low nibble)
int drop = ~(SIMDID_AVX512_BASE | SIMDID_AVX512_IMPL_MASK | 0x000Fu);
uint16_t ext_mask = drop & (l512 | r512); /* only consider bits set in either */
int lp = popcount16(l512 & ext_mask);
int rp = popcount16(r512 & ext_mask);
if (lp != rp)
return lp > rp ? 1 : -1;
/* 5. IMPL strength */
int ls = impl_strength(l512);
int rs = impl_strength(r512);
if (ls != rs)
return ls > rs ? 1 : -1;
}
/* 6. uArchHi topology bits [1..0]: unified(3) > P(2) > E(1) > unknown(0) */
unsigned lu = FIELD_UARCH(left);
unsigned ru = FIELD_UARCH(right);
unsigned lt = lu & 0x3u;
unsigned rt = ru & 0x3u;
if (lt != rt)
return lt > rt ? 1 : -1;
/* 7. uArchHi SIMD_DUAL: non-dual (SIMD_DUAL=0) > dual (SIMD_DUAL=1) */
int ld = (lu & SIMDID_UARCH_SIMD_DUAL) != 0;
int rd = (ru & SIMDID_UARCH_SIMD_DUAL) != 0;
if (ld != rd)
return ld ? -1 : 1; /* non-dual is stronger */
return 0;
}
#endif
src/Nitload.Native/nif-src/sk.fma.x86.c
+332
View File
@@ -0,0 +1,332 @@
#include "nif_target.h"
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
#include "nif_intrinsic.h"
#include "nif_intrinsic.std.h"
#include "nif_intrinsic.context.x86.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
#define GET_INDEX(col_len, x, y) (col_len * x + y)
// float
NIF_SK_PROC_DECL_STD(float, sse, , fma)
{
__sk_param_std(float);
register __m128 c0 = _mm_load_ps((float *)&ans->x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans->x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans->x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans->x[3]);
register __m128 bcol;
const __m128 a0 = _mm_load_ps((float *)&left->x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->x[2]);
const __m128 a3 = _mm_load_ps((float *)&left->x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right->x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right->x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right->x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right->x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
_mm_store_ps((float *)&ans->x[0], c0);
_mm_store_ps((float *)&ans->x[1], c1);
_mm_store_ps((float *)&ans->x[2], c2);
_mm_store_ps((float *)&ans->x[3], c3);
}
NIF_SK_PROC_DECL_STD(float, avx, , fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left->x[0]);
c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23);
register __m256 a1 = _mm256_broadcast_ps(&left->x[1]);
c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23);
register __m256 a2 = _mm256_broadcast_ps(&left->x[2]);
c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23);
register __m256 a3 = _mm256_broadcast_ps(&left->x[3]);
c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23);
_mm256_store_ps((float *)&ans->y[0], c01);
_mm256_store_ps((float *)&ans->y[1], c23);
}
NIF_SK_PROC_DECL_STD(float, avx, _fma, fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left->x[0]);
c01 = _mm256_fmadd_ps(a0, b01, c01);
c23 = _mm256_fmadd_ps(a0, b23, c23);
register __m256 a1 = _mm256_broadcast_ps(&left->x[1]);
c01 = _mm256_fmadd_ps(a1, b01, c01);
c23 = _mm256_fmadd_ps(a1, b23, c23);
register __m256 a2 = _mm256_broadcast_ps(&left->x[2]);
c01 = _mm256_fmadd_ps(a2, b01, c01);
c23 = _mm256_fmadd_ps(a2, b23, c23);
register __m256 a3 = _mm256_broadcast_ps(&left->x[3]);
c01 = _mm256_fmadd_ps(a3, b01, c01);
c23 = _mm256_fmadd_ps(a3, b23, c23);
_mm256_store_ps((float *)&ans->y[0], c01);
_mm256_store_ps((float *)&ans->y[1], c23);
}
NIF_SK_PROC_DECL_STD(float, 512, , fma)
{
__sk_param_std(float);
register __m512 c = _mm512_load_ps((float *)ans);
// full right load：[kb_col0 | kb_col1 | kb_col2 | kb_col3]
register __m512 b = _mm512_load_ps((float *)right);
// k=0: broadcast ka_col0 到4个lane，permute_ps 取 kb(0,j)
register __m512 ak;
ak = _mm512_broadcast_f32x4(left->x[0]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(0)), c);
// k=1
ak = _mm512_broadcast_f32x4(left->x[1]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(1)), c);
// k=2
ak = _mm512_broadcast_f32x4(left->x[2]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(2)), c);
// k=3
ak = _mm512_broadcast_f32x4(left->x[3]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(3)), c);
_mm512_store_ps((float *)ans, c);
}
// double
// clang-format off
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
// clang-format on
NIF_SK_PROC_DECL_STD(double, sse, , fma)
{
__sk_param_std(double);
const double *left_values = &left->m[0][0];
const double *right_values = &right->m[0][0];
register __m128d c0_lo = ans->x[0];
register __m128d c0_hi = ans->x[1];
register __m128d c1_lo = ans->x[2];
register __m128d c1_hi = ans->x[3];
register __m128d c2_lo = ans->x[4];
register __m128d c2_hi = ans->x[5];
register __m128d c3_lo = ans->x[6];
register __m128d c3_hi = ans->x[7];
DOUBLE_KERNEL_FMA_CYCLE(0);
DOUBLE_KERNEL_FMA_CYCLE(1);
DOUBLE_KERNEL_FMA_CYCLE(2);
DOUBLE_KERNEL_FMA_CYCLE(3);
ans->x[0] = c0_lo;
ans->x[1] = c0_hi;
ans->x[2] = c1_lo;
ans->x[3] = c1_hi;
ans->x[4] = c2_lo;
ans->x[5] = c2_hi;
ans->x[6] = c3_lo;
ans->x[7] = c3_hi;
}
// clang-format off
NIF_SK_PROC_DECL_STD(double, avx, , fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->y[3]);
register __m256d bcol;
const __m256d a0 = _mm256_load_pd((double *)&left->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
_mm256_store_pd((double *)&ans->y[0], c0);
_mm256_store_pd((double *)&ans->y[1], c1);
_mm256_store_pd((double *)&ans->y[2], c2);
_mm256_store_pd((double *)&ans->y[3], c3);
}
// clang-format on
NIF_SK_PROC_DECL_STD(double, avx, _fma, fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->y[3]);
register __m256d bcol;
const __m256d a0 = _mm256_load_pd((double *)&left->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
_mm256_store_pd((double *)&ans->y[0], c0);
_mm256_store_pd((double *)&ans->y[1], c1);
_mm256_store_pd((double *)&ans->y[2], c2);
_mm256_store_pd((double *)&ans->y[3], c3);
}
NIF_SK_PROC_DECL_STD(double, 512, , fma)
{
__sk_param_std(double);
register __m512d c01 = _mm512_load_pd((double *)&ans->z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->z[1]);
register __m512d a01 = _mm512_load_pd((double *)&left->z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left->z[1]);
register __m512d b0 = _mm512_broadcast_f64x4(right->y[0]);
c01 = _mm512_fmadd_pd(a01, b0, c01);
c23 = _mm512_fmadd_pd(a23, b0, c23);
register __m512d b1 = _mm512_broadcast_f64x4(right->y[1]);
c01 = _mm512_fmadd_pd(a01, b1, c01);
c23 = _mm512_fmadd_pd(a23, b1, c23);
register __m512d b2 = _mm512_broadcast_f64x4(right->y[2]);
c01 = _mm512_fmadd_pd(a01, b2, c01);
c23 = _mm512_fmadd_pd(a23, b2, c23);
register __m512d b3 = _mm512_broadcast_f64x4(right->y[3]);
c01 = _mm512_fmadd_pd(a01, b3, c01);
c23 = _mm512_fmadd_pd(a23, b3, c23);
_mm512_store_pd((double *)&ans->z[0], c01);
_mm512_store_pd((double *)&ans->z[1], c23);
}
#endif
src/Nitload.Native/nif-src/thread.linux.c
+327
View File
@@ -0,0 +1,327 @@
#define _GNU_SOURCE
#include "nif_target.h"
#if NIF_IS_OS(NIF_OS_LINUX)
#include <errno.h>
#include <inttypes.h>
#include <limits.h>
#include <linux/mempolicy.h>
#include <pthread.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
typedef int (*nif_linux_id_visitor)(int value, void *context);
typedef struct nif_linux_counter_context {
int count;
} nif_linux_counter_context;
typedef struct nif_linux_max_context {
int max_value;
} nif_linux_max_context;
typedef struct nif_linux_fill_cpu_context {
uint64_t *internal;
uint32_t *global;
int index;
} nif_linux_fill_cpu_context;
typedef struct nif_linux_fill_node_count_context {
int *count_of_each_numa;
} nif_linux_fill_node_count_context;
static int nif_linux_read_first_line(char const *path, char *buffer, size_t buffer_size)
{
FILE *file = fopen(path, "r");
if (file == NULL)
return 0;
char *line = fgets(buffer, (int)buffer_size, file);
fclose(file);
return line != NULL;
}
static int nif_linux_parse_nonnegative_int(char const **cursor, int *value)
{
char *end = NULL;
uintmax_t parsed_value = 0;
if (**cursor == '-')
return 0;
errno = 0;
parsed_value = strtoumax(*cursor, &end, 10);
if (end == *cursor || errno == ERANGE || parsed_value > (uintmax_t)INT_MAX)
return 0;
*cursor = end;
*value = (int)parsed_value;
return 1;
}
static int nif_linux_visit_id_list(char const *text, nif_linux_id_visitor visitor, void *context)
{
char const *cursor = text;
while (*cursor != '\0') {
while (*cursor == ' ' || *cursor == '\t' || *cursor == '\n' || *cursor == ',')
cursor++;
if (*cursor == '\0')
break;
int begin = 0;
if (!nif_linux_parse_nonnegative_int(&cursor, &begin))
return 0;
int finish = begin;
if (*cursor == '-') {
cursor++;
if (!nif_linux_parse_nonnegative_int(&cursor, &finish))
return 0;
}
if (finish < begin)
return 0;
for (int value = begin;; value++) {
if (!visitor(value, context))
return 0;
if (value == finish)
break;
}
}
return 1;
}
static int nif_linux_count_id(int value, void *context)
{
(void)value;
((nif_linux_counter_context *)context)->count++;
return 1;
}
static int nif_linux_track_max_id(int value, void *context)
{
nif_linux_max_context *max_context = (nif_linux_max_context *)context;
if (value > max_context->max_value)
max_context->max_value = value;
return 1;
}
static int nif_linux_fill_cpu_id(int value, void *context)
{
nif_linux_fill_cpu_context *fill_context = (nif_linux_fill_cpu_context *)context;
if (fill_context->internal != NULL)
fill_context->internal[fill_context->index] = (uint64_t)fill_context->index;
if (fill_context->global != NULL)
fill_context->global[fill_context->index] = (uint32_t)value;
fill_context->index++;
return 1;
}
static int nif_linux_count_cpu_list(char const *path)
{
char buffer[256];
if (!nif_linux_read_first_line(path, buffer, sizeof(buffer)))
return 0;
nif_linux_counter_context counter = { 0 };
if (!nif_linux_visit_id_list(buffer, nif_linux_count_id, &counter))
return 0;
return counter.count;
}
static int nif_linux_fill_node_cpu_count(int value, void *context)
{
char path[128];
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", value);
((nif_linux_fill_node_count_context *)context)->count_of_each_numa[value] =
nif_linux_count_cpu_list(path);
return 1;
}
static int nif_linux_node_exists(int numa_id)
{
char path[128];
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d", numa_id);
return access(path, F_OK) == 0;
}
static size_t nif_linux_round_up_to_page_size(size_t size)
{
int64_t page_size_value = (int64_t)sysconf(_SC_PAGESIZE);
size_t effective_page_size = (page_size_value > 0) ? (size_t)page_size_value : 4096u;
size_t rounded = size + effective_page_size - 1;
if (rounded < size)
return 0;
return rounded - (rounded % effective_page_size);
}
static int nif_linux_bind_memory_to_node(void *address, size_t size, int numa_id, uint32_t flags)
{
#if defined(SYS_mbind)
size_t bits_per_word = sizeof(unsigned long) * 8u;
size_t word_count = ((size_t)numa_id / bits_per_word) + 1u;
unsigned long *node_mask = (unsigned long *)calloc(word_count, sizeof(unsigned long));
if (node_mask == NULL)
return 0;
node_mask[(size_t)numa_id / bits_per_word] = 1ul << ((size_t)numa_id % bits_per_word);
int syscall_result = (int)syscall(SYS_mbind, address, size, MPOL_BIND | MPOL_F_STATIC_NODES,
node_mask, (unsigned long)numa_id + 1ul, flags);
free(node_mask);
return syscall_result == 0;
#else
(void)address;
(void)size;
(void)numa_id;
(void)flags;
return 0;
#endif
}
static int nif_linux_populate_writable_mapping(void *address, size_t size)
{
#if defined(MADV_POPULATE_WRITE)
if (madvise(address, size, MADV_POPULATE_WRITE) == 0)
return 1;
return errno == EINVAL;
#else
(void)address;
(void)size;
return 1;
#endif
}
/* Pin the calling thread to a single logical processor.
* core_id >= 0 : bind to that logical processor (global index, across groups).
* core_id < 0 : clear affinity constraint within the current group. */
NIF_API void NIF_DECL nif_set_thread_affinity(int core_id, int numa_id)
{
int cpu_count = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (cpu_count <= 0)
return;
cpu_set_t *set = CPU_ALLOC(cpu_count);
size_t size = CPU_ALLOC_SIZE(cpu_count);
if (set == NULL)
return;
CPU_ZERO_S(size, set);
if (core_id < 0) {
for (int i = 0; i < cpu_count; i++)
CPU_SET_S((int)i, size, set);
} else {
CPU_SET_S(core_id, size, set);
}
pthread_setaffinity_np(pthread_self(), size, set);
CPU_FREE(set);
}
NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
{
char buffer[256];
if (!nif_linux_read_first_line("/sys/devices/system/node/online", buffer, sizeof(buffer)))
return 0;
nif_linux_max_context max_context = { -1 };
if (!nif_linux_visit_id_list(buffer, nif_linux_track_max_id, &max_context) ||
max_context.max_value < 0)
return 0;
if (out_numa_count != NULL)
*out_numa_count = max_context.max_value + 1;
if (count_of_each_numa != NULL) {
for (int index = 0; index <= max_context.max_value; index++)
count_of_each_numa[index] = 0;
nif_linux_fill_node_count_context fill_context = { count_of_each_numa };
if (!nif_linux_visit_id_list(buffer, nif_linux_fill_node_cpu_count, &fill_context))
return 0;
}
return 1;
}
NIF_API int NIF_DECL nif_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global)
{
if (numa_id < 0)
return 0;
char path[128];
char buffer[256];
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa_id);
if (!nif_linux_read_first_line(path, buffer, sizeof(buffer)))
return 0;
nif_linux_fill_cpu_context fill_context = { internal, global, 0 };
return nif_linux_visit_id_list(buffer, nif_linux_fill_cpu_id, &fill_context);
}
NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
{
size_t allocation_size = nif_linux_round_up_to_page_size(size);
if (allocation_size == 0)
return NULL;
int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(MAP_POPULATE)
if (numa_id >= 0)
mmap_flags |= MAP_POPULATE;
#endif
void *memory = mmap(NULL, allocation_size, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
if (memory == MAP_FAILED)
return NULL;
if (numa_id < 0)
return memory;
uint32_t bind_flags = 0u;
#if defined(MAP_POPULATE)
bind_flags = MPOL_MF_MOVE | MPOL_MF_STRICT;
#endif
if (!nif_linux_node_exists(numa_id) ||
!nif_linux_bind_memory_to_node(memory, allocation_size, numa_id, bind_flags) ||
!nif_linux_populate_writable_mapping(memory, allocation_size)) {
munmap(memory, allocation_size);
return NULL;
}
return memory;
}
NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id)
{
(void)numa_id;
if (ptr == NULL || size == 0)
return;
size_t allocation_size = nif_linux_round_up_to_page_size(size);
if (allocation_size != 0)
munmap(ptr, allocation_size);
}
#endif
src/Nitload.Native/nif-src/thread.windows.c
+95
View File
@@ -0,0 +1,95 @@
#include "nif_target.h"
#include "NitloadFrameWork.h"
#if NIF_IS_OS(NIF_OS_WINDOWS)
/* Pin the calling thread to a single logical processor.
* core_id >= 0 : bind to that logical processor (global index, across groups).
* core_id < 0 : clear affinity constraint within the current group. */
NIF_API void NIF_DECL nif_set_thread_affinity(int core_id, int numa_id)
{
if (core_id < 0 || numa_id < 0) {
/* Query which group the thread currently belongs to */
GROUP_AFFINITY current;
GetThreadGroupAffinity(GetCurrentThread(), &current);
/* Build a full mask for that group's actual core count */
DWORD count = GetActiveProcessorCount(current.Group);
GROUP_AFFINITY ga;
memset(&ga, 0, sizeof(ga));
ga.Group = current.Group;
ga.Mask = (count >= 64) ? ~(KAFFINITY)0 : ((KAFFINITY)1 << count) - 1;
SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL);
return;
}
int index = core_id % 64;
DWORD count = GetActiveProcessorCount(numa_id);
if ((DWORD)index >= count)
return;
GROUP_AFFINITY ga;
memset(&ga, 0, sizeof(ga));
ga.Group = (WORD)numa_id;
ga.Mask = (KAFFINITY)1 << index;
SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL);
}
NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
{
ULONG highest;
if (!GetNumaHighestNodeNumber(&highest))
return 0;
*out_numa_count = (int)highest + 1;
for (UCHAR n = 0; n <= (UCHAR)highest; n++) {
ULONGLONG mask = 0;
GetNumaNodeProcessorMask(n, &mask);
count_of_each_numa[n] = (int)__popcnt64(mask);
}
return 1;
}
NIF_API int NIF_DECL nif_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global)
{
ULONGLONG mask = 0;
if (!GetNumaNodeProcessorMask((UCHAR)numa_id, &mask))
return 0;
int intra = 0;
while (mask) {
DWORD idx;
_BitScanForward64(&idx, mask);
if (internal != NULL)
internal[intra] = (uint64_t)intra;
global[intra] = (uint32_t)idx;
intra++;
mask &= mask - 1;
}
return 1;
}
NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
{
if (size == 0)
return NULL;
if (numa_id < 0)
return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
ULONG highest = 0;
if (!GetNumaHighestNodeNumber(&highest) || (ULONG)numa_id > highest)
return NULL;
return VirtualAllocExNuma(GetCurrentProcess(), NULL, size, MEM_RESERVE | MEM_COMMIT,
PAGE_READWRITE, (DWORD)numa_id);
}
NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id)
{
(void)size;
(void)numa_id;
if (ptr != NULL)
VirtualFree(ptr, 0, MEM_RELEASE);
}
#endif
src/Nitload.Native/nif-src/vector.c
+32
View File
@@ -0,0 +1,32 @@
#include "nif_intrinsic.h"
#include "nif_internal.h"
INTERNAL void NIF_DECL f32_normalize(VEC_SEGMENT(float) *restrict source, VEC_SEGMENT(float) *restrict target)
{
register __m128 s = source->f32_x[0];
register __m128 a_length_sq = _mm_mul_ps(s, s);
a_length_sq = _mm_add_ps(a_length_sq,
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
a_length_sq = _mm_add_ss(a_length_sq,
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
__m128 a_inv_length = _mm_rsqrt_ps(a_length_sq);
s = _mm_mul_ps(s, a_inv_length);
__m128 dot_product = _mm_mul_ps(target->f32_x[0], s);
dot_product = _mm_add_ps(dot_product,
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(2, 3, 0, 1)));
dot_product = _mm_add_ss(dot_product,
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(0, 0, 3, 2)));
__m128 b_orthogonal = _mm_sub_ps(target->f32_x[0], _mm_mul_ps(s, dot_product));
__m128 b_length_sq = _mm_mul_ps(b_orthogonal, b_orthogonal);
b_length_sq = _mm_add_ps(b_length_sq,
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
b_length_sq = _mm_add_ss(b_length_sq,
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
__m128 b_inv_length = _mm_rsqrt_ps(b_length_sq);
target->f32_x[0] = _mm_mul_ps(b_orthogonal, b_inv_length);
}
src/Nitload.Native/nif-src/vector_dot.c
+67
View File
@@ -0,0 +1,67 @@
#include "nif_intrinsic.h"
#include "nif_internal.h"
static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 };
INTERNAL void FORCEINLINE NIF_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
__m128 const *vector, __m128 *ans)
{
*ans = _mm_dp_ps(transpose->x[0], *vector, 0B11110001);
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 1));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
INTERNAL void NIF_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans)
{
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
__m256 tmp = _mm256_dp_ps(transpose->y[0], _double, 0B11110001);
tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->y[0], _double, 0B11110010),
0B00100010);
*ans = _mm256_castps256_ps128(
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
}
INTERNAL void NIF_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans)
{
ans[0] = _mm_dp_ps(transpose->x[0], vector[0], 0B11110001);
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
ans[1] = _mm_dp_ps(transpose->x[0], vector[1], 0B11110001);
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
INTERNAL void NIF_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans)
{
__m256 temp = transpose->y[0];
__m256 vector_stream = _mm256_set_m128(vector[1], vector[0]);
__m256 result = _mm256_dp_ps(temp, vector_stream, 0B11110001);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010);
temp = transpose->y[1];
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000);
ans[0] = _mm256_castps256_ps128(result);
ans[1] = _mm256_extractf128_ps(result, 1);
ans[1] = _mm_shuffle_ps(ans[1], ans[1], _MM_SHUFFLE(2, 3, 0, 1));
}