HLSF use long as aloloc size
This commit is contained in:
Nitload
@@ -1,75 +1,75 @@
|
||||
#include "StgSharpC.h"
|
||||
#include "gl.h"
|
||||
#include "glfw3.h"
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
extern GladGLContext *currentContext;
|
||||
extern char infolog[512];
|
||||
|
||||
SSCAPI int SSCDECL glCheckShaderStat(GladGLContext *context, uint64_t shaderHandle, int key,
|
||||
char **logRef)
|
||||
{
|
||||
int stat = 0;
|
||||
context->GetShaderiv(shaderHandle, key, &stat);
|
||||
if (!stat) {
|
||||
context->GetShaderInfoLog(shaderHandle, 512, NULL, infolog);
|
||||
}
|
||||
*logRef = infolog;
|
||||
return stat;
|
||||
}
|
||||
|
||||
SSCAPI void SSCDECL initGL(int majorVersion, int minorVersion)
|
||||
{
|
||||
glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, majorVersion);
|
||||
glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, minorVersion);
|
||||
glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
|
||||
glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);
|
||||
}
|
||||
|
||||
SSCAPI GLFWglproc SSCDECL loadGlfuncDefault(char const *procName)
|
||||
{
|
||||
void *ret = glfwGetProcAddress(procName);
|
||||
// printf("%llu\n", (uint64_t)ret);
|
||||
if (ret == NULL) {
|
||||
strcpy_s(infolog,30, "Failed to load OpenGL api:");
|
||||
for (size_t i = 0; i < sizeof(procName); i++) {
|
||||
infolog[30 + i] = procName[i];
|
||||
}
|
||||
//printf("%s\n", infolog);
|
||||
}
|
||||
return (GLFWglproc)ret;
|
||||
}
|
||||
|
||||
SSCAPI unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
|
||||
{
|
||||
context->LinkProgram(shaderProgram);
|
||||
|
||||
int success;
|
||||
context->GetProgramiv(shaderProgram, GL_LINK_STATUS, &success);
|
||||
if (!success) {
|
||||
context->GetProgramInfoLog(shaderProgram, 512, NULL, infolog);
|
||||
return 0;
|
||||
} else {
|
||||
return shaderProgram;
|
||||
}
|
||||
}
|
||||
|
||||
SSCAPI void SSCDECL loadImageData(char const *location, Image *out, imageLoader loader)
|
||||
{
|
||||
if (loader == NULL) {
|
||||
loader = stbi_load;
|
||||
}
|
||||
out->pixelPtr = loader(location, &(out->width), &(out->height), &(out->channel), 0);
|
||||
}
|
||||
|
||||
SSCAPI void SSCDECL unloadImageData(Image *restrict out)
|
||||
{
|
||||
free(out->pixelPtr);
|
||||
}
|
||||
|
||||
SSCAPI char *SSCDECL readLog()
|
||||
{
|
||||
infolog[511] = '\0';
|
||||
return infolog;
|
||||
}
|
||||
#include "StgSharpC.h"
|
||||
#include "gl.h"
|
||||
#include "glfw3.h"
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
extern GladGLContext *currentContext;
|
||||
extern char infolog[512];
|
||||
|
||||
SSCAPI int SSCDECL glCheckShaderStat(GladGLContext *context, uint64_t shaderHandle, int key,
|
||||
char **logRef)
|
||||
{
|
||||
int stat = 0;
|
||||
context->GetShaderiv(shaderHandle, key, &stat);
|
||||
if (!stat) {
|
||||
context->GetShaderInfoLog(shaderHandle, 512, NULL, infolog);
|
||||
}
|
||||
*logRef = infolog;
|
||||
return stat;
|
||||
}
|
||||
|
||||
SSCAPI void SSCDECL initGL(int majorVersion, int minorVersion)
|
||||
{
|
||||
glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, majorVersion);
|
||||
glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, minorVersion);
|
||||
glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
|
||||
glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);
|
||||
}
|
||||
|
||||
SSCAPI GLFWglproc SSCDECL loadGlfuncDefault(char const *procName)
|
||||
{
|
||||
void *ret = glfwGetProcAddress(procName);
|
||||
// printf("%llu\n", (uint64_t)ret);
|
||||
if (ret == NULL) {
|
||||
strcpy_s(infolog,30, "Failed to load OpenGL api:");
|
||||
for (size_t i = 0; i < sizeof(procName); i++) {
|
||||
infolog[30 + i] = procName[i];
|
||||
}
|
||||
//printf("%s\n", infolog);
|
||||
}
|
||||
return (GLFWglproc)ret;
|
||||
}
|
||||
|
||||
SSCAPI unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
|
||||
{
|
||||
context->LinkProgram(shaderProgram);
|
||||
|
||||
int success;
|
||||
context->GetProgramiv(shaderProgram, GL_LINK_STATUS, &success);
|
||||
if (!success) {
|
||||
context->GetProgramInfoLog(shaderProgram, 512, NULL, infolog);
|
||||
return 0;
|
||||
} else {
|
||||
return shaderProgram;
|
||||
}
|
||||
}
|
||||
|
||||
SSCAPI void SSCDECL loadImageData(char const *location, Image *out, imageLoader loader)
|
||||
{
|
||||
if (loader == NULL) {
|
||||
loader = stbi_load;
|
||||
}
|
||||
out->pixelPtr = loader(location, &(out->width), &(out->height), &(out->channel), 0);
|
||||
}
|
||||
|
||||
SSCAPI void SSCDECL unloadImageData(Image *restrict out)
|
||||
{
|
||||
free(out->pixelPtr);
|
||||
}
|
||||
|
||||
SSCAPI char *SSCDECL readLog()
|
||||
{
|
||||
infolog[511] = '\0';
|
||||
return infolog;
|
||||
}
|
||||
|
||||
+214
-214
@@ -1,214 +1,214 @@
|
||||
#include "sn_target.h"
|
||||
#include "sn_intrinsic.h"
|
||||
|
||||
static SIMDID global_simdid = { 0 };
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <string.h>
|
||||
#if defined(_MSC_VER)
|
||||
#include <intrin.h>
|
||||
#else
|
||||
#include <cpuid.h>
|
||||
#endif
|
||||
|
||||
static void get_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx)
|
||||
{
|
||||
#if defined(_MSC_VER)
|
||||
int regs[4];
|
||||
__cpuidex(regs, leaf, subleaf);
|
||||
*eax = regs[0];
|
||||
*ebx = regs[1];
|
||||
*ecx = regs[2];
|
||||
*edx = regs[3];
|
||||
#else
|
||||
unsigned int a, b, c, d;
|
||||
__cpuid_count((unsigned int)leaf, (unsigned int)subleaf, a, b, c, d);
|
||||
*eax = (int)a;
|
||||
*ebx = (int)b;
|
||||
*ecx = (int)c;
|
||||
*edx = (int)d;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int xgetbv_has_xmm_ymm(void)
|
||||
{
|
||||
#if defined(_MSC_VER)
|
||||
unsigned __int64 xcr0 = _xgetbv(0);
|
||||
#else
|
||||
unsigned long long xcr0 = __builtin_ia32_xgetbv(0);
|
||||
#endif
|
||||
return ((xcr0 & 0x6) == 0x6);
|
||||
}
|
||||
|
||||
static int xgetbv_has_avx512_state(void)
|
||||
{
|
||||
const unsigned long long mask = (1ull << 0) | (1ull << 1) | (1ull << 2) | (1ull << 5) |
|
||||
(1ull << 6);
|
||||
#if defined(_MSC_VER)
|
||||
unsigned __int64 xcr0 = _xgetbv(0);
|
||||
#else
|
||||
unsigned long long xcr0 = __builtin_ia32_xgetbv(0);
|
||||
#endif
|
||||
return ((xcr0 & mask) == mask);
|
||||
}
|
||||
|
||||
static int is_intel_dual_avx512(int family, int model)
|
||||
{
|
||||
return (family == 6) && (model == 0x55); /* SKX/CLX/CPX */
|
||||
}
|
||||
|
||||
static int is_intel_native_avx512(int family, int model)
|
||||
{
|
||||
if (family != 6)
|
||||
return 0;
|
||||
switch (model) {
|
||||
case 0x6A: /* ICX */
|
||||
case 0x6C: /* ICX stepping */
|
||||
case 0x8F: /* SPR/EMR */
|
||||
case 0xCF: /* GNR */
|
||||
return 1;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
SN_API SIMDID SN_DECL sn_get_simd_level()
|
||||
{
|
||||
if (global_simdid.mask) {
|
||||
return global_simdid;
|
||||
}
|
||||
|
||||
int eax = 0, ebx = 0, ecx = 0, edx = 0;
|
||||
|
||||
/* Vendor */
|
||||
get_cpuid(0, 0, &eax, &ebx, &ecx, &edx);
|
||||
char vendor[13];
|
||||
((int *)vendor)[0] = ebx;
|
||||
((int *)vendor)[1] = edx;
|
||||
((int *)vendor)[2] = ecx;
|
||||
vendor[12] = '\0';
|
||||
const int is_intel = (strcmp(vendor, "GenuineIntel") == 0);
|
||||
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0) ||
|
||||
(strcmp(vendor, "HygonGenuine") == 0);
|
||||
|
||||
/* CPUID.1 */
|
||||
get_cpuid(1, 0, &eax, &ebx, &ecx, &edx);
|
||||
int family = ((eax >> 8) & 0xF) + ((eax >> 20) & 0xFF0);
|
||||
int model = ((eax >> 4) & 0xF) | ((eax >> 12) & 0xF0);
|
||||
|
||||
int has_sse42 = (ecx & (1 << 20)) != 0;
|
||||
int has_avx = (ecx & (1 << 28)) != 0;
|
||||
int has_osxsave = (ecx & (1 << 27)) != 0;
|
||||
int has_fma = (ecx & (1 << 12)) != 0;
|
||||
int has_f16c = (ecx & (1 << 29)) != 0;
|
||||
|
||||
uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
|
||||
uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL :
|
||||
is_amd ? SIMDID_MANU_AMD :
|
||||
SIMDID_MANU_UNKNOWN);
|
||||
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
|
||||
uint64_t avx512_bits = 0;
|
||||
uint64_t amx_bits = 0;
|
||||
uint64_t avx_bits = 0;
|
||||
|
||||
/* Baseline main level */
|
||||
if (has_sse42)
|
||||
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE);
|
||||
|
||||
if (has_avx && has_osxsave && xgetbv_has_xmm_ymm()) {
|
||||
avx_bits |= SIMDID_AVX;
|
||||
|
||||
/* CPUID.7.0 */
|
||||
get_cpuid(7, 0, &eax, &ebx, &ecx, &edx);
|
||||
const int has_bmi1 = (ebx & (1 << 3)) != 0;
|
||||
const int has_bmi2 = (ebx & (1 << 8)) != 0;
|
||||
const int has_avx2 = (ebx & (1 << 5)) != 0;
|
||||
const int has_avx512f = (ebx & (1 << 16)) != 0;
|
||||
const int has_avx512dq = (ebx & (1 << 17)) != 0;
|
||||
const int has_avx512bw = (ebx & (1 << 30)) != 0;
|
||||
const int has_avx512vl = (ebx & (1u << 31)) != 0;
|
||||
const int has_avx512_vnni = (ebx & (1 << 11)) != 0;
|
||||
const int has_vbmi = (ecx & (1 << 1)) != 0;
|
||||
const int has_vbmi2 = (ecx & (1 << 6)) != 0;
|
||||
|
||||
/* leaf 7, subleaf 1 for BF16/FP16 */
|
||||
get_cpuid(7, 1, &eax, &ebx, &ecx, &edx);
|
||||
const int has_bf16 = (eax & (1 << 5)) != 0; /* AVX512_BF16 */
|
||||
const int has_fp16 = (eax & (1 << 23)) != 0; /* AVX512_FP16 */
|
||||
|
||||
/* AMX in leaf 7 subleaf 0 EDX bits 24/25/26 */
|
||||
const int has_amx_tile = (edx & (1 << 24)) != 0;
|
||||
const int has_amx_int8 = (edx & (1 << 25)) != 0;
|
||||
const int has_amx_bf16 = (edx & (1 << 26)) != 0;
|
||||
|
||||
if (has_avx2) {
|
||||
avx_bits |= SIMDID_AVX2;
|
||||
if (has_fma)
|
||||
avx_bits |= SIMDID_AVX_FMA;
|
||||
if (has_f16c)
|
||||
avx_bits |= SIMDID_AVX_F16C;
|
||||
|
||||
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2);
|
||||
}
|
||||
|
||||
if (has_avx512f && xgetbv_has_avx512_state()) {
|
||||
/* Ensure base subset present */
|
||||
if (has_avx512bw && has_avx512vl && has_avx512dq) {
|
||||
avx512_bits |= SIMDID_AVX512_BASE;
|
||||
}
|
||||
if (has_avx512_vnni)
|
||||
avx512_bits |= SIMDID_AVX512_VNNI;
|
||||
if (has_bf16)
|
||||
avx512_bits |= SIMDID_AVX512_BF16;
|
||||
if (has_fp16)
|
||||
avx512_bits |= SIMDID_AVX512_FP16;
|
||||
if (has_vbmi)
|
||||
avx512_bits |= SIMDID_AVX512_VBMI;
|
||||
if (has_vbmi2)
|
||||
avx512_bits |= SIMDID_AVX512_VBMI2;
|
||||
|
||||
/* impl classification */
|
||||
const int glue = is_amd || is_intel_dual_avx512(family, model);
|
||||
const int native512 = is_intel_native_avx512(family, model);
|
||||
if (glue) {
|
||||
avx512_bits |= SIMDID_AVX512_IMPL_DUAL;
|
||||
} else if (native512) {
|
||||
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
|
||||
} else {
|
||||
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
|
||||
}
|
||||
|
||||
/* main level for AVX512 */
|
||||
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);
|
||||
|
||||
/* AMX promotion if present */
|
||||
if (has_amx_tile) {
|
||||
amx_bits |= SIMDID_AMX_TILE;
|
||||
if (has_amx_int8)
|
||||
amx_bits |= SIMDID_AMX_INT8;
|
||||
if (has_amx_bf16)
|
||||
amx_bits |= SIMDID_AMX_BF16;
|
||||
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
global_simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) |
|
||||
SIMDID_PACK_AMX(amx_bits) | SIMDID_PACK_AVX(avx_bits);
|
||||
return global_simdid;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
SIMDID sn_get_simd_level()
|
||||
{
|
||||
SIMDID simd_id;
|
||||
simd_id.mask = 0;
|
||||
return simd_id; /* non-x86: no SIMD level probing */
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
#include "sn_intrinsic.h"
|
||||
|
||||
static SIMDID global_simdid = { 0 };
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <string.h>
|
||||
#if defined(_MSC_VER)
|
||||
#include <intrin.h>
|
||||
#else
|
||||
#include <cpuid.h>
|
||||
#endif
|
||||
|
||||
static void get_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx)
|
||||
{
|
||||
#if defined(_MSC_VER)
|
||||
int regs[4];
|
||||
__cpuidex(regs, leaf, subleaf);
|
||||
*eax = regs[0];
|
||||
*ebx = regs[1];
|
||||
*ecx = regs[2];
|
||||
*edx = regs[3];
|
||||
#else
|
||||
unsigned int a, b, c, d;
|
||||
__cpuid_count((unsigned int)leaf, (unsigned int)subleaf, a, b, c, d);
|
||||
*eax = (int)a;
|
||||
*ebx = (int)b;
|
||||
*ecx = (int)c;
|
||||
*edx = (int)d;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int xgetbv_has_xmm_ymm(void)
|
||||
{
|
||||
#if defined(_MSC_VER)
|
||||
unsigned __int64 xcr0 = _xgetbv(0);
|
||||
#else
|
||||
unsigned long long xcr0 = __builtin_ia32_xgetbv(0);
|
||||
#endif
|
||||
return ((xcr0 & 0x6) == 0x6);
|
||||
}
|
||||
|
||||
static int xgetbv_has_avx512_state(void)
|
||||
{
|
||||
const unsigned long long mask = (1ull << 0) | (1ull << 1) | (1ull << 2) | (1ull << 5) |
|
||||
(1ull << 6);
|
||||
#if defined(_MSC_VER)
|
||||
unsigned __int64 xcr0 = _xgetbv(0);
|
||||
#else
|
||||
unsigned long long xcr0 = __builtin_ia32_xgetbv(0);
|
||||
#endif
|
||||
return ((xcr0 & mask) == mask);
|
||||
}
|
||||
|
||||
static int is_intel_dual_avx512(int family, int model)
|
||||
{
|
||||
return (family == 6) && (model == 0x55); /* SKX/CLX/CPX */
|
||||
}
|
||||
|
||||
static int is_intel_native_avx512(int family, int model)
|
||||
{
|
||||
if (family != 6)
|
||||
return 0;
|
||||
switch (model) {
|
||||
case 0x6A: /* ICX */
|
||||
case 0x6C: /* ICX stepping */
|
||||
case 0x8F: /* SPR/EMR */
|
||||
case 0xCF: /* GNR */
|
||||
return 1;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
SN_API SIMDID SN_DECL sn_get_simd_level()
|
||||
{
|
||||
if (global_simdid.mask) {
|
||||
return global_simdid;
|
||||
}
|
||||
|
||||
int eax = 0, ebx = 0, ecx = 0, edx = 0;
|
||||
|
||||
/* Vendor */
|
||||
get_cpuid(0, 0, &eax, &ebx, &ecx, &edx);
|
||||
char vendor[13];
|
||||
((int *)vendor)[0] = ebx;
|
||||
((int *)vendor)[1] = edx;
|
||||
((int *)vendor)[2] = ecx;
|
||||
vendor[12] = '\0';
|
||||
const int is_intel = (strcmp(vendor, "GenuineIntel") == 0);
|
||||
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0) ||
|
||||
(strcmp(vendor, "HygonGenuine") == 0);
|
||||
|
||||
/* CPUID.1 */
|
||||
get_cpuid(1, 0, &eax, &ebx, &ecx, &edx);
|
||||
int family = ((eax >> 8) & 0xF) + ((eax >> 20) & 0xFF0);
|
||||
int model = ((eax >> 4) & 0xF) | ((eax >> 12) & 0xF0);
|
||||
|
||||
int has_sse42 = (ecx & (1 << 20)) != 0;
|
||||
int has_avx = (ecx & (1 << 28)) != 0;
|
||||
int has_osxsave = (ecx & (1 << 27)) != 0;
|
||||
int has_fma = (ecx & (1 << 12)) != 0;
|
||||
int has_f16c = (ecx & (1 << 29)) != 0;
|
||||
|
||||
uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
|
||||
uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL :
|
||||
is_amd ? SIMDID_MANU_AMD :
|
||||
SIMDID_MANU_UNKNOWN);
|
||||
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
|
||||
uint64_t avx512_bits = 0;
|
||||
uint64_t amx_bits = 0;
|
||||
uint64_t avx_bits = 0;
|
||||
|
||||
/* Baseline main level */
|
||||
if (has_sse42)
|
||||
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE);
|
||||
|
||||
if (has_avx && has_osxsave && xgetbv_has_xmm_ymm()) {
|
||||
avx_bits |= SIMDID_AVX;
|
||||
|
||||
/* CPUID.7.0 */
|
||||
get_cpuid(7, 0, &eax, &ebx, &ecx, &edx);
|
||||
const int has_bmi1 = (ebx & (1 << 3)) != 0;
|
||||
const int has_bmi2 = (ebx & (1 << 8)) != 0;
|
||||
const int has_avx2 = (ebx & (1 << 5)) != 0;
|
||||
const int has_avx512f = (ebx & (1 << 16)) != 0;
|
||||
const int has_avx512dq = (ebx & (1 << 17)) != 0;
|
||||
const int has_avx512bw = (ebx & (1 << 30)) != 0;
|
||||
const int has_avx512vl = (ebx & (1u << 31)) != 0;
|
||||
const int has_avx512_vnni = (ebx & (1 << 11)) != 0;
|
||||
const int has_vbmi = (ecx & (1 << 1)) != 0;
|
||||
const int has_vbmi2 = (ecx & (1 << 6)) != 0;
|
||||
|
||||
/* leaf 7, subleaf 1 for BF16/FP16 */
|
||||
get_cpuid(7, 1, &eax, &ebx, &ecx, &edx);
|
||||
const int has_bf16 = (eax & (1 << 5)) != 0; /* AVX512_BF16 */
|
||||
const int has_fp16 = (eax & (1 << 23)) != 0; /* AVX512_FP16 */
|
||||
|
||||
/* AMX in leaf 7 subleaf 0 EDX bits 24/25/26 */
|
||||
const int has_amx_tile = (edx & (1 << 24)) != 0;
|
||||
const int has_amx_int8 = (edx & (1 << 25)) != 0;
|
||||
const int has_amx_bf16 = (edx & (1 << 26)) != 0;
|
||||
|
||||
if (has_avx2) {
|
||||
avx_bits |= SIMDID_AVX2;
|
||||
if (has_fma)
|
||||
avx_bits |= SIMDID_AVX_FMA;
|
||||
if (has_f16c)
|
||||
avx_bits |= SIMDID_AVX_F16C;
|
||||
|
||||
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2);
|
||||
}
|
||||
|
||||
if (has_avx512f && xgetbv_has_avx512_state()) {
|
||||
/* Ensure base subset present */
|
||||
if (has_avx512bw && has_avx512vl && has_avx512dq) {
|
||||
avx512_bits |= SIMDID_AVX512_BASE;
|
||||
}
|
||||
if (has_avx512_vnni)
|
||||
avx512_bits |= SIMDID_AVX512_VNNI;
|
||||
if (has_bf16)
|
||||
avx512_bits |= SIMDID_AVX512_BF16;
|
||||
if (has_fp16)
|
||||
avx512_bits |= SIMDID_AVX512_FP16;
|
||||
if (has_vbmi)
|
||||
avx512_bits |= SIMDID_AVX512_VBMI;
|
||||
if (has_vbmi2)
|
||||
avx512_bits |= SIMDID_AVX512_VBMI2;
|
||||
|
||||
/* impl classification */
|
||||
const int glue = is_amd || is_intel_dual_avx512(family, model);
|
||||
const int native512 = is_intel_native_avx512(family, model);
|
||||
if (glue) {
|
||||
avx512_bits |= SIMDID_AVX512_IMPL_DUAL;
|
||||
} else if (native512) {
|
||||
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
|
||||
} else {
|
||||
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
|
||||
}
|
||||
|
||||
/* main level for AVX512 */
|
||||
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);
|
||||
|
||||
/* AMX promotion if present */
|
||||
if (has_amx_tile) {
|
||||
amx_bits |= SIMDID_AMX_TILE;
|
||||
if (has_amx_int8)
|
||||
amx_bits |= SIMDID_AMX_INT8;
|
||||
if (has_amx_bf16)
|
||||
amx_bits |= SIMDID_AMX_BF16;
|
||||
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
global_simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) |
|
||||
SIMDID_PACK_AMX(amx_bits) | SIMDID_PACK_AVX(avx_bits);
|
||||
return global_simdid;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
SIMDID sn_get_simd_level()
|
||||
{
|
||||
SIMDID simd_id;
|
||||
simd_id.mask = 0;
|
||||
return simd_id; /* non-x86: no SIMD level probing */
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,47 +1,47 @@
|
||||
#include "StgSharpNative.h"
|
||||
#include "sn_matkernel.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
/* Define and initialize a global zero kernel aligned to 64 bytes for universal ISA usage */
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
__declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
static void init_zero_kernel(void)
|
||||
{
|
||||
for (int c = 0; c < 4; ++c) {
|
||||
for (int r = 0; r < 4; ++r) {
|
||||
SN_ZERO_KERNEL.m[r][c] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
|
||||
{
|
||||
switch (ul_reason_for_call) {
|
||||
case DLL_PROCESS_ATTACH:
|
||||
init_zero_kernel();
|
||||
return 1;
|
||||
case DLL_THREAD_ATTACH:
|
||||
return 1;
|
||||
case DLL_THREAD_DETACH:
|
||||
return 1;
|
||||
case DLL_PROCESS_DETACH:
|
||||
return 1;
|
||||
break;
|
||||
}
|
||||
return TRUE;
|
||||
}
|
||||
#else
|
||||
attribute((constructor)) void DllMain()
|
||||
{
|
||||
init_zero_kernel();
|
||||
}
|
||||
|
||||
#endif // _WIN32
|
||||
#include "StgSharpNative.h"
|
||||
#include "sn_matkernel.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
/* Define and initialize a global zero kernel aligned to 64 bytes for universal ISA usage */
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
__declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
static void init_zero_kernel(void)
|
||||
{
|
||||
for (int c = 0; c < 4; ++c) {
|
||||
for (int r = 0; r < 4; ++r) {
|
||||
SN_ZERO_KERNEL.m[r][c] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
|
||||
{
|
||||
switch (ul_reason_for_call) {
|
||||
case DLL_PROCESS_ATTACH:
|
||||
init_zero_kernel();
|
||||
return 1;
|
||||
case DLL_THREAD_ATTACH:
|
||||
return 1;
|
||||
case DLL_THREAD_DETACH:
|
||||
return 1;
|
||||
case DLL_PROCESS_DETACH:
|
||||
return 1;
|
||||
break;
|
||||
}
|
||||
return TRUE;
|
||||
}
|
||||
#else
|
||||
attribute((constructor)) void DllMain()
|
||||
{
|
||||
init_zero_kernel();
|
||||
}
|
||||
|
||||
#endif // _WIN32
|
||||
|
||||
+123
-123
@@ -1,123 +1,123 @@
|
||||
#include "StgSharpNative.h"
|
||||
#include "glfw_function.h"
|
||||
#include <string.h>
|
||||
|
||||
SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table)
|
||||
{
|
||||
table->glfwCreateCursor = glfwCreateCursor;
|
||||
table->glfwCreateStandardCursor = glfwCreateStandardCursor;
|
||||
table->glfwCreateWindow = glfwCreateWindow;
|
||||
table->glfwDefaultWindowHints = glfwDefaultWindowHints;
|
||||
table->glfwDestroyCursor = glfwDestroyCursor;
|
||||
table->glfwDestroyWindow = glfwDestroyWindow;
|
||||
table->glfwExtensionSupported = glfwExtensionSupported;
|
||||
table->glfwFocusWindow = glfwFocusWindow;
|
||||
table->glfwGetClipboardString = glfwGetClipboardString;
|
||||
table->glfwGetCurrentContext = glfwGetCurrentContext;
|
||||
table->glfwGetCursorPos = glfwGetCursorPos;
|
||||
table->glfwGetError = glfwGetError;
|
||||
table->glfwGetFrameBufferSize = glfwGetFramebufferSize;
|
||||
table->glfwGetGamepadName = glfwGetGamepadName;
|
||||
table->glfwGetGamepadState = glfwGetGamepadState;
|
||||
table->glfwGetGammaRamp = glfwGetGammaRamp;
|
||||
table->glfwGetInputMode = glfwGetInputMode;
|
||||
table->glfwGetJoystickAxes = glfwGetJoystickAxes;
|
||||
table->glfwGetJoystickButtons = glfwGetJoystickButtons;
|
||||
table->glfwGetJoystickGUID = glfwGetJoystickGUID;
|
||||
table->glfwGetJoystickHats = glfwGetJoystickHats;
|
||||
table->glfwGetJoystickName = glfwGetJoystickName;
|
||||
table->glfwGetJoystickUserPointer = glfwGetJoystickUserPointer;
|
||||
table->glfwGetKey = glfwGetKey;
|
||||
table->glfwGetKeyName = glfwGetKeyName;
|
||||
table->glfwGetKeyScancode = glfwGetKeyScancode;
|
||||
table->glfwGetMonitorContentScale = glfwGetMonitorContentScale;
|
||||
table->glfwGetMonitorName = glfwGetMonitorName;
|
||||
table->glfwGetMonitorPhysicalSize = glfwGetMonitorPhysicalSize;
|
||||
table->glfwGetMonitorPos = glfwGetMonitorPos;
|
||||
table->glfwGetMonitors = glfwGetMonitors;
|
||||
table->glfwGetMonitorUserPointer = glfwGetMonitorUserPointer;
|
||||
table->glfwGetMonitorWorkarea = glfwGetMonitorWorkarea;
|
||||
table->glfwGetMouseButton = glfwGetMouseButton;
|
||||
table->glfwGetPrimaryMonitor = glfwGetPrimaryMonitor;
|
||||
table->glfwGetProcAddress = glfwGetProcAddress;
|
||||
table->glfwGetRequiredInstanceExtensions = glfwGetRequiredInstanceExtensions;
|
||||
table->glfwGetTime = glfwGetTime;
|
||||
table->glfwGetTimerFrequency = glfwGetTimerFrequency;
|
||||
table->glfwGetTimerValue = glfwGetTimerValue;
|
||||
table->glfwGetVersion = glfwGetVersion;
|
||||
table->glfwGetVersionString = glfwGetVersionString;
|
||||
table->glfwGetVideoMode = glfwGetVideoMode;
|
||||
table->glfwGetVideoModes = glfwGetVideoModes;
|
||||
table->glfwGetWindowAttrib = glfwGetWindowAttrib;
|
||||
table->glfwGetWindowContentScale = glfwGetWindowContentScale;
|
||||
table->glfwGetWindowFrameSize = glfwGetWindowFrameSize;
|
||||
table->glfwGetWindowMonitor = glfwGetWindowMonitor;
|
||||
table->glfwGetWindowOpacity = glfwGetWindowOpacity;
|
||||
table->glfwGetWindowPos = glfwGetWindowPos;
|
||||
table->glfwGetWindowSize = glfwGetWindowSize;
|
||||
table->glfwGetWindowUserPointer = glfwGetWindowUserPointer;
|
||||
table->glfwHideWindow = glfwHideWindow;
|
||||
table->glfwIconifyWindow = glfwIconifyWindow;
|
||||
table->glfwInit = glfwInit;
|
||||
table->glfwInitHint = glfwInitHint;
|
||||
table->glfwJoystickIsGamePad = glfwJoystickIsGamepad;
|
||||
table->glfwJoystickPresent = glfwJoystickPresent;
|
||||
table->glfwMakeContextCurrent = glfwMakeContextCurrent;
|
||||
table->glfwMaximizeWindow = glfwMaximizeWindow;
|
||||
table->glfwPollEvents = glfwPollEvents;
|
||||
table->glfwPostEmptyEvent = glfwPostEmptyEvent;
|
||||
table->glfwRawMouseMotionSupported = glfwRawMouseMotionSupported;
|
||||
table->glfwRequestWindowAttention = glfwRequestWindowAttention;
|
||||
table->glfwRestoreWindow = glfwRestoreWindow;
|
||||
table->glfwSetCharCallback = glfwSetCharCallback;
|
||||
table->glfwSetCharModsCallback = glfwSetCharModsCallback;
|
||||
table->glfwSetClipboardString = glfwSetClipboardString;
|
||||
table->glfwSetCursor = glfwSetCursor;
|
||||
table->glfwSetCursorEnterCallback = glfwSetCursorEnterCallback;
|
||||
table->glfwSetCursorPos = glfwSetCursorPos;
|
||||
table->glfwSetCursorPosCallback = glfwSetCursorPosCallback;
|
||||
table->glfwSetDropCallback = glfwSetDropCallback;
|
||||
table->glfwSetErrorCallback = glfwSetErrorCallback;
|
||||
table->glfwSetFramebufferSizeCallback = glfwSetFramebufferSizeCallback;
|
||||
table->glfwSetGamma = glfwSetGamma;
|
||||
table->glfwSetGammaRamp = glfwSetGammaRamp;
|
||||
table->glfwSetInputMode = glfwSetInputMode;
|
||||
table->glfwSetJoystickCallback = glfwSetJoystickCallback;
|
||||
table->glfwSetJoystickUserPointer = glfwSetJoystickUserPointer;
|
||||
table->glfwSetKeyCallback = glfwSetKeyCallback;
|
||||
table->glfwSetMonitorCallback = glfwSetMonitorCallback;
|
||||
table->glfwSetMonitorUserPointer = glfwSetMonitorUserPointer;
|
||||
table->glfwSetMouseButtonCallback = glfwSetMouseButtonCallback;
|
||||
table->glfwSetScrollCallback = glfwSetScrollCallback;
|
||||
table->glfwSetTime = glfwSetTime;
|
||||
table->glfwSetWindowAspectRatio = glfwSetWindowAspectRatio;
|
||||
table->glfwSetWindowAttrib = glfwSetWindowAttrib;
|
||||
table->glfwSetWindowCloseCallback = glfwSetWindowCloseCallback;
|
||||
table->glfwSetWindowContentScaleCallback = glfwSetWindowContentScaleCallback;
|
||||
table->glfwSetWindowFocusCallback = glfwSetWindowFocusCallback;
|
||||
table->glfwSetWindowIcon = glfwSetWindowIcon;
|
||||
table->glfwSetWindowIconifyCallback = glfwSetWindowIconifyCallback;
|
||||
table->glfwSetWindowMaximizeCallback = glfwSetWindowMaximizeCallback;
|
||||
table->glfwSetWindowMonitor = glfwSetWindowMonitor;
|
||||
table->glfwSetWindowOpacity = glfwSetWindowOpacity;
|
||||
table->glfwSetWindowPos = glfwSetWindowPos;
|
||||
table->glfwSetWindowPosCallback = glfwSetWindowPosCallback;
|
||||
table->glfwSetWindowRefreshCallback = glfwSetWindowRefreshCallback;
|
||||
table->glfwSetWindowShouldClose = glfwSetWindowShouldClose;
|
||||
table->glfwSetWindowSize = glfwSetWindowSize;
|
||||
table->glfwSetWindowSizeCallback = glfwSetWindowSizeCallback;
|
||||
table->glfwSetWindowSizeLimits = glfwSetWindowSizeLimits;
|
||||
table->glfwSetWindowTitle = glfwSetWindowTitle;
|
||||
table->glfwSetWindowUserPointer = glfwSetWindowUserPointer;
|
||||
table->glfwShowWindow = glfwShowWindow;
|
||||
table->glfwSwapBuffers = glfwSwapBuffers;
|
||||
table->glfwSwapInterval = glfwSwapInterval;
|
||||
table->glfwTerminate = glfwTerminate;
|
||||
table->glfwUpdateGamePadMappings = glfwUpdateGamepadMappings;
|
||||
table->glfwVulkanSupported = glfwVulkanSupported;
|
||||
table->glfwWaitEvents = glfwWaitEvents;
|
||||
table->glfwWaitEventsTimeout = glfwWaitEventsTimeout;
|
||||
table->glfwWindowHint = glfwWindowHint;
|
||||
table->glfwWindowHintString = glfwWindowHintString;
|
||||
table->glfwWindowShouldClose = glfwWindowShouldClose;
|
||||
}
|
||||
#include "StgSharpNative.h"
|
||||
#include "glfw_function.h"
|
||||
#include <string.h>
|
||||
|
||||
SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table)
|
||||
{
|
||||
table->glfwCreateCursor = glfwCreateCursor;
|
||||
table->glfwCreateStandardCursor = glfwCreateStandardCursor;
|
||||
table->glfwCreateWindow = glfwCreateWindow;
|
||||
table->glfwDefaultWindowHints = glfwDefaultWindowHints;
|
||||
table->glfwDestroyCursor = glfwDestroyCursor;
|
||||
table->glfwDestroyWindow = glfwDestroyWindow;
|
||||
table->glfwExtensionSupported = glfwExtensionSupported;
|
||||
table->glfwFocusWindow = glfwFocusWindow;
|
||||
table->glfwGetClipboardString = glfwGetClipboardString;
|
||||
table->glfwGetCurrentContext = glfwGetCurrentContext;
|
||||
table->glfwGetCursorPos = glfwGetCursorPos;
|
||||
table->glfwGetError = glfwGetError;
|
||||
table->glfwGetFrameBufferSize = glfwGetFramebufferSize;
|
||||
table->glfwGetGamepadName = glfwGetGamepadName;
|
||||
table->glfwGetGamepadState = glfwGetGamepadState;
|
||||
table->glfwGetGammaRamp = glfwGetGammaRamp;
|
||||
table->glfwGetInputMode = glfwGetInputMode;
|
||||
table->glfwGetJoystickAxes = glfwGetJoystickAxes;
|
||||
table->glfwGetJoystickButtons = glfwGetJoystickButtons;
|
||||
table->glfwGetJoystickGUID = glfwGetJoystickGUID;
|
||||
table->glfwGetJoystickHats = glfwGetJoystickHats;
|
||||
table->glfwGetJoystickName = glfwGetJoystickName;
|
||||
table->glfwGetJoystickUserPointer = glfwGetJoystickUserPointer;
|
||||
table->glfwGetKey = glfwGetKey;
|
||||
table->glfwGetKeyName = glfwGetKeyName;
|
||||
table->glfwGetKeyScancode = glfwGetKeyScancode;
|
||||
table->glfwGetMonitorContentScale = glfwGetMonitorContentScale;
|
||||
table->glfwGetMonitorName = glfwGetMonitorName;
|
||||
table->glfwGetMonitorPhysicalSize = glfwGetMonitorPhysicalSize;
|
||||
table->glfwGetMonitorPos = glfwGetMonitorPos;
|
||||
table->glfwGetMonitors = glfwGetMonitors;
|
||||
table->glfwGetMonitorUserPointer = glfwGetMonitorUserPointer;
|
||||
table->glfwGetMonitorWorkarea = glfwGetMonitorWorkarea;
|
||||
table->glfwGetMouseButton = glfwGetMouseButton;
|
||||
table->glfwGetPrimaryMonitor = glfwGetPrimaryMonitor;
|
||||
table->glfwGetProcAddress = glfwGetProcAddress;
|
||||
table->glfwGetRequiredInstanceExtensions = glfwGetRequiredInstanceExtensions;
|
||||
table->glfwGetTime = glfwGetTime;
|
||||
table->glfwGetTimerFrequency = glfwGetTimerFrequency;
|
||||
table->glfwGetTimerValue = glfwGetTimerValue;
|
||||
table->glfwGetVersion = glfwGetVersion;
|
||||
table->glfwGetVersionString = glfwGetVersionString;
|
||||
table->glfwGetVideoMode = glfwGetVideoMode;
|
||||
table->glfwGetVideoModes = glfwGetVideoModes;
|
||||
table->glfwGetWindowAttrib = glfwGetWindowAttrib;
|
||||
table->glfwGetWindowContentScale = glfwGetWindowContentScale;
|
||||
table->glfwGetWindowFrameSize = glfwGetWindowFrameSize;
|
||||
table->glfwGetWindowMonitor = glfwGetWindowMonitor;
|
||||
table->glfwGetWindowOpacity = glfwGetWindowOpacity;
|
||||
table->glfwGetWindowPos = glfwGetWindowPos;
|
||||
table->glfwGetWindowSize = glfwGetWindowSize;
|
||||
table->glfwGetWindowUserPointer = glfwGetWindowUserPointer;
|
||||
table->glfwHideWindow = glfwHideWindow;
|
||||
table->glfwIconifyWindow = glfwIconifyWindow;
|
||||
table->glfwInit = glfwInit;
|
||||
table->glfwInitHint = glfwInitHint;
|
||||
table->glfwJoystickIsGamePad = glfwJoystickIsGamepad;
|
||||
table->glfwJoystickPresent = glfwJoystickPresent;
|
||||
table->glfwMakeContextCurrent = glfwMakeContextCurrent;
|
||||
table->glfwMaximizeWindow = glfwMaximizeWindow;
|
||||
table->glfwPollEvents = glfwPollEvents;
|
||||
table->glfwPostEmptyEvent = glfwPostEmptyEvent;
|
||||
table->glfwRawMouseMotionSupported = glfwRawMouseMotionSupported;
|
||||
table->glfwRequestWindowAttention = glfwRequestWindowAttention;
|
||||
table->glfwRestoreWindow = glfwRestoreWindow;
|
||||
table->glfwSetCharCallback = glfwSetCharCallback;
|
||||
table->glfwSetCharModsCallback = glfwSetCharModsCallback;
|
||||
table->glfwSetClipboardString = glfwSetClipboardString;
|
||||
table->glfwSetCursor = glfwSetCursor;
|
||||
table->glfwSetCursorEnterCallback = glfwSetCursorEnterCallback;
|
||||
table->glfwSetCursorPos = glfwSetCursorPos;
|
||||
table->glfwSetCursorPosCallback = glfwSetCursorPosCallback;
|
||||
table->glfwSetDropCallback = glfwSetDropCallback;
|
||||
table->glfwSetErrorCallback = glfwSetErrorCallback;
|
||||
table->glfwSetFramebufferSizeCallback = glfwSetFramebufferSizeCallback;
|
||||
table->glfwSetGamma = glfwSetGamma;
|
||||
table->glfwSetGammaRamp = glfwSetGammaRamp;
|
||||
table->glfwSetInputMode = glfwSetInputMode;
|
||||
table->glfwSetJoystickCallback = glfwSetJoystickCallback;
|
||||
table->glfwSetJoystickUserPointer = glfwSetJoystickUserPointer;
|
||||
table->glfwSetKeyCallback = glfwSetKeyCallback;
|
||||
table->glfwSetMonitorCallback = glfwSetMonitorCallback;
|
||||
table->glfwSetMonitorUserPointer = glfwSetMonitorUserPointer;
|
||||
table->glfwSetMouseButtonCallback = glfwSetMouseButtonCallback;
|
||||
table->glfwSetScrollCallback = glfwSetScrollCallback;
|
||||
table->glfwSetTime = glfwSetTime;
|
||||
table->glfwSetWindowAspectRatio = glfwSetWindowAspectRatio;
|
||||
table->glfwSetWindowAttrib = glfwSetWindowAttrib;
|
||||
table->glfwSetWindowCloseCallback = glfwSetWindowCloseCallback;
|
||||
table->glfwSetWindowContentScaleCallback = glfwSetWindowContentScaleCallback;
|
||||
table->glfwSetWindowFocusCallback = glfwSetWindowFocusCallback;
|
||||
table->glfwSetWindowIcon = glfwSetWindowIcon;
|
||||
table->glfwSetWindowIconifyCallback = glfwSetWindowIconifyCallback;
|
||||
table->glfwSetWindowMaximizeCallback = glfwSetWindowMaximizeCallback;
|
||||
table->glfwSetWindowMonitor = glfwSetWindowMonitor;
|
||||
table->glfwSetWindowOpacity = glfwSetWindowOpacity;
|
||||
table->glfwSetWindowPos = glfwSetWindowPos;
|
||||
table->glfwSetWindowPosCallback = glfwSetWindowPosCallback;
|
||||
table->glfwSetWindowRefreshCallback = glfwSetWindowRefreshCallback;
|
||||
table->glfwSetWindowShouldClose = glfwSetWindowShouldClose;
|
||||
table->glfwSetWindowSize = glfwSetWindowSize;
|
||||
table->glfwSetWindowSizeCallback = glfwSetWindowSizeCallback;
|
||||
table->glfwSetWindowSizeLimits = glfwSetWindowSizeLimits;
|
||||
table->glfwSetWindowTitle = glfwSetWindowTitle;
|
||||
table->glfwSetWindowUserPointer = glfwSetWindowUserPointer;
|
||||
table->glfwShowWindow = glfwShowWindow;
|
||||
table->glfwSwapBuffers = glfwSwapBuffers;
|
||||
table->glfwSwapInterval = glfwSwapInterval;
|
||||
table->glfwTerminate = glfwTerminate;
|
||||
table->glfwUpdateGamePadMappings = glfwUpdateGamepadMappings;
|
||||
table->glfwVulkanSupported = glfwVulkanSupported;
|
||||
table->glfwWaitEvents = glfwWaitEvents;
|
||||
table->glfwWaitEventsTimeout = glfwWaitEventsTimeout;
|
||||
table->glfwWindowHint = glfwWindowHint;
|
||||
table->glfwWindowHintString = glfwWindowHintString;
|
||||
table->glfwWindowShouldClose = glfwWindowShouldClose;
|
||||
}
|
||||
|
||||
@@ -1,149 +1,149 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "StgSharpNative.h"
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
#include <intrin.h>
|
||||
#include <isa_availability.h>
|
||||
#else
|
||||
#include <cpuid.h>
|
||||
#endif // _MSC_VER
|
||||
|
||||
PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
|
||||
|
||||
SN_API SIMDID load_intrinsic_function(void *intrinsic_context)
|
||||
{
|
||||
sn_null_assert(intrinsic_context);
|
||||
|
||||
SIMDID simdid = sn_get_simd_level();
|
||||
|
||||
sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
|
||||
|
||||
switch (simdid.make_byte[1]) {
|
||||
case SIMDID_MAIN_LVL_SSE:
|
||||
load_intrinsic_sse(context, simdid);
|
||||
break;
|
||||
case SIMDID_MAIN_LVL_AVX2:
|
||||
load_intrinsic_avx(context, simdid);
|
||||
break;
|
||||
case SIMDID_MAIN_LVL_AVX512:
|
||||
load_intrinsic_512(context, simdid);
|
||||
break;
|
||||
}
|
||||
return simdid;
|
||||
}
|
||||
|
||||
void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
context->index_pair = index_pair_sse;
|
||||
|
||||
context->mat[F32].buffer_add = BUF_PROC(float, sse, , add);
|
||||
context->mat[F32].buffer_fill = BUF_PROC(float, sse, , fill);
|
||||
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, , scalar_mul);
|
||||
context->mat[F32].buffer_sub = BUF_PROC(float, sse, , sub);
|
||||
context->mat[F32].buffer_transpose = KER_PROC(float, sse, , transpose);
|
||||
context->mat[F32].build_panel = NULL;
|
||||
context->mat[F32].clear_panel = PNL_PROC(float, sse, , clear);
|
||||
context->mat[F32].ker_fma = KER_PROC(float, sse, , fma);
|
||||
context->mat[F32].ker_tile_fma = KERTILE_PROC(float, sse, , fma);
|
||||
context->mat[F32].pivot = NULL;
|
||||
context->mat[F32].store_panel = NULL;
|
||||
|
||||
context->mat[F64].buffer_add = BUF_PROC(double, sse, , add);
|
||||
context->mat[F64].buffer_fill = BUF_PROC(double, sse, , fill);
|
||||
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, sse, , scalar_mul);
|
||||
context->mat[F64].buffer_sub = BUF_PROC(double, sse, , sub);
|
||||
context->mat[F64].buffer_transpose = NULL;
|
||||
context->mat[F64].build_panel = NULL;
|
||||
context->mat[F64].clear_panel = PNL_PROC(double, sse, , clear);
|
||||
context->mat[F64].ker_fma = KER_PROC(double, sse, , fma);
|
||||
context->mat[F64].ker_tile_fma = KERTILE_PROC(double, sse, , fma);
|
||||
context->mat[F64].pivot = NULL;
|
||||
context->mat[F64].store_panel = NULL;
|
||||
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
char avx_seg = id.make_byte[5];
|
||||
char is_fma = (avx_seg & 1 << 3) != 0;
|
||||
char is_f16c = (avx_seg & 1 << 4) != 0;
|
||||
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
context->index_pair = index_pair_sse;
|
||||
|
||||
context->mat[F32].buffer_add = BUF_PROC(float, avx, , add);
|
||||
context->mat[F32].buffer_fill = BUF_PROC(float, avx, , fill);
|
||||
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, , scalar_mul);
|
||||
context->mat[F32].buffer_sub = BUF_PROC(float, avx, , sub);
|
||||
context->mat[F32].buffer_transpose = KER_PROC(float, avx, , transpose);
|
||||
context->mat[F32].build_panel = BUILD_PANEL(avx, float);
|
||||
context->mat[F32].clear_panel = PNL_PROC(float, avx, , clear);
|
||||
context->mat[F32].ker_fma = is_fma ? KER_PROC(float, avx, , fma) :
|
||||
KER_PROC(float, avx, _fma, fma);
|
||||
context->mat[F32].ker_tile_fma = is_fma ? KERTILE_PROC(float, avx, , fma) :
|
||||
KERTILE_PROC(float, avx, _fma, fma);
|
||||
context->mat[F32].pivot = NULL;
|
||||
context->mat[F32].store_panel = STORE_PANEL(avx, float);
|
||||
|
||||
context->mat[F64].buffer_add = BUF_PROC(double, avx, , add);
|
||||
context->mat[F64].buffer_fill = BUF_PROC(double, avx, , fill);
|
||||
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, avx, , scalar_mul);
|
||||
context->mat[F64].buffer_sub = BUF_PROC(double, avx, , sub);
|
||||
context->mat[F64].buffer_transpose = NULL;
|
||||
context->mat[F64].build_panel = NULL;
|
||||
context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear);
|
||||
context->mat[F64].ker_fma = is_fma ? KER_PROC(double, avx, , fma) :
|
||||
KER_PROC(double, avx, _fma, fma);
|
||||
context->mat[F64].ker_tile_fma = is_fma ? KERTILE_PROC(double, avx, , fma) :
|
||||
KERTILE_PROC(double, avx, _fma, fma);
|
||||
context->mat[F64].pivot = NULL;
|
||||
context->mat[F64].store_panel = NULL;
|
||||
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
context->index_pair = index_pair_sse;
|
||||
|
||||
context->mat[F32].buffer_add = BUF_PROC(float, 512, , add);
|
||||
context->mat[F32].buffer_fill = BUF_PROC(float, 512, , fill);
|
||||
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, , scalar_mul);
|
||||
context->mat[F32].buffer_sub = BUF_PROC(float, 512, , sub);
|
||||
context->mat[F32].buffer_transpose = KER_PROC(float, 512, , transpose);
|
||||
context->mat[F32].build_panel = BUILD_PANEL(512, float);
|
||||
context->mat[F32].clear_panel = PNL_PROC(float, 512, , clear);
|
||||
context->mat[F32].ker_fma = KER_PROC(float, 512, , fma);
|
||||
context->mat[F32].ker_tile_fma = KERTILE_PROC(float, 512, , fma);
|
||||
context->mat[F32].pivot = NULL;
|
||||
context->mat[F32].store_panel = STORE_PANEL(512, float);
|
||||
|
||||
context->mat[F64].buffer_add = BUF_PROC(double, 512, , add);
|
||||
context->mat[F64].buffer_fill = BUF_PROC(double, 512, , fill);
|
||||
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, 512, , scalar_mul);
|
||||
context->mat[F64].buffer_sub = BUF_PROC(double, 512, , sub);
|
||||
context->mat[F64].buffer_transpose = NULL;
|
||||
context->mat[F64].build_panel = BUILD_PANEL(512, double);
|
||||
context->mat[F64].clear_panel = PNL_PROC(double, 512, , clear);
|
||||
context->mat[F64].ker_fma = KER_PROC(double, 512, , fma);
|
||||
context->mat[F64].ker_tile_fma = KERTILE_PROC(double, 512, , fma);
|
||||
context->mat[F64].pivot = NULL;
|
||||
context->mat[F64].store_panel = STORE_PANEL(512, double);
|
||||
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "StgSharpNative.h"
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
#include <intrin.h>
|
||||
#include <isa_availability.h>
|
||||
#else
|
||||
#include <cpuid.h>
|
||||
#endif // _MSC_VER
|
||||
|
||||
PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
|
||||
|
||||
SN_API SIMDID load_intrinsic_function(void *intrinsic_context)
|
||||
{
|
||||
sn_null_assert(intrinsic_context);
|
||||
|
||||
SIMDID simdid = sn_get_simd_level();
|
||||
|
||||
sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
|
||||
|
||||
switch (simdid.make_byte[1]) {
|
||||
case SIMDID_MAIN_LVL_SSE:
|
||||
load_intrinsic_sse(context, simdid);
|
||||
break;
|
||||
case SIMDID_MAIN_LVL_AVX2:
|
||||
load_intrinsic_avx(context, simdid);
|
||||
break;
|
||||
case SIMDID_MAIN_LVL_AVX512:
|
||||
load_intrinsic_512(context, simdid);
|
||||
break;
|
||||
}
|
||||
return simdid;
|
||||
}
|
||||
|
||||
void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
context->index_pair = index_pair_sse;
|
||||
|
||||
context->mat[F32].buffer_add = BUF_PROC(float, sse, , add);
|
||||
context->mat[F32].buffer_fill = BUF_PROC(float, sse, , fill);
|
||||
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, , scalar_mul);
|
||||
context->mat[F32].buffer_sub = BUF_PROC(float, sse, , sub);
|
||||
context->mat[F32].buffer_transpose = KER_PROC(float, sse, , transpose);
|
||||
context->mat[F32].build_panel = NULL;
|
||||
context->mat[F32].clear_panel = PNL_PROC(float, sse, , clear);
|
||||
context->mat[F32].ker_fma = KER_PROC(float, sse, , fma);
|
||||
context->mat[F32].ker_tile_fma = KERTILE_PROC(float, sse, , fma);
|
||||
context->mat[F32].pivot = NULL;
|
||||
context->mat[F32].store_panel = NULL;
|
||||
|
||||
context->mat[F64].buffer_add = BUF_PROC(double, sse, , add);
|
||||
context->mat[F64].buffer_fill = BUF_PROC(double, sse, , fill);
|
||||
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, sse, , scalar_mul);
|
||||
context->mat[F64].buffer_sub = BUF_PROC(double, sse, , sub);
|
||||
context->mat[F64].buffer_transpose = NULL;
|
||||
context->mat[F64].build_panel = NULL;
|
||||
context->mat[F64].clear_panel = PNL_PROC(double, sse, , clear);
|
||||
context->mat[F64].ker_fma = KER_PROC(double, sse, , fma);
|
||||
context->mat[F64].ker_tile_fma = KERTILE_PROC(double, sse, , fma);
|
||||
context->mat[F64].pivot = NULL;
|
||||
context->mat[F64].store_panel = NULL;
|
||||
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
char avx_seg = id.make_byte[5];
|
||||
char is_fma = (avx_seg & 1 << 3) != 0;
|
||||
char is_f16c = (avx_seg & 1 << 4) != 0;
|
||||
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
context->index_pair = index_pair_sse;
|
||||
|
||||
context->mat[F32].buffer_add = BUF_PROC(float, avx, , add);
|
||||
context->mat[F32].buffer_fill = BUF_PROC(float, avx, , fill);
|
||||
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, , scalar_mul);
|
||||
context->mat[F32].buffer_sub = BUF_PROC(float, avx, , sub);
|
||||
context->mat[F32].buffer_transpose = KER_PROC(float, avx, , transpose);
|
||||
context->mat[F32].build_panel = BUILD_PANEL(avx, float);
|
||||
context->mat[F32].clear_panel = PNL_PROC(float, avx, , clear);
|
||||
context->mat[F32].ker_fma = is_fma ? KER_PROC(float, avx, , fma) :
|
||||
KER_PROC(float, avx, _fma, fma);
|
||||
context->mat[F32].ker_tile_fma = is_fma ? KERTILE_PROC(float, avx, , fma) :
|
||||
KERTILE_PROC(float, avx, _fma, fma);
|
||||
context->mat[F32].pivot = NULL;
|
||||
context->mat[F32].store_panel = STORE_PANEL(avx, float);
|
||||
|
||||
context->mat[F64].buffer_add = BUF_PROC(double, avx, , add);
|
||||
context->mat[F64].buffer_fill = BUF_PROC(double, avx, , fill);
|
||||
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, avx, , scalar_mul);
|
||||
context->mat[F64].buffer_sub = BUF_PROC(double, avx, , sub);
|
||||
context->mat[F64].buffer_transpose = NULL;
|
||||
context->mat[F64].build_panel = NULL;
|
||||
context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear);
|
||||
context->mat[F64].ker_fma = is_fma ? KER_PROC(double, avx, , fma) :
|
||||
KER_PROC(double, avx, _fma, fma);
|
||||
context->mat[F64].ker_tile_fma = is_fma ? KERTILE_PROC(double, avx, , fma) :
|
||||
KERTILE_PROC(double, avx, _fma, fma);
|
||||
context->mat[F64].pivot = NULL;
|
||||
context->mat[F64].store_panel = NULL;
|
||||
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
context->index_pair = index_pair_sse;
|
||||
|
||||
context->mat[F32].buffer_add = BUF_PROC(float, 512, , add);
|
||||
context->mat[F32].buffer_fill = BUF_PROC(float, 512, , fill);
|
||||
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, , scalar_mul);
|
||||
context->mat[F32].buffer_sub = BUF_PROC(float, 512, , sub);
|
||||
context->mat[F32].buffer_transpose = KER_PROC(float, 512, , transpose);
|
||||
context->mat[F32].build_panel = BUILD_PANEL(512, float);
|
||||
context->mat[F32].clear_panel = PNL_PROC(float, 512, , clear);
|
||||
context->mat[F32].ker_fma = KER_PROC(float, 512, , fma);
|
||||
context->mat[F32].ker_tile_fma = KERTILE_PROC(float, 512, , fma);
|
||||
context->mat[F32].pivot = NULL;
|
||||
context->mat[F32].store_panel = STORE_PANEL(512, float);
|
||||
|
||||
context->mat[F64].buffer_add = BUF_PROC(double, 512, , add);
|
||||
context->mat[F64].buffer_fill = BUF_PROC(double, 512, , fill);
|
||||
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, 512, , scalar_mul);
|
||||
context->mat[F64].buffer_sub = BUF_PROC(double, 512, , sub);
|
||||
context->mat[F64].buffer_transpose = NULL;
|
||||
context->mat[F64].build_panel = BUILD_PANEL(512, double);
|
||||
context->mat[F64].clear_panel = PNL_PROC(double, 512, , clear);
|
||||
context->mat[F64].ker_fma = KER_PROC(double, 512, , fma);
|
||||
context->mat[F64].ker_tile_fma = KERTILE_PROC(double, 512, , fma);
|
||||
context->mat[F64].pivot = NULL;
|
||||
context->mat[F64].store_panel = STORE_PANEL(512, double);
|
||||
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,100 +1,100 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, sse, , clear)
|
||||
{
|
||||
register __m128 zero = _mm_setzero_ps();
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, avx, , clear)
|
||||
{
|
||||
register __m256 zero = _mm256_setzero_ps();
|
||||
for (size_t i = 0; i < 8; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, 512, , clear)
|
||||
{
|
||||
register __m512 zero = _mm512_setzero_ps();
|
||||
for (size_t i = 0; i < 16; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, sse, , clear)
|
||||
{
|
||||
register __m128 zero = _mm_setzero_ps();
|
||||
PANEL_COL_VEC((*ans), 0, 0) = zero;
|
||||
PANEL_COL_VEC((*ans), 1, 0) = zero;
|
||||
PANEL_COL_VEC((*ans), 2, 0) = zero;
|
||||
PANEL_COL_VEC((*ans), 3, 0) = zero;
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, avx, , clear)
|
||||
{
|
||||
register __m256 zero = _mm256_setzero_ps();
|
||||
for (size_t i = 0; i < 8; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, 512, , clear)
|
||||
{
|
||||
register __m512 zero = _mm512_setzero_ps();
|
||||
for (size_t i = 0; i < 16; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
// Matrix transpose implementations
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, sse, ,transpose)
|
||||
{
|
||||
register __m128 col0 = right->f32_x[0];
|
||||
register __m128 col1 = right->f32_x[1];
|
||||
register __m128 col2 = right->f32_x[2];
|
||||
register __m128 col3 = right->f32_x[3];
|
||||
register __m128 tmp0 = _mm_unpacklo_ps(col0, col1);
|
||||
register __m128 tmp1 = _mm_unpackhi_ps(col0, col1);
|
||||
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
|
||||
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
|
||||
|
||||
ans->f32_x[0] = _mm_movelh_ps(tmp0, tmp2);
|
||||
ans->f32_x[1] = _mm_movehl_ps(tmp2, tmp0);
|
||||
ans->f32_x[2] = _mm_movelh_ps(tmp1, tmp3);
|
||||
ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1);
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, avx, ,transpose)
|
||||
{
|
||||
register __m256 col_low = right->f32_y[0];
|
||||
register __m256 col_high = right->f32_y[1];
|
||||
|
||||
register __m256 unpack_low = _mm256_unpacklo_ps(col_low, col_high);
|
||||
register __m256 unpack_high = _mm256_unpackhi_ps(col_low, col_high);
|
||||
|
||||
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
|
||||
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
|
||||
|
||||
ans->f32_y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
|
||||
ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, 512, ,transpose)
|
||||
{
|
||||
register __m512 matrix = right->f32_z[0];
|
||||
|
||||
const __m512i transpose_indices =
|
||||
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
|
||||
|
||||
ans->f32_z[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, sse, , clear)
|
||||
{
|
||||
register __m128 zero = _mm_setzero_ps();
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, avx, , clear)
|
||||
{
|
||||
register __m256 zero = _mm256_setzero_ps();
|
||||
for (size_t i = 0; i < 8; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, 512, , clear)
|
||||
{
|
||||
register __m512 zero = _mm512_setzero_ps();
|
||||
for (size_t i = 0; i < 16; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, sse, , clear)
|
||||
{
|
||||
register __m128 zero = _mm_setzero_ps();
|
||||
PANEL_COL_VEC((*ans), 0, 0) = zero;
|
||||
PANEL_COL_VEC((*ans), 1, 0) = zero;
|
||||
PANEL_COL_VEC((*ans), 2, 0) = zero;
|
||||
PANEL_COL_VEC((*ans), 3, 0) = zero;
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, avx, , clear)
|
||||
{
|
||||
register __m256 zero = _mm256_setzero_ps();
|
||||
for (size_t i = 0; i < 8; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, 512, , clear)
|
||||
{
|
||||
register __m512 zero = _mm512_setzero_ps();
|
||||
for (size_t i = 0; i < 16; i++) {
|
||||
PANEL_COL_VEC((*ans), i, 0) = zero;
|
||||
}
|
||||
}
|
||||
|
||||
// Matrix transpose implementations
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, sse, ,transpose)
|
||||
{
|
||||
register __m128 col0 = right->f32_x[0];
|
||||
register __m128 col1 = right->f32_x[1];
|
||||
register __m128 col2 = right->f32_x[2];
|
||||
register __m128 col3 = right->f32_x[3];
|
||||
register __m128 tmp0 = _mm_unpacklo_ps(col0, col1);
|
||||
register __m128 tmp1 = _mm_unpackhi_ps(col0, col1);
|
||||
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
|
||||
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
|
||||
|
||||
ans->f32_x[0] = _mm_movelh_ps(tmp0, tmp2);
|
||||
ans->f32_x[1] = _mm_movehl_ps(tmp2, tmp0);
|
||||
ans->f32_x[2] = _mm_movelh_ps(tmp1, tmp3);
|
||||
ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1);
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, avx, ,transpose)
|
||||
{
|
||||
register __m256 col_low = right->f32_y[0];
|
||||
register __m256 col_high = right->f32_y[1];
|
||||
|
||||
register __m256 unpack_low = _mm256_unpacklo_ps(col_low, col_high);
|
||||
register __m256 unpack_high = _mm256_unpackhi_ps(col_low, col_high);
|
||||
|
||||
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
|
||||
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
|
||||
|
||||
ans->f32_y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
|
||||
ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, 512, ,transpose)
|
||||
{
|
||||
register __m512 matrix = right->f32_z[0];
|
||||
|
||||
const __m512i transpose_indices =
|
||||
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
|
||||
|
||||
ans->f32_z[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,324 +1,324 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, ,fill)
|
||||
{
|
||||
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f32_x[0] = vec;
|
||||
ans[i].f32_x[1] = vec;
|
||||
ans[i].f32_x[2] = vec;
|
||||
ans[i].f32_x[3] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, ,fill)
|
||||
{
|
||||
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f32_y[0] = vec;
|
||||
ans[i].f32_y[1] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, ,fill)
|
||||
{
|
||||
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f32_z[0] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, ,fill)
|
||||
{
|
||||
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f64_x[0] = vec;
|
||||
ans[i].f64_x[1] = vec;
|
||||
ans[i].f64_x[2] = vec;
|
||||
ans[i].f64_x[3] = vec;
|
||||
ans[i].f64_x[4] = vec;
|
||||
ans[i].f64_x[5] = vec;
|
||||
ans[i].f64_x[6] = vec;
|
||||
ans[i].f64_x[7] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, ,fill)
|
||||
{
|
||||
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f64_y[0] = vec;
|
||||
ans[i].f64_y[1] = vec;
|
||||
ans[i].f64_y[2] = vec;
|
||||
ans[i].f64_y[3] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, ,fill)
|
||||
{
|
||||
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f64_z[0] = vec;
|
||||
ans[i].f64_z[1] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128 c0 = _mm_add_ps(left[i].f32_x[0], right[i].f32_x[0]);
|
||||
register __m128 c1 = _mm_add_ps(left[i].f32_x[1], right[i].f32_x[1]);
|
||||
register __m128 c2 = _mm_add_ps(left[i].f32_x[2], right[i].f32_x[2]);
|
||||
register __m128 c3 = _mm_add_ps(left[i].f32_x[3], right[i].f32_x[3]);
|
||||
|
||||
ans[i].f32_x[0] = c0;
|
||||
ans[i].f32_x[1] = c1;
|
||||
ans[i].f32_x[2] = c2;
|
||||
ans[i].f32_x[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
size_t s = sizeof(MAT_KERNEL(float));
|
||||
register __m256 c0 = _mm256_add_ps(left[i].f32_y[0], right[i].f32_y[0]);
|
||||
register __m256 c1 = _mm256_add_ps(left[i].f32_y[1], right[i].f32_y[1]);
|
||||
ans[i].f32_y[0] = c0;
|
||||
ans[i].f32_y[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]);
|
||||
ans[i].f32_z[0] = c0;
|
||||
}
|
||||
}
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
|
||||
_mm_castps_pd(right[i].f32_x[0]));
|
||||
register __m128d c1 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[1]),
|
||||
_mm_castps_pd(right[i].f32_x[1]));
|
||||
register __m128d c2 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[2]),
|
||||
_mm_castps_pd(right[i].f32_x[2]));
|
||||
register __m128d c3 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[3]),
|
||||
_mm_castps_pd(right[i].f32_x[3]));
|
||||
register __m128d c4 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[4]),
|
||||
_mm_castps_pd(right[i].f32_x[4]));
|
||||
register __m128d c5 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[5]),
|
||||
_mm_castps_pd(right[i].f32_x[5]));
|
||||
register __m128d c6 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[6]),
|
||||
_mm_castps_pd(right[i].f32_x[6]));
|
||||
register __m128d c7 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[7]),
|
||||
_mm_castps_pd(right[i].f32_x[7]));
|
||||
|
||||
ans[i].f32_x[0] = _mm_castpd_ps(c0);
|
||||
ans[i].f32_x[1] = _mm_castpd_ps(c1);
|
||||
ans[i].f32_x[2] = _mm_castpd_ps(c2);
|
||||
ans[i].f32_x[3] = _mm_castpd_ps(c3);
|
||||
ans[i].f32_x[4] = _mm_castpd_ps(c4);
|
||||
ans[i].f32_x[5] = _mm_castpd_ps(c5);
|
||||
ans[i].f32_x[6] = _mm_castpd_ps(c6);
|
||||
ans[i].f32_x[7] = _mm_castpd_ps(c7);
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
size_t s = sizeof(MAT_KERNEL(float));
|
||||
register __m256d c0 = _mm256_add_pd(left[i].f64_y[0], right[i].f64_y[0]);
|
||||
register __m256d c1 = _mm256_add_pd(left[i].f64_y[1], right[i].f64_y[1]);
|
||||
register __m256d c2 = _mm256_add_pd(left[i].f64_y[0], right[i].f64_y[0]);
|
||||
register __m256d c3 = _mm256_add_pd(left[i].f64_y[1], right[i].f64_y[1]);
|
||||
ans[i].f64_y[0] = c0;
|
||||
ans[i].f64_y[1] = c1;
|
||||
ans[i].f64_y[2] = c2;
|
||||
ans[i].f64_y[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
|
||||
register __m512d c1 = _mm512_add_pd(left[i].f64_z[1], right[i].f64_z[1]);
|
||||
ans[i].f64_z[0] = c0;
|
||||
ans[i].f64_z[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]);
|
||||
register __m128 c1 = _mm_sub_ps(left[i].f32_x[1], right[i].f32_x[1]);
|
||||
register __m128 c2 = _mm_sub_ps(left[i].f32_x[2], right[i].f32_x[2]);
|
||||
register __m128 c3 = _mm_sub_ps(left[i].f32_x[3], right[i].f32_x[3]);
|
||||
|
||||
ans[i].f32_x[0] = c0;
|
||||
ans[i].f32_x[1] = c1;
|
||||
ans[i].f32_x[2] = c2;
|
||||
ans[i].f32_x[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
|
||||
register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]);
|
||||
ans[i].f32_y[0] = c0;
|
||||
ans[i].f32_y[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
|
||||
ans[i].f32_z[0] = c0;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
|
||||
register __m128d c1 = _mm_sub_pd(left[i].f64_x[1], right[i].f64_x[1]);
|
||||
register __m128d c2 = _mm_sub_pd(left[i].f64_x[2], right[i].f64_x[2]);
|
||||
register __m128d c3 = _mm_sub_pd(left[i].f64_x[3], right[i].f64_x[3]);
|
||||
register __m128d c4 = _mm_sub_pd(left[i].f64_x[4], right[i].f64_x[4]);
|
||||
register __m128d c5 = _mm_sub_pd(left[i].f64_x[5], right[i].f64_x[5]);
|
||||
register __m128d c6 = _mm_sub_pd(left[i].f64_x[6], right[i].f64_x[6]);
|
||||
register __m128d c7 = _mm_sub_pd(left[i].f64_x[7], right[i].f64_x[7]);
|
||||
|
||||
ans[i].f64_x[0] = c0;
|
||||
ans[i].f64_x[1] = c1;
|
||||
ans[i].f64_x[2] = c2;
|
||||
ans[i].f64_x[3] = c3;
|
||||
ans[i].f64_x[4] = c4;
|
||||
ans[i].f64_x[5] = c5;
|
||||
ans[i].f64_x[6] = c6;
|
||||
ans[i].f64_x[7] = c7;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
|
||||
register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]);
|
||||
ans[i].f32_y[0] = c0;
|
||||
ans[i].f32_y[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
|
||||
ans[i].f32_z[0] = c0;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, ,scalar_mul)
|
||||
{
|
||||
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128 c0 = _mm_mul_ps(right[i].f32_x[0], vec);
|
||||
register __m128 c1 = _mm_mul_ps(right[i].f32_x[1], vec);
|
||||
register __m128 c2 = _mm_mul_ps(right[i].f32_x[2], vec);
|
||||
register __m128 c3 = _mm_mul_ps(right[i].f32_x[3], vec);
|
||||
|
||||
ans[i].f32_x[0] = c0;
|
||||
ans[i].f32_x[1] = c1;
|
||||
ans[i].f32_x[2] = c2;
|
||||
ans[i].f32_x[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, ,scalar_mul)
|
||||
{
|
||||
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256 c0 = _mm256_mul_ps(right[i].f32_y[0], vec);
|
||||
register __m256 c1 = _mm256_mul_ps(right[i].f32_y[1], vec);
|
||||
|
||||
ans[i].f32_y[0] = c0;
|
||||
ans[i].f32_y[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, ,scalar_mul)
|
||||
{
|
||||
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_mul_ps(right[i].f32_z[0], vec);
|
||||
|
||||
ans[i].f32_z[0] = c0;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, ,scalar_mul)
|
||||
{
|
||||
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128d c0 = _mm_mul_pd(right[i].f64_x[0], vec);
|
||||
register __m128d c1 = _mm_mul_pd(right[i].f64_x[1], vec);
|
||||
register __m128d c2 = _mm_mul_pd(right[i].f64_x[2], vec);
|
||||
register __m128d c3 = _mm_mul_pd(right[i].f64_x[3], vec);
|
||||
register __m128d c4 = _mm_mul_pd(right[i].f64_x[4], vec);
|
||||
register __m128d c5 = _mm_mul_pd(right[i].f64_x[5], vec);
|
||||
register __m128d c6 = _mm_mul_pd(right[i].f64_x[6], vec);
|
||||
register __m128d c7 = _mm_mul_pd(right[i].f64_x[7], vec);
|
||||
|
||||
ans[i].f64_x[0] = c0;
|
||||
ans[i].f64_x[1] = c1;
|
||||
ans[i].f64_x[2] = c2;
|
||||
ans[i].f64_x[3] = c3;
|
||||
ans[i].f64_x[4] = c4;
|
||||
ans[i].f64_x[5] = c5;
|
||||
ans[i].f64_x[6] = c6;
|
||||
ans[i].f64_x[7] = c7;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, ,scalar_mul)
|
||||
{
|
||||
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256d c0 = _mm256_mul_pd(right[i].f64_y[0], vec);
|
||||
register __m256d c1 = _mm256_mul_pd(right[i].f64_y[1], vec);
|
||||
register __m256d c2 = _mm256_mul_pd(right[i].f64_y[2], vec);
|
||||
register __m256d c3 = _mm256_mul_pd(right[i].f64_y[3], vec);
|
||||
|
||||
ans[i].f64_y[0] = c0;
|
||||
ans[i].f64_y[1] = c1;
|
||||
ans[i].f64_y[2] = c2;
|
||||
ans[i].f64_y[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, ,scalar_mul)
|
||||
{
|
||||
register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512d c0 = _mm512_mul_pd(right[i].f64_z[0], vec);
|
||||
register __m512d c1 = _mm512_mul_pd(right[i].f64_z[1], vec);
|
||||
|
||||
ans[i].f64_z[0] = c0;
|
||||
ans[i].f64_z[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, ,fill)
|
||||
{
|
||||
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f32_x[0] = vec;
|
||||
ans[i].f32_x[1] = vec;
|
||||
ans[i].f32_x[2] = vec;
|
||||
ans[i].f32_x[3] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, ,fill)
|
||||
{
|
||||
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f32_y[0] = vec;
|
||||
ans[i].f32_y[1] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, ,fill)
|
||||
{
|
||||
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f32_z[0] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, ,fill)
|
||||
{
|
||||
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f64_x[0] = vec;
|
||||
ans[i].f64_x[1] = vec;
|
||||
ans[i].f64_x[2] = vec;
|
||||
ans[i].f64_x[3] = vec;
|
||||
ans[i].f64_x[4] = vec;
|
||||
ans[i].f64_x[5] = vec;
|
||||
ans[i].f64_x[6] = vec;
|
||||
ans[i].f64_x[7] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, ,fill)
|
||||
{
|
||||
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f64_y[0] = vec;
|
||||
ans[i].f64_y[1] = vec;
|
||||
ans[i].f64_y[2] = vec;
|
||||
ans[i].f64_y[3] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, ,fill)
|
||||
{
|
||||
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
ans[i].f64_z[0] = vec;
|
||||
ans[i].f64_z[1] = vec;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128 c0 = _mm_add_ps(left[i].f32_x[0], right[i].f32_x[0]);
|
||||
register __m128 c1 = _mm_add_ps(left[i].f32_x[1], right[i].f32_x[1]);
|
||||
register __m128 c2 = _mm_add_ps(left[i].f32_x[2], right[i].f32_x[2]);
|
||||
register __m128 c3 = _mm_add_ps(left[i].f32_x[3], right[i].f32_x[3]);
|
||||
|
||||
ans[i].f32_x[0] = c0;
|
||||
ans[i].f32_x[1] = c1;
|
||||
ans[i].f32_x[2] = c2;
|
||||
ans[i].f32_x[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
size_t s = sizeof(MAT_KERNEL(float));
|
||||
register __m256 c0 = _mm256_add_ps(left[i].f32_y[0], right[i].f32_y[0]);
|
||||
register __m256 c1 = _mm256_add_ps(left[i].f32_y[1], right[i].f32_y[1]);
|
||||
ans[i].f32_y[0] = c0;
|
||||
ans[i].f32_y[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]);
|
||||
ans[i].f32_z[0] = c0;
|
||||
}
|
||||
}
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
|
||||
_mm_castps_pd(right[i].f32_x[0]));
|
||||
register __m128d c1 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[1]),
|
||||
_mm_castps_pd(right[i].f32_x[1]));
|
||||
register __m128d c2 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[2]),
|
||||
_mm_castps_pd(right[i].f32_x[2]));
|
||||
register __m128d c3 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[3]),
|
||||
_mm_castps_pd(right[i].f32_x[3]));
|
||||
register __m128d c4 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[4]),
|
||||
_mm_castps_pd(right[i].f32_x[4]));
|
||||
register __m128d c5 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[5]),
|
||||
_mm_castps_pd(right[i].f32_x[5]));
|
||||
register __m128d c6 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[6]),
|
||||
_mm_castps_pd(right[i].f32_x[6]));
|
||||
register __m128d c7 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[7]),
|
||||
_mm_castps_pd(right[i].f32_x[7]));
|
||||
|
||||
ans[i].f32_x[0] = _mm_castpd_ps(c0);
|
||||
ans[i].f32_x[1] = _mm_castpd_ps(c1);
|
||||
ans[i].f32_x[2] = _mm_castpd_ps(c2);
|
||||
ans[i].f32_x[3] = _mm_castpd_ps(c3);
|
||||
ans[i].f32_x[4] = _mm_castpd_ps(c4);
|
||||
ans[i].f32_x[5] = _mm_castpd_ps(c5);
|
||||
ans[i].f32_x[6] = _mm_castpd_ps(c6);
|
||||
ans[i].f32_x[7] = _mm_castpd_ps(c7);
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
size_t s = sizeof(MAT_KERNEL(float));
|
||||
register __m256d c0 = _mm256_add_pd(left[i].f64_y[0], right[i].f64_y[0]);
|
||||
register __m256d c1 = _mm256_add_pd(left[i].f64_y[1], right[i].f64_y[1]);
|
||||
register __m256d c2 = _mm256_add_pd(left[i].f64_y[0], right[i].f64_y[0]);
|
||||
register __m256d c3 = _mm256_add_pd(left[i].f64_y[1], right[i].f64_y[1]);
|
||||
ans[i].f64_y[0] = c0;
|
||||
ans[i].f64_y[1] = c1;
|
||||
ans[i].f64_y[2] = c2;
|
||||
ans[i].f64_y[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
|
||||
register __m512d c1 = _mm512_add_pd(left[i].f64_z[1], right[i].f64_z[1]);
|
||||
ans[i].f64_z[0] = c0;
|
||||
ans[i].f64_z[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]);
|
||||
register __m128 c1 = _mm_sub_ps(left[i].f32_x[1], right[i].f32_x[1]);
|
||||
register __m128 c2 = _mm_sub_ps(left[i].f32_x[2], right[i].f32_x[2]);
|
||||
register __m128 c3 = _mm_sub_ps(left[i].f32_x[3], right[i].f32_x[3]);
|
||||
|
||||
ans[i].f32_x[0] = c0;
|
||||
ans[i].f32_x[1] = c1;
|
||||
ans[i].f32_x[2] = c2;
|
||||
ans[i].f32_x[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
|
||||
register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]);
|
||||
ans[i].f32_y[0] = c0;
|
||||
ans[i].f32_y[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
|
||||
ans[i].f32_z[0] = c0;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
|
||||
register __m128d c1 = _mm_sub_pd(left[i].f64_x[1], right[i].f64_x[1]);
|
||||
register __m128d c2 = _mm_sub_pd(left[i].f64_x[2], right[i].f64_x[2]);
|
||||
register __m128d c3 = _mm_sub_pd(left[i].f64_x[3], right[i].f64_x[3]);
|
||||
register __m128d c4 = _mm_sub_pd(left[i].f64_x[4], right[i].f64_x[4]);
|
||||
register __m128d c5 = _mm_sub_pd(left[i].f64_x[5], right[i].f64_x[5]);
|
||||
register __m128d c6 = _mm_sub_pd(left[i].f64_x[6], right[i].f64_x[6]);
|
||||
register __m128d c7 = _mm_sub_pd(left[i].f64_x[7], right[i].f64_x[7]);
|
||||
|
||||
ans[i].f64_x[0] = c0;
|
||||
ans[i].f64_x[1] = c1;
|
||||
ans[i].f64_x[2] = c2;
|
||||
ans[i].f64_x[3] = c3;
|
||||
ans[i].f64_x[4] = c4;
|
||||
ans[i].f64_x[5] = c5;
|
||||
ans[i].f64_x[6] = c6;
|
||||
ans[i].f64_x[7] = c7;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
|
||||
register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]);
|
||||
ans[i].f32_y[0] = c0;
|
||||
ans[i].f32_y[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
|
||||
ans[i].f32_z[0] = c0;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, ,scalar_mul)
|
||||
{
|
||||
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128 c0 = _mm_mul_ps(right[i].f32_x[0], vec);
|
||||
register __m128 c1 = _mm_mul_ps(right[i].f32_x[1], vec);
|
||||
register __m128 c2 = _mm_mul_ps(right[i].f32_x[2], vec);
|
||||
register __m128 c3 = _mm_mul_ps(right[i].f32_x[3], vec);
|
||||
|
||||
ans[i].f32_x[0] = c0;
|
||||
ans[i].f32_x[1] = c1;
|
||||
ans[i].f32_x[2] = c2;
|
||||
ans[i].f32_x[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, ,scalar_mul)
|
||||
{
|
||||
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256 c0 = _mm256_mul_ps(right[i].f32_y[0], vec);
|
||||
register __m256 c1 = _mm256_mul_ps(right[i].f32_y[1], vec);
|
||||
|
||||
ans[i].f32_y[0] = c0;
|
||||
ans[i].f32_y[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, ,scalar_mul)
|
||||
{
|
||||
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_mul_ps(right[i].f32_z[0], vec);
|
||||
|
||||
ans[i].f32_z[0] = c0;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, ,scalar_mul)
|
||||
{
|
||||
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128d c0 = _mm_mul_pd(right[i].f64_x[0], vec);
|
||||
register __m128d c1 = _mm_mul_pd(right[i].f64_x[1], vec);
|
||||
register __m128d c2 = _mm_mul_pd(right[i].f64_x[2], vec);
|
||||
register __m128d c3 = _mm_mul_pd(right[i].f64_x[3], vec);
|
||||
register __m128d c4 = _mm_mul_pd(right[i].f64_x[4], vec);
|
||||
register __m128d c5 = _mm_mul_pd(right[i].f64_x[5], vec);
|
||||
register __m128d c6 = _mm_mul_pd(right[i].f64_x[6], vec);
|
||||
register __m128d c7 = _mm_mul_pd(right[i].f64_x[7], vec);
|
||||
|
||||
ans[i].f64_x[0] = c0;
|
||||
ans[i].f64_x[1] = c1;
|
||||
ans[i].f64_x[2] = c2;
|
||||
ans[i].f64_x[3] = c3;
|
||||
ans[i].f64_x[4] = c4;
|
||||
ans[i].f64_x[5] = c5;
|
||||
ans[i].f64_x[6] = c6;
|
||||
ans[i].f64_x[7] = c7;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, ,scalar_mul)
|
||||
{
|
||||
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256d c0 = _mm256_mul_pd(right[i].f64_y[0], vec);
|
||||
register __m256d c1 = _mm256_mul_pd(right[i].f64_y[1], vec);
|
||||
register __m256d c2 = _mm256_mul_pd(right[i].f64_y[2], vec);
|
||||
register __m256d c3 = _mm256_mul_pd(right[i].f64_y[3], vec);
|
||||
|
||||
ans[i].f64_y[0] = c0;
|
||||
ans[i].f64_y[1] = c1;
|
||||
ans[i].f64_y[2] = c2;
|
||||
ans[i].f64_y[3] = c3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, ,scalar_mul)
|
||||
{
|
||||
register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512d c0 = _mm512_mul_pd(right[i].f64_z[0], vec);
|
||||
register __m512d c1 = _mm512_mul_pd(right[i].f64_z[1], vec);
|
||||
|
||||
ans[i].f64_z[0] = c0;
|
||||
ans[i].f64_z[1] = c1;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
+361
-361
@@ -1,361 +1,361 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
|
||||
|
||||
// float
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
|
||||
{
|
||||
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
|
||||
const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]);
|
||||
const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]);
|
||||
const __m128 a3 = _mm_load_ps((float *)&left->f32_x[3]);
|
||||
|
||||
__m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]);
|
||||
__m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]);
|
||||
__m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]);
|
||||
__m128 c3 = _mm_load_ps((float *)&ans->f32_x[3]);
|
||||
|
||||
__m128 bcol;
|
||||
|
||||
/* Column 0 */
|
||||
bcol = _mm_load_ps((float *)&right->f32_x[0]);
|
||||
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
|
||||
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
|
||||
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
|
||||
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
|
||||
|
||||
/* Column 1 */
|
||||
bcol = _mm_load_ps((float *)&right->f32_x[1]);
|
||||
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
|
||||
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
|
||||
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
|
||||
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
|
||||
|
||||
/* Column 2 */
|
||||
bcol = _mm_load_ps((float *)&right->f32_x[2]);
|
||||
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
|
||||
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
|
||||
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
|
||||
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
|
||||
|
||||
/* Column 3 */
|
||||
bcol = _mm_load_ps((float *)&right->f32_x[3]);
|
||||
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
|
||||
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
|
||||
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
|
||||
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
|
||||
|
||||
_mm_store_ps((float *)&ans->f32_x[0], c0);
|
||||
_mm_store_ps((float *)&ans->f32_x[1], c1);
|
||||
_mm_store_ps((float *)&ans->f32_x[2], c2);
|
||||
_mm_store_ps((float *)&ans->f32_x[3], c3);
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
|
||||
{
|
||||
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
|
||||
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
|
||||
|
||||
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
|
||||
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
|
||||
|
||||
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
|
||||
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
|
||||
b01 = _mm256_permutevar_ps(b01, b01_shift);
|
||||
b23 = _mm256_permutevar_ps(b23, b23_shift);
|
||||
|
||||
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
|
||||
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
|
||||
|
||||
//group 0
|
||||
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
|
||||
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
|
||||
c01 = _mm256_add_ps(_mm256_mul_ps(a01, b01_perm), c01);
|
||||
c23 = _mm256_add_ps(_mm256_mul_ps(a23, b23_perm), c23);
|
||||
|
||||
//group 2
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
|
||||
c01 = _mm256_add_ps(_mm256_mul_ps(a23, b01_perm), c01);
|
||||
c23 = _mm256_add_ps(_mm256_mul_ps(a01, b23_perm), c23);
|
||||
|
||||
//group 1
|
||||
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
|
||||
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
|
||||
c01 = _mm256_add_ps(_mm256_mul_ps(a12, b01_perm), c01);
|
||||
c23 = _mm256_add_ps(_mm256_mul_ps(a30, b23_perm), c23);
|
||||
|
||||
//group 3
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
|
||||
c01 = _mm256_add_ps(_mm256_mul_ps(a30, b01_perm), c01);
|
||||
c23 = _mm256_add_ps(_mm256_mul_ps(a12, b23_perm), c23);
|
||||
|
||||
ans->f32_y[0] = c01;
|
||||
ans->f32_y[1] = c23;
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
|
||||
{
|
||||
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
|
||||
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
|
||||
|
||||
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
|
||||
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
|
||||
|
||||
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
|
||||
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
|
||||
b01 = _mm256_permutevar_ps(b01, b01_shift);
|
||||
b23 = _mm256_permutevar_ps(b23, b23_shift);
|
||||
|
||||
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
|
||||
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
|
||||
|
||||
//group 0
|
||||
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
|
||||
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
|
||||
c01 = _mm256_fmadd_ps(a01, b01_perm, c01);
|
||||
c23 = _mm256_fmadd_ps(a23, b23_perm, c23);
|
||||
|
||||
//group 2
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
|
||||
c01 = _mm256_fmadd_ps(a23, b01_perm, c01);
|
||||
c23 = _mm256_fmadd_ps(a01, b23_perm, c23);
|
||||
|
||||
//group 1
|
||||
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
|
||||
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
|
||||
c01 = _mm256_fmadd_ps(a12, b01_perm, c01);
|
||||
c23 = _mm256_fmadd_ps(a30, b23_perm, c23);
|
||||
|
||||
//group 3
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
|
||||
c01 = _mm256_fmadd_ps(a30, b01_perm, c01);
|
||||
c23 = _mm256_fmadd_ps(a12, b23_perm, c23);
|
||||
|
||||
ans->f32_y[0] = c01;
|
||||
ans->f32_y[1] = c23;
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
|
||||
{
|
||||
static const __m512i b_shift = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15,
|
||||
12, 13, 14 } };
|
||||
|
||||
register __m512 a = _mm512_load_ps((float *)left);
|
||||
register __m512 b = _mm512_load_ps((float *)right);
|
||||
register __m512 c = _mm512_load_ps((float *)ans);
|
||||
|
||||
b = _mm512_permutexvar_ps(b_shift, b);
|
||||
register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
|
||||
register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0));
|
||||
c = _mm512_fmadd_ps(a_perm, b_perm, c);
|
||||
|
||||
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
|
||||
b_perm = _mm512_permute_ps(b, _FMA_SELECT(1));
|
||||
c = _mm512_fmadd_ps(a_perm, b_perm, c);
|
||||
|
||||
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
b_perm = _mm512_permute_ps(b, _FMA_SELECT(2));
|
||||
c = _mm512_fmadd_ps(a_perm, b_perm, c);
|
||||
|
||||
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
|
||||
b_perm = _mm512_permute_ps(b, _FMA_SELECT(3));
|
||||
c = _mm512_fmadd_ps(a_perm, b_perm, c);
|
||||
|
||||
_mm512_store_ps((float *)ans, c);
|
||||
}
|
||||
|
||||
// double
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
|
||||
{
|
||||
for (int col = 0; col < 4; ++col) {
|
||||
/* current answer column (two chunks: rows 0-1 and 2-3) */
|
||||
register __m128d c_lo = _mm_load_pd(ans->f64_x + (col * 2 + 0));
|
||||
register __m128d c_hi = _mm_load_pd(ans->f64_x + (col * 2 + 1));
|
||||
|
||||
/* right matrix column split into two chunks */
|
||||
register __m128d b_lo = _mm_load_pd(right + (2 * col + 0));
|
||||
register __m128d b_hi = _mm_load_pd(right + (2 * col + 1));
|
||||
|
||||
/* k = 0 */
|
||||
register __m128d a_lo = _mm_load_pd(left + (0 * 2 + 0));
|
||||
register __m128d a_hi = _mm_load_pd(left + (0 * 2 + 1));
|
||||
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0);
|
||||
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
|
||||
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
|
||||
|
||||
/* k = 1 */
|
||||
a_lo = _mm_load_pd(left + (1 * 2 + 0));
|
||||
a_hi = _mm_load_pd(left + (1 * 2 + 1));
|
||||
b = _mm_shuffle_pd(b_lo, b_lo, 0x3);
|
||||
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
|
||||
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
|
||||
|
||||
/* k = 2 */
|
||||
a_lo = _mm_load_pd(left + (1 * 2 + 0));
|
||||
a_hi = _mm_load_pd(left + (1 * 2 + 1));
|
||||
b = _mm_shuffle_pd(b_hi, b_hi, 0x0);
|
||||
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
|
||||
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
|
||||
|
||||
/* k = 3 */
|
||||
a_lo = _mm_load_pd(left + (1 * 2 + 0));
|
||||
a_hi = _mm_load_pd(left + (1 * 2 + 1));
|
||||
b = _mm_shuffle_pd(b_hi, b_hi, 0x3);
|
||||
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
|
||||
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
|
||||
|
||||
ans->f64_x[col * 2 + 0] = c_lo;
|
||||
ans->f64_x[col * 2 + 1] = c_hi;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
|
||||
{
|
||||
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
|
||||
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
|
||||
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
|
||||
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
|
||||
|
||||
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
|
||||
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
|
||||
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
|
||||
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
|
||||
|
||||
register __m256d ccol;
|
||||
|
||||
/* Column 0 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3))));
|
||||
ans->f64_y[0] = ccol;
|
||||
/* Column 1 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3))));
|
||||
ans->f64_y[1] = ccol;
|
||||
/* Column 2 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3))));
|
||||
ans->f64_y[2] = ccol;
|
||||
/* Column 3 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3))));
|
||||
ans->f64_y[3] = ccol;
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
|
||||
{
|
||||
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
|
||||
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
|
||||
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
|
||||
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
|
||||
|
||||
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
|
||||
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
|
||||
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
|
||||
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
|
||||
|
||||
register __m256d ccol;
|
||||
|
||||
/* Column 0 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
|
||||
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3)), ccol);
|
||||
ans->f64_y[0] = ccol;
|
||||
/* Column 1 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
|
||||
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3)), ccol);
|
||||
ans->f64_y[1] = ccol;
|
||||
/* Column 2 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
|
||||
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3)), ccol);
|
||||
ans->f64_y[2] = ccol;
|
||||
/* Column 3 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
|
||||
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3)), ccol);
|
||||
ans->f64_y[3] = ccol;
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
|
||||
{
|
||||
static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
|
||||
static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
|
||||
|
||||
register __m512d a01 = _mm512_load_pd((double *)&left->f64_z[0]);
|
||||
register __m512d a23 = _mm512_load_pd((double *)&left->f64_z[1]);
|
||||
|
||||
register __m512d b01 = _mm512_load_pd((double *)&right->f64_z[0]);
|
||||
register __m512d b23 = _mm512_load_pd((double *)&right->f64_z[1]);
|
||||
b01 = _mm512_permutexvar_pd(b01_shift, b01);
|
||||
b23 = _mm512_permutexvar_pd(b23_shift, b23);
|
||||
|
||||
register __m512d c01 = _mm512_load_pd((float *)&ans->f64_z[0]);
|
||||
register __m512d c23 = _mm512_load_pd((float *)&ans->f64_z[1]);
|
||||
|
||||
//group 0
|
||||
register __m512d b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(0));
|
||||
register __m512d b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(0));
|
||||
c01 = _mm512_fmadd_pd(a01, b01_perm, c01);
|
||||
c23 = _mm512_fmadd_pd(a23, b23_perm, c23);
|
||||
|
||||
//group 2
|
||||
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(2));
|
||||
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(2));
|
||||
c01 = _mm512_fmadd_pd(a23, b01_perm, c01);
|
||||
c23 = _mm512_fmadd_pd(a01, b23_perm, c23);
|
||||
|
||||
//group 1
|
||||
register __m512d a12 = _mm512_shuffle_f64x2(a01, a23, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
register __m512d a30 = _mm512_shuffle_f64x2(a23, a01, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(1));
|
||||
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(1));
|
||||
c01 = _mm512_fmadd_pd(a12, b01_perm, c01);
|
||||
c23 = _mm512_fmadd_pd(a30, b23_perm, c23);
|
||||
|
||||
//group 3
|
||||
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(3));
|
||||
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(3));
|
||||
c01 = _mm512_fmadd_pd(a30, b01_perm, c01);
|
||||
c23 = _mm512_fmadd_pd(a12, b23_perm, c23);
|
||||
|
||||
ans->f64_z[0] = c01;
|
||||
ans->f64_z[1] = c23;
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
|
||||
|
||||
// float
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
|
||||
{
|
||||
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
|
||||
const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]);
|
||||
const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]);
|
||||
const __m128 a3 = _mm_load_ps((float *)&left->f32_x[3]);
|
||||
|
||||
__m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]);
|
||||
__m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]);
|
||||
__m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]);
|
||||
__m128 c3 = _mm_load_ps((float *)&ans->f32_x[3]);
|
||||
|
||||
__m128 bcol;
|
||||
|
||||
/* Column 0 */
|
||||
bcol = _mm_load_ps((float *)&right->f32_x[0]);
|
||||
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
|
||||
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
|
||||
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
|
||||
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
|
||||
|
||||
/* Column 1 */
|
||||
bcol = _mm_load_ps((float *)&right->f32_x[1]);
|
||||
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
|
||||
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
|
||||
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
|
||||
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
|
||||
|
||||
/* Column 2 */
|
||||
bcol = _mm_load_ps((float *)&right->f32_x[2]);
|
||||
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
|
||||
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
|
||||
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
|
||||
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
|
||||
|
||||
/* Column 3 */
|
||||
bcol = _mm_load_ps((float *)&right->f32_x[3]);
|
||||
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
|
||||
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
|
||||
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
|
||||
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
|
||||
|
||||
_mm_store_ps((float *)&ans->f32_x[0], c0);
|
||||
_mm_store_ps((float *)&ans->f32_x[1], c1);
|
||||
_mm_store_ps((float *)&ans->f32_x[2], c2);
|
||||
_mm_store_ps((float *)&ans->f32_x[3], c3);
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
|
||||
{
|
||||
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
|
||||
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
|
||||
|
||||
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
|
||||
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
|
||||
|
||||
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
|
||||
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
|
||||
b01 = _mm256_permutevar_ps(b01, b01_shift);
|
||||
b23 = _mm256_permutevar_ps(b23, b23_shift);
|
||||
|
||||
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
|
||||
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
|
||||
|
||||
//group 0
|
||||
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
|
||||
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
|
||||
c01 = _mm256_add_ps(_mm256_mul_ps(a01, b01_perm), c01);
|
||||
c23 = _mm256_add_ps(_mm256_mul_ps(a23, b23_perm), c23);
|
||||
|
||||
//group 2
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
|
||||
c01 = _mm256_add_ps(_mm256_mul_ps(a23, b01_perm), c01);
|
||||
c23 = _mm256_add_ps(_mm256_mul_ps(a01, b23_perm), c23);
|
||||
|
||||
//group 1
|
||||
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
|
||||
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
|
||||
c01 = _mm256_add_ps(_mm256_mul_ps(a12, b01_perm), c01);
|
||||
c23 = _mm256_add_ps(_mm256_mul_ps(a30, b23_perm), c23);
|
||||
|
||||
//group 3
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
|
||||
c01 = _mm256_add_ps(_mm256_mul_ps(a30, b01_perm), c01);
|
||||
c23 = _mm256_add_ps(_mm256_mul_ps(a12, b23_perm), c23);
|
||||
|
||||
ans->f32_y[0] = c01;
|
||||
ans->f32_y[1] = c23;
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
|
||||
{
|
||||
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
|
||||
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
|
||||
|
||||
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
|
||||
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
|
||||
|
||||
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
|
||||
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
|
||||
b01 = _mm256_permutevar_ps(b01, b01_shift);
|
||||
b23 = _mm256_permutevar_ps(b23, b23_shift);
|
||||
|
||||
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
|
||||
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
|
||||
|
||||
//group 0
|
||||
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
|
||||
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
|
||||
c01 = _mm256_fmadd_ps(a01, b01_perm, c01);
|
||||
c23 = _mm256_fmadd_ps(a23, b23_perm, c23);
|
||||
|
||||
//group 2
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
|
||||
c01 = _mm256_fmadd_ps(a23, b01_perm, c01);
|
||||
c23 = _mm256_fmadd_ps(a01, b23_perm, c23);
|
||||
|
||||
//group 1
|
||||
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
|
||||
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
|
||||
c01 = _mm256_fmadd_ps(a12, b01_perm, c01);
|
||||
c23 = _mm256_fmadd_ps(a30, b23_perm, c23);
|
||||
|
||||
//group 3
|
||||
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
|
||||
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
|
||||
c01 = _mm256_fmadd_ps(a30, b01_perm, c01);
|
||||
c23 = _mm256_fmadd_ps(a12, b23_perm, c23);
|
||||
|
||||
ans->f32_y[0] = c01;
|
||||
ans->f32_y[1] = c23;
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
|
||||
{
|
||||
static const __m512i b_shift = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15,
|
||||
12, 13, 14 } };
|
||||
|
||||
register __m512 a = _mm512_load_ps((float *)left);
|
||||
register __m512 b = _mm512_load_ps((float *)right);
|
||||
register __m512 c = _mm512_load_ps((float *)ans);
|
||||
|
||||
b = _mm512_permutexvar_ps(b_shift, b);
|
||||
register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
|
||||
register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0));
|
||||
c = _mm512_fmadd_ps(a_perm, b_perm, c);
|
||||
|
||||
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
|
||||
b_perm = _mm512_permute_ps(b, _FMA_SELECT(1));
|
||||
c = _mm512_fmadd_ps(a_perm, b_perm, c);
|
||||
|
||||
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
b_perm = _mm512_permute_ps(b, _FMA_SELECT(2));
|
||||
c = _mm512_fmadd_ps(a_perm, b_perm, c);
|
||||
|
||||
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
|
||||
b_perm = _mm512_permute_ps(b, _FMA_SELECT(3));
|
||||
c = _mm512_fmadd_ps(a_perm, b_perm, c);
|
||||
|
||||
_mm512_store_ps((float *)ans, c);
|
||||
}
|
||||
|
||||
// double
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
|
||||
{
|
||||
for (int col = 0; col < 4; ++col) {
|
||||
/* current answer column (two chunks: rows 0-1 and 2-3) */
|
||||
register __m128d c_lo = _mm_load_pd(ans->f64_x + (col * 2 + 0));
|
||||
register __m128d c_hi = _mm_load_pd(ans->f64_x + (col * 2 + 1));
|
||||
|
||||
/* right matrix column split into two chunks */
|
||||
register __m128d b_lo = _mm_load_pd(right + (2 * col + 0));
|
||||
register __m128d b_hi = _mm_load_pd(right + (2 * col + 1));
|
||||
|
||||
/* k = 0 */
|
||||
register __m128d a_lo = _mm_load_pd(left + (0 * 2 + 0));
|
||||
register __m128d a_hi = _mm_load_pd(left + (0 * 2 + 1));
|
||||
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0);
|
||||
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
|
||||
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
|
||||
|
||||
/* k = 1 */
|
||||
a_lo = _mm_load_pd(left + (1 * 2 + 0));
|
||||
a_hi = _mm_load_pd(left + (1 * 2 + 1));
|
||||
b = _mm_shuffle_pd(b_lo, b_lo, 0x3);
|
||||
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
|
||||
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
|
||||
|
||||
/* k = 2 */
|
||||
a_lo = _mm_load_pd(left + (1 * 2 + 0));
|
||||
a_hi = _mm_load_pd(left + (1 * 2 + 1));
|
||||
b = _mm_shuffle_pd(b_hi, b_hi, 0x0);
|
||||
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
|
||||
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
|
||||
|
||||
/* k = 3 */
|
||||
a_lo = _mm_load_pd(left + (1 * 2 + 0));
|
||||
a_hi = _mm_load_pd(left + (1 * 2 + 1));
|
||||
b = _mm_shuffle_pd(b_hi, b_hi, 0x3);
|
||||
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
|
||||
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
|
||||
|
||||
ans->f64_x[col * 2 + 0] = c_lo;
|
||||
ans->f64_x[col * 2 + 1] = c_hi;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
|
||||
{
|
||||
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
|
||||
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
|
||||
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
|
||||
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
|
||||
|
||||
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
|
||||
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
|
||||
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
|
||||
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
|
||||
|
||||
register __m256d ccol;
|
||||
|
||||
/* Column 0 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3))));
|
||||
ans->f64_y[0] = ccol;
|
||||
/* Column 1 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3))));
|
||||
ans->f64_y[1] = ccol;
|
||||
/* Column 2 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3))));
|
||||
ans->f64_y[2] = ccol;
|
||||
/* Column 3 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2))));
|
||||
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3))));
|
||||
ans->f64_y[3] = ccol;
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
|
||||
{
|
||||
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
|
||||
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
|
||||
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
|
||||
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
|
||||
|
||||
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
|
||||
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
|
||||
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
|
||||
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
|
||||
|
||||
register __m256d ccol;
|
||||
|
||||
/* Column 0 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
|
||||
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3)), ccol);
|
||||
ans->f64_y[0] = ccol;
|
||||
/* Column 1 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
|
||||
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3)), ccol);
|
||||
ans->f64_y[1] = ccol;
|
||||
/* Column 2 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
|
||||
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3)), ccol);
|
||||
ans->f64_y[2] = ccol;
|
||||
/* Column 3 */
|
||||
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
|
||||
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2)), ccol);
|
||||
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3)), ccol);
|
||||
ans->f64_y[3] = ccol;
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
|
||||
{
|
||||
static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
|
||||
static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
|
||||
|
||||
register __m512d a01 = _mm512_load_pd((double *)&left->f64_z[0]);
|
||||
register __m512d a23 = _mm512_load_pd((double *)&left->f64_z[1]);
|
||||
|
||||
register __m512d b01 = _mm512_load_pd((double *)&right->f64_z[0]);
|
||||
register __m512d b23 = _mm512_load_pd((double *)&right->f64_z[1]);
|
||||
b01 = _mm512_permutexvar_pd(b01_shift, b01);
|
||||
b23 = _mm512_permutexvar_pd(b23_shift, b23);
|
||||
|
||||
register __m512d c01 = _mm512_load_pd((float *)&ans->f64_z[0]);
|
||||
register __m512d c23 = _mm512_load_pd((float *)&ans->f64_z[1]);
|
||||
|
||||
//group 0
|
||||
register __m512d b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(0));
|
||||
register __m512d b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(0));
|
||||
c01 = _mm512_fmadd_pd(a01, b01_perm, c01);
|
||||
c23 = _mm512_fmadd_pd(a23, b23_perm, c23);
|
||||
|
||||
//group 2
|
||||
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(2));
|
||||
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(2));
|
||||
c01 = _mm512_fmadd_pd(a23, b01_perm, c01);
|
||||
c23 = _mm512_fmadd_pd(a01, b23_perm, c23);
|
||||
|
||||
//group 1
|
||||
register __m512d a12 = _mm512_shuffle_f64x2(a01, a23, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
register __m512d a30 = _mm512_shuffle_f64x2(a23, a01, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(1));
|
||||
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(1));
|
||||
c01 = _mm512_fmadd_pd(a12, b01_perm, c01);
|
||||
c23 = _mm512_fmadd_pd(a30, b23_perm, c23);
|
||||
|
||||
//group 3
|
||||
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(3));
|
||||
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(3));
|
||||
c01 = _mm512_fmadd_pd(a30, b01_perm, c01);
|
||||
c23 = _mm512_fmadd_pd(a12, b23_perm, c23);
|
||||
|
||||
ans->f64_z[0] = c01;
|
||||
ans->f64_z[1] = c23;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,301 +1,301 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_matkernel.h"
|
||||
|
||||
DECLARE_BUILD_PANEL(avx, float)
|
||||
{
|
||||
const int Q = 8; /* fixed for AVX float panel */
|
||||
|
||||
/* Neighbor kernels; substitute global zero kernel where out-of-bounds */
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index);
|
||||
const MAT_KERNEL(float) *const k_rt =
|
||||
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
|
||||
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(float) *const k_lb =
|
||||
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(float) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
|
||||
|
||||
register __m128 top128, bot128;
|
||||
register __m256 col256;
|
||||
|
||||
for (int pc = 0; pc < Q; ++pc) {
|
||||
const int is_right_half = (pc >= 4);
|
||||
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
|
||||
|
||||
const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt;
|
||||
const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb;
|
||||
|
||||
/* Vector loads: each kernel column is a compact __m128 */
|
||||
top128 = src_top->f32_x[kc_col];
|
||||
bot128 = src_bot->f32_x[kc_col];
|
||||
|
||||
/* Combine into an AVX column: low 128 = top, high 128 = bottom */
|
||||
col256 = _mm256_set_m128(bot128, top128);
|
||||
|
||||
PANEL_COL_VEC((*panel), pc, 0) = col256;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUILD_PANEL(512, float)
|
||||
{
|
||||
const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(float) * k[4][4];
|
||||
|
||||
for (int kc = 0; kc < 4; ++kc) {
|
||||
for (int kr = 0; kr < 4; ++kr) {
|
||||
const int col = col_index + kc;
|
||||
const int row = row_index + kr;
|
||||
k[kc][kr] = (col < col_length && row < row_length) ?
|
||||
(matrix + (row * col_length + col)) :
|
||||
zero;
|
||||
}
|
||||
}
|
||||
|
||||
/* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */
|
||||
for (int pc = 0; pc < 16; ++pc) {
|
||||
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
|
||||
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
|
||||
|
||||
const __m128 r0 = k[blk_col][0]->f32_x[kc_col];
|
||||
const __m128 r1 = k[blk_col][1]->f32_x[kc_col];
|
||||
const __m128 r2 = k[blk_col][2]->f32_x[kc_col];
|
||||
const __m128 r3 = k[blk_col][3]->f32_x[kc_col];
|
||||
|
||||
const __m256 top = _mm256_set_m128(r1, r0);
|
||||
const __m256 bot = _mm256_set_m128(r3, r2);
|
||||
|
||||
__m512 col = _mm512_castps256_ps512(top);
|
||||
col = _mm512_insertf32x8(col, bot, 1);
|
||||
|
||||
PANEL_COL_VEC((*panel), pc, 0) = col;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUILD_PANEL(512, double)
|
||||
{
|
||||
const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */
|
||||
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index);
|
||||
const MAT_KERNEL(double) *const k_rt =
|
||||
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
|
||||
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(double) *const k_lb =
|
||||
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(double) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
|
||||
|
||||
for (int pc = 0; pc < Q; ++pc) {
|
||||
const int is_right_half = (pc >= 4);
|
||||
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
|
||||
|
||||
const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt;
|
||||
const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb;
|
||||
|
||||
const __m256d top256 = src_top->f64_y[kc_col];
|
||||
const __m256d bot256 = src_bot->f64_y[kc_col];
|
||||
|
||||
__m512d col = _mm512_castpd256_pd512(top256);
|
||||
col = _mm512_insertf64x4(col, bot256, 1);
|
||||
|
||||
PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col);
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(avx, float)
|
||||
{
|
||||
const int Q = 8; /* fixed for AVX float panel */
|
||||
|
||||
/* Local dummy kernel to swallow out-of-bounds writes; no init needed */
|
||||
MAT_KERNEL(float) zero;
|
||||
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(float) *const k_rt =
|
||||
has_right && (row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
MAT_KERNEL(float) *const k_lb =
|
||||
has_bottom && (col_index < col_length) ?
|
||||
(matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(float) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
|
||||
register __m128 top128, bot128;
|
||||
int i = 0;
|
||||
for (; i < Q / 2; i++) {
|
||||
const int kc_col = i; /* column index inside 4x4 kernel */
|
||||
|
||||
/* Load the panel column vector */
|
||||
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
|
||||
/* Split into top/bottom halves */
|
||||
top128 = _mm256_castps256_ps128(col256); /* low 128 */
|
||||
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
|
||||
|
||||
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
|
||||
k_rt->f32_x[kc_col] = top128;
|
||||
k_rb->f32_x[kc_col] = bot128;
|
||||
}
|
||||
for (; i < Q; i++) {
|
||||
const int kc_col = i - 4; /* column index inside 4x4 kernel */
|
||||
|
||||
/* Load the panel column vector */
|
||||
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
|
||||
/* Split into top/bottom halves */
|
||||
top128 = _mm256_castps256_ps128(col256); /* low 128 */
|
||||
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
|
||||
|
||||
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
|
||||
k_lt->f32_x[kc_col] = top128;
|
||||
k_lb->f32_x[kc_col] = bot128;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(512, float)
|
||||
{
|
||||
MAT_KERNEL(float) zero;
|
||||
MAT_KERNEL(float) * k[4][4];
|
||||
|
||||
for (int kc = 0; kc < 4; ++kc) {
|
||||
for (int kr = 0; kr < 4; ++kr) {
|
||||
const int col = col_index + kc;
|
||||
const int row = row_index + kr;
|
||||
k[kc][kr] = (col < col_length && row < row_length) ?
|
||||
(matrix + (row * col_length + col)) :
|
||||
&zero;
|
||||
}
|
||||
}
|
||||
|
||||
for (int pc = 0; pc < 16; ++pc) {
|
||||
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
|
||||
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
|
||||
|
||||
const __m512 col = PANEL_COL_VEC((*panel), pc, 0);
|
||||
|
||||
const __m256 top = _mm512_castps512_ps256(col);
|
||||
const __m256 bot = _mm512_extractf32x8_ps(col, 1);
|
||||
|
||||
const __m128 r0 = _mm256_castps256_ps128(top);
|
||||
const __m128 r1 = _mm256_extractf128_ps(top, 1);
|
||||
const __m128 r2 = _mm256_castps256_ps128(bot);
|
||||
const __m128 r3 = _mm256_extractf128_ps(bot, 1);
|
||||
|
||||
k[blk_col][0]->f32_x[kc_col] = r0;
|
||||
k[blk_col][1]->f32_x[kc_col] = r1;
|
||||
k[blk_col][2]->f32_x[kc_col] = r2;
|
||||
k[blk_col][3]->f32_x[kc_col] = r3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(sse, double)
|
||||
{
|
||||
MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index);
|
||||
|
||||
for (int pc = 0; pc < 4; ++pc) {
|
||||
const int base = pc * 2;
|
||||
dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0));
|
||||
dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1));
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(avx, double)
|
||||
{
|
||||
const int Q = 8; /* fixed for AVX double panel */
|
||||
|
||||
MAT_KERNEL(double) zero;
|
||||
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_rt =
|
||||
has_right && (row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_lb =
|
||||
has_bottom && (col_index < col_length) ?
|
||||
(matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
|
||||
for (int i = 0; i < Q; ++i) {
|
||||
const int kc_col = (i < 4) ? i : (i - 4);
|
||||
const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0);
|
||||
const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1);
|
||||
|
||||
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
|
||||
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
|
||||
|
||||
top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk);
|
||||
bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk);
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(512, double)
|
||||
{
|
||||
const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */
|
||||
|
||||
MAT_KERNEL(double) zero;
|
||||
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_rt =
|
||||
has_right && (row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_lb =
|
||||
has_bottom && (col_index < col_length) ?
|
||||
(matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
|
||||
for (int i = 0; i < Q; ++i) {
|
||||
const int kc_col = (i < 4) ? i : (i - 4);
|
||||
const __m512 col = PANEL_COL_VEC((*panel), i, 0);
|
||||
const __m512d col_pd = _mm512_castps_pd(col);
|
||||
|
||||
const __m256d top256 = _mm512_castpd512_pd256(col_pd);
|
||||
const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1);
|
||||
|
||||
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
|
||||
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
|
||||
|
||||
top_dst->f64_y[kc_col] = top256;
|
||||
bot_dst->f64_y[kc_col] = bot256;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_matkernel.h"
|
||||
|
||||
DECLARE_BUILD_PANEL(avx, float)
|
||||
{
|
||||
const int Q = 8; /* fixed for AVX float panel */
|
||||
|
||||
/* Neighbor kernels; substitute global zero kernel where out-of-bounds */
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index);
|
||||
const MAT_KERNEL(float) *const k_rt =
|
||||
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
|
||||
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(float) *const k_lb =
|
||||
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(float) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
|
||||
|
||||
register __m128 top128, bot128;
|
||||
register __m256 col256;
|
||||
|
||||
for (int pc = 0; pc < Q; ++pc) {
|
||||
const int is_right_half = (pc >= 4);
|
||||
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
|
||||
|
||||
const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt;
|
||||
const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb;
|
||||
|
||||
/* Vector loads: each kernel column is a compact __m128 */
|
||||
top128 = src_top->f32_x[kc_col];
|
||||
bot128 = src_bot->f32_x[kc_col];
|
||||
|
||||
/* Combine into an AVX column: low 128 = top, high 128 = bottom */
|
||||
col256 = _mm256_set_m128(bot128, top128);
|
||||
|
||||
PANEL_COL_VEC((*panel), pc, 0) = col256;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUILD_PANEL(512, float)
|
||||
{
|
||||
const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(float) * k[4][4];
|
||||
|
||||
for (int kc = 0; kc < 4; ++kc) {
|
||||
for (int kr = 0; kr < 4; ++kr) {
|
||||
const int col = col_index + kc;
|
||||
const int row = row_index + kr;
|
||||
k[kc][kr] = (col < col_length && row < row_length) ?
|
||||
(matrix + (row * col_length + col)) :
|
||||
zero;
|
||||
}
|
||||
}
|
||||
|
||||
/* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */
|
||||
for (int pc = 0; pc < 16; ++pc) {
|
||||
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
|
||||
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
|
||||
|
||||
const __m128 r0 = k[blk_col][0]->f32_x[kc_col];
|
||||
const __m128 r1 = k[blk_col][1]->f32_x[kc_col];
|
||||
const __m128 r2 = k[blk_col][2]->f32_x[kc_col];
|
||||
const __m128 r3 = k[blk_col][3]->f32_x[kc_col];
|
||||
|
||||
const __m256 top = _mm256_set_m128(r1, r0);
|
||||
const __m256 bot = _mm256_set_m128(r3, r2);
|
||||
|
||||
__m512 col = _mm512_castps256_ps512(top);
|
||||
col = _mm512_insertf32x8(col, bot, 1);
|
||||
|
||||
PANEL_COL_VEC((*panel), pc, 0) = col;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUILD_PANEL(512, double)
|
||||
{
|
||||
const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */
|
||||
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index);
|
||||
const MAT_KERNEL(double) *const k_rt =
|
||||
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
|
||||
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(double) *const k_lb =
|
||||
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
|
||||
const MAT_KERNEL(double) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
|
||||
|
||||
for (int pc = 0; pc < Q; ++pc) {
|
||||
const int is_right_half = (pc >= 4);
|
||||
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
|
||||
|
||||
const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt;
|
||||
const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb;
|
||||
|
||||
const __m256d top256 = src_top->f64_y[kc_col];
|
||||
const __m256d bot256 = src_bot->f64_y[kc_col];
|
||||
|
||||
__m512d col = _mm512_castpd256_pd512(top256);
|
||||
col = _mm512_insertf64x4(col, bot256, 1);
|
||||
|
||||
PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col);
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(avx, float)
|
||||
{
|
||||
const int Q = 8; /* fixed for AVX float panel */
|
||||
|
||||
/* Local dummy kernel to swallow out-of-bounds writes; no init needed */
|
||||
MAT_KERNEL(float) zero;
|
||||
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(float) *const k_rt =
|
||||
has_right && (row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
MAT_KERNEL(float) *const k_lb =
|
||||
has_bottom && (col_index < col_length) ?
|
||||
(matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(float) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
|
||||
register __m128 top128, bot128;
|
||||
int i = 0;
|
||||
for (; i < Q / 2; i++) {
|
||||
const int kc_col = i; /* column index inside 4x4 kernel */
|
||||
|
||||
/* Load the panel column vector */
|
||||
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
|
||||
/* Split into top/bottom halves */
|
||||
top128 = _mm256_castps256_ps128(col256); /* low 128 */
|
||||
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
|
||||
|
||||
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
|
||||
k_rt->f32_x[kc_col] = top128;
|
||||
k_rb->f32_x[kc_col] = bot128;
|
||||
}
|
||||
for (; i < Q; i++) {
|
||||
const int kc_col = i - 4; /* column index inside 4x4 kernel */
|
||||
|
||||
/* Load the panel column vector */
|
||||
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
|
||||
/* Split into top/bottom halves */
|
||||
top128 = _mm256_castps256_ps128(col256); /* low 128 */
|
||||
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
|
||||
|
||||
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
|
||||
k_lt->f32_x[kc_col] = top128;
|
||||
k_lb->f32_x[kc_col] = bot128;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(512, float)
|
||||
{
|
||||
MAT_KERNEL(float) zero;
|
||||
MAT_KERNEL(float) * k[4][4];
|
||||
|
||||
for (int kc = 0; kc < 4; ++kc) {
|
||||
for (int kr = 0; kr < 4; ++kr) {
|
||||
const int col = col_index + kc;
|
||||
const int row = row_index + kr;
|
||||
k[kc][kr] = (col < col_length && row < row_length) ?
|
||||
(matrix + (row * col_length + col)) :
|
||||
&zero;
|
||||
}
|
||||
}
|
||||
|
||||
for (int pc = 0; pc < 16; ++pc) {
|
||||
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
|
||||
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
|
||||
|
||||
const __m512 col = PANEL_COL_VEC((*panel), pc, 0);
|
||||
|
||||
const __m256 top = _mm512_castps512_ps256(col);
|
||||
const __m256 bot = _mm512_extractf32x8_ps(col, 1);
|
||||
|
||||
const __m128 r0 = _mm256_castps256_ps128(top);
|
||||
const __m128 r1 = _mm256_extractf128_ps(top, 1);
|
||||
const __m128 r2 = _mm256_castps256_ps128(bot);
|
||||
const __m128 r3 = _mm256_extractf128_ps(bot, 1);
|
||||
|
||||
k[blk_col][0]->f32_x[kc_col] = r0;
|
||||
k[blk_col][1]->f32_x[kc_col] = r1;
|
||||
k[blk_col][2]->f32_x[kc_col] = r2;
|
||||
k[blk_col][3]->f32_x[kc_col] = r3;
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(sse, double)
|
||||
{
|
||||
MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index);
|
||||
|
||||
for (int pc = 0; pc < 4; ++pc) {
|
||||
const int base = pc * 2;
|
||||
dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0));
|
||||
dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1));
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(avx, double)
|
||||
{
|
||||
const int Q = 8; /* fixed for AVX double panel */
|
||||
|
||||
MAT_KERNEL(double) zero;
|
||||
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_rt =
|
||||
has_right && (row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_lb =
|
||||
has_bottom && (col_index < col_length) ?
|
||||
(matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
|
||||
for (int i = 0; i < Q; ++i) {
|
||||
const int kc_col = (i < 4) ? i : (i - 4);
|
||||
const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0);
|
||||
const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1);
|
||||
|
||||
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
|
||||
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
|
||||
|
||||
top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk);
|
||||
bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk);
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_STORE_PANEL(512, double)
|
||||
{
|
||||
const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */
|
||||
|
||||
MAT_KERNEL(double) zero;
|
||||
|
||||
const int has_right = (col_index + 1) < col_length;
|
||||
const int has_bottom = (row_index + 1) < row_length;
|
||||
|
||||
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_rt =
|
||||
has_right && (row_index < row_length) ?
|
||||
(matrix + (row_index * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_lb =
|
||||
has_bottom && (col_index < col_length) ?
|
||||
(matrix + ((row_index + 1) * col_length + col_index)) :
|
||||
&zero;
|
||||
MAT_KERNEL(double) *const k_rb =
|
||||
(has_right && has_bottom) ?
|
||||
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
|
||||
&zero;
|
||||
|
||||
for (int i = 0; i < Q; ++i) {
|
||||
const int kc_col = (i < 4) ? i : (i - 4);
|
||||
const __m512 col = PANEL_COL_VEC((*panel), i, 0);
|
||||
const __m512d col_pd = _mm512_castps_pd(col);
|
||||
|
||||
const __m256d top256 = _mm512_castpd512_pd256(col_pd);
|
||||
const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1);
|
||||
|
||||
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
|
||||
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
|
||||
|
||||
top_dst->f64_y[kc_col] = top256;
|
||||
bot_dst->f64_y[kc_col] = bot256;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,180 +1,180 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include <math.h>
|
||||
|
||||
static const float quality_threshold = 1e-6f;
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu)
|
||||
{
|
||||
/* Aliases */
|
||||
MAT_KERNEL(float) *upper = left; /* U */
|
||||
MAT_KERNEL(float) *lower = right; /* L (unit diag) */
|
||||
MAT_KERNEL(float) *source = ans; /* A */
|
||||
unsigned char *pivot = (unsigned char *)&scalar->data[0]; /* length = 4 */
|
||||
int *success = (int *)&scalar->data[1];
|
||||
|
||||
/* Init permutation and outputs */
|
||||
pivot[0] = 0;
|
||||
pivot[1] = 1;
|
||||
pivot[2] = 2;
|
||||
pivot[3] = 3;
|
||||
*success = 1;
|
||||
|
||||
/* Working copy of A in column-major a[col][row], load via SIMD view */
|
||||
MAT_KERNEL(float) a;
|
||||
const __m128 z = _mm_setzero_ps();
|
||||
for (int c = 0; c < 4; ++c) {
|
||||
__m128 col = _mm_load_ps((float *)&source->f32_x[c]);
|
||||
_mm_storeu_ps((float *)&a.f32_x[c], col);
|
||||
_mm_store_ps((float *)&upper->f32_x[c], z);
|
||||
_mm_store_ps((float *)&lower->f32_x[c], z);
|
||||
}
|
||||
|
||||
const float eps = 1e-12f;
|
||||
|
||||
for (int k = 0; k < 4; ++k) {
|
||||
/* Pivot search on column k, rows k..3 */
|
||||
int p = k;
|
||||
float maxabs = fabsf(a.m[k][p]);
|
||||
for (int r = k + 1; r < 4; ++r) {
|
||||
float v = fabsf(a.m[k][r]);
|
||||
if (v > maxabs) {
|
||||
maxabs = v;
|
||||
p = r;
|
||||
}
|
||||
}
|
||||
|
||||
if (maxabs < eps) {
|
||||
*success = 0;
|
||||
/* leave outputs zeroed */
|
||||
return;
|
||||
}
|
||||
|
||||
/* Swap rows if needed */
|
||||
if (p != k) {
|
||||
for (int c = 0; c < 4; ++c) {
|
||||
float tmp = a.m[c][k];
|
||||
a.m[c][k] = a.m[c][p];
|
||||
a.m[c][p] = tmp;
|
||||
}
|
||||
unsigned char tmpi = pivot[k];
|
||||
pivot[k] = pivot[p];
|
||||
pivot[p] = tmpi;
|
||||
}
|
||||
|
||||
float pivot_val = a.m[k][k];
|
||||
|
||||
/* Eliminate below pivot: store multipliers in L column k */
|
||||
for (int r = k + 1; r < 4; ++r) {
|
||||
float factor = a.m[k][r] / pivot_val;
|
||||
lower->m[k][r] = factor;
|
||||
for (int c = k; c < 4; ++c) {
|
||||
a.m[c][r] -= factor * a.m[c][k];
|
||||
}
|
||||
}
|
||||
|
||||
/* Set U row k (columns k..3) and L diagonal */
|
||||
lower->m[k][k] = 1.0f;
|
||||
for (int c = k; c < 4; ++c) {
|
||||
upper->m[c][k] = a.m[c][k];
|
||||
}
|
||||
}
|
||||
|
||||
/* Ensure remaining diagonal ones for L */
|
||||
for (int d = 0; d < 4; ++d) {
|
||||
if (lower->m[d][d] == 0.0f)
|
||||
lower->m[d][d] = 1.0f;
|
||||
}
|
||||
}
|
||||
|
||||
#define DOT_F32_SSE(a, b, ans) \
|
||||
do { \
|
||||
__m128 _p = _mm_mul_ps((a), (b)); /* p0 p1 p2 p3 */ \
|
||||
__m128 _t = _mm_shuffle_ps(_p, _p, _MM_SHUFFLE(2, 3, 0, 1)); \
|
||||
_p = _mm_add_ps(_p, _t); \
|
||||
_t = _mm_movehl_ps(_t, _p); \
|
||||
ans = _mm_cvtss_f32(_mm_add_ss(_p, _t)); \
|
||||
} while (0)
|
||||
|
||||
#define FIND_MAX_F32_SSE(a, b) \
|
||||
do { \
|
||||
__m128 _t = _mm_movehl_ps(a, a); \
|
||||
a = _mm_max_ps(a, _t); \
|
||||
_t = _mm_shuffle_ps(a, a, _MM_SHUFFLE(2, 3, 0, 1)); \
|
||||
b = _mm_max_ps(a, _t); \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* layout of scalarpack:
|
||||
*
|
||||
*/
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality)
|
||||
{
|
||||
static const __m128 tiny_source = { .m128_f32 = { 1e-6f, 1e-6f, 1e-6f, 1e-6f } };
|
||||
|
||||
register __m128 tiny = _mm_load_ps(tiny_source.m128_f32);
|
||||
|
||||
register __m128 col0 = _mm_load_ps((float *)&right->f32_x[0]);
|
||||
register __m128 col1 = _mm_load_ps((float *)&right->f32_x[1]);
|
||||
register __m128 col2 = _mm_load_ps((float *)&right->f32_x[2]);
|
||||
register __m128 col3 = _mm_load_ps((float *)&right->f32_x[3]);
|
||||
|
||||
__m128 self_dot_src = { 0 };
|
||||
// try in reigister mix later
|
||||
DOT_F32_SSE(col0, col0, self_dot_src.m128_f32[0]);
|
||||
DOT_F32_SSE(col1, col1, self_dot_src.m128_f32[1]);
|
||||
DOT_F32_SSE(col2, col2, self_dot_src.m128_f32[2]);
|
||||
DOT_F32_SSE(col3, col3, self_dot_src.m128_f32[3]);
|
||||
|
||||
register __m128 self_dot = _mm_load_ps(self_dot_src.m128_f32);
|
||||
|
||||
register __m128 max;
|
||||
FIND_MAX_F32_SSE(self_dot, max);
|
||||
//float norm = _mm_cvtss_f32(max);
|
||||
|
||||
// min on cross
|
||||
register __m128 col_shuff_1 = _mm_shuffle_ps(col1, col1, _MM_SHUFFLE(0, 3, 2, 1));
|
||||
register __m128 col_shuff_2 = _mm_shuffle_ps(col2, col2, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
register __m128 col_shuff_3 = _mm_shuffle_ps(col3, col3, _MM_SHUFFLE(2, 1, 0, 3));
|
||||
register __m128 min = _mm_min_ps(col0, col_shuff_1);
|
||||
register __m128 min0 = _mm_min_ps(col_shuff_2, col_shuff_3);
|
||||
min = _mm_min_ps(min, min0);
|
||||
|
||||
float quality = _mm_cvtss_f32(_mm_div_ps(min, _mm_add_ps(max, tiny)));
|
||||
ans->f32_x[0].m128_f32[0] = quality;
|
||||
if (quality < quality_threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
__m128 cross_dot_src[2];
|
||||
DOT_F32_SSE(col0, col1, cross_dot_src[0].m128_f32[0]);
|
||||
DOT_F32_SSE(col0, col2, cross_dot_src[0].m128_f32[1]);
|
||||
DOT_F32_SSE(col0, col3, cross_dot_src[0].m128_f32[2]);
|
||||
DOT_F32_SSE(col1, col2, cross_dot_src[0].m128_f32[3]);
|
||||
DOT_F32_SSE(col1, col3, cross_dot_src[1].m128_f32[0]);
|
||||
DOT_F32_SSE(col2, col3, cross_dot_src[1].m128_f32[1]);
|
||||
|
||||
register __m128 cross_dot_0 = _mm_load_ps(cross_dot_src[0].m128_f32);
|
||||
register __m128 cross_dot_1 = _mm_load_ps(cross_dot_src[1].m128_f32);
|
||||
|
||||
register __m128 base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(1, 0, 0, 0));
|
||||
register __m128 base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 2, 1));
|
||||
|
||||
register __m128 relation_0 =
|
||||
_mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
|
||||
|
||||
base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(2, 3, 2, 1));
|
||||
base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 3, 3));
|
||||
|
||||
register __m128 relation_1 =
|
||||
_mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
|
||||
|
||||
relation_0 = _mm_max_ps(relation_0, relation_1);
|
||||
FIND_MAX_F32_SSE(relation_0, relation_0);
|
||||
ans->f32_x[0].m128_f32[1] = _mm_cvtss_f32(relation_0);
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include <math.h>
|
||||
|
||||
static const float quality_threshold = 1e-6f;
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu)
|
||||
{
|
||||
/* Aliases */
|
||||
MAT_KERNEL(float) *upper = left; /* U */
|
||||
MAT_KERNEL(float) *lower = right; /* L (unit diag) */
|
||||
MAT_KERNEL(float) *source = ans; /* A */
|
||||
unsigned char *pivot = (unsigned char *)&scalar->data[0]; /* length = 4 */
|
||||
int *success = (int *)&scalar->data[1];
|
||||
|
||||
/* Init permutation and outputs */
|
||||
pivot[0] = 0;
|
||||
pivot[1] = 1;
|
||||
pivot[2] = 2;
|
||||
pivot[3] = 3;
|
||||
*success = 1;
|
||||
|
||||
/* Working copy of A in column-major a[col][row], load via SIMD view */
|
||||
MAT_KERNEL(float) a;
|
||||
const __m128 z = _mm_setzero_ps();
|
||||
for (int c = 0; c < 4; ++c) {
|
||||
__m128 col = _mm_load_ps((float *)&source->f32_x[c]);
|
||||
_mm_storeu_ps((float *)&a.f32_x[c], col);
|
||||
_mm_store_ps((float *)&upper->f32_x[c], z);
|
||||
_mm_store_ps((float *)&lower->f32_x[c], z);
|
||||
}
|
||||
|
||||
const float eps = 1e-12f;
|
||||
|
||||
for (int k = 0; k < 4; ++k) {
|
||||
/* Pivot search on column k, rows k..3 */
|
||||
int p = k;
|
||||
float maxabs = fabsf(a.m[k][p]);
|
||||
for (int r = k + 1; r < 4; ++r) {
|
||||
float v = fabsf(a.m[k][r]);
|
||||
if (v > maxabs) {
|
||||
maxabs = v;
|
||||
p = r;
|
||||
}
|
||||
}
|
||||
|
||||
if (maxabs < eps) {
|
||||
*success = 0;
|
||||
/* leave outputs zeroed */
|
||||
return;
|
||||
}
|
||||
|
||||
/* Swap rows if needed */
|
||||
if (p != k) {
|
||||
for (int c = 0; c < 4; ++c) {
|
||||
float tmp = a.m[c][k];
|
||||
a.m[c][k] = a.m[c][p];
|
||||
a.m[c][p] = tmp;
|
||||
}
|
||||
unsigned char tmpi = pivot[k];
|
||||
pivot[k] = pivot[p];
|
||||
pivot[p] = tmpi;
|
||||
}
|
||||
|
||||
float pivot_val = a.m[k][k];
|
||||
|
||||
/* Eliminate below pivot: store multipliers in L column k */
|
||||
for (int r = k + 1; r < 4; ++r) {
|
||||
float factor = a.m[k][r] / pivot_val;
|
||||
lower->m[k][r] = factor;
|
||||
for (int c = k; c < 4; ++c) {
|
||||
a.m[c][r] -= factor * a.m[c][k];
|
||||
}
|
||||
}
|
||||
|
||||
/* Set U row k (columns k..3) and L diagonal */
|
||||
lower->m[k][k] = 1.0f;
|
||||
for (int c = k; c < 4; ++c) {
|
||||
upper->m[c][k] = a.m[c][k];
|
||||
}
|
||||
}
|
||||
|
||||
/* Ensure remaining diagonal ones for L */
|
||||
for (int d = 0; d < 4; ++d) {
|
||||
if (lower->m[d][d] == 0.0f)
|
||||
lower->m[d][d] = 1.0f;
|
||||
}
|
||||
}
|
||||
|
||||
#define DOT_F32_SSE(a, b, ans) \
|
||||
do { \
|
||||
__m128 _p = _mm_mul_ps((a), (b)); /* p0 p1 p2 p3 */ \
|
||||
__m128 _t = _mm_shuffle_ps(_p, _p, _MM_SHUFFLE(2, 3, 0, 1)); \
|
||||
_p = _mm_add_ps(_p, _t); \
|
||||
_t = _mm_movehl_ps(_t, _p); \
|
||||
ans = _mm_cvtss_f32(_mm_add_ss(_p, _t)); \
|
||||
} while (0)
|
||||
|
||||
#define FIND_MAX_F32_SSE(a, b) \
|
||||
do { \
|
||||
__m128 _t = _mm_movehl_ps(a, a); \
|
||||
a = _mm_max_ps(a, _t); \
|
||||
_t = _mm_shuffle_ps(a, a, _MM_SHUFFLE(2, 3, 0, 1)); \
|
||||
b = _mm_max_ps(a, _t); \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* layout of scalarpack:
|
||||
*
|
||||
*/
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality)
|
||||
{
|
||||
static const __m128 tiny_source = { .m128_f32 = { 1e-6f, 1e-6f, 1e-6f, 1e-6f } };
|
||||
|
||||
register __m128 tiny = _mm_load_ps(tiny_source.m128_f32);
|
||||
|
||||
register __m128 col0 = _mm_load_ps((float *)&right->f32_x[0]);
|
||||
register __m128 col1 = _mm_load_ps((float *)&right->f32_x[1]);
|
||||
register __m128 col2 = _mm_load_ps((float *)&right->f32_x[2]);
|
||||
register __m128 col3 = _mm_load_ps((float *)&right->f32_x[3]);
|
||||
|
||||
__m128 self_dot_src = { 0 };
|
||||
// try in reigister mix later
|
||||
DOT_F32_SSE(col0, col0, self_dot_src.m128_f32[0]);
|
||||
DOT_F32_SSE(col1, col1, self_dot_src.m128_f32[1]);
|
||||
DOT_F32_SSE(col2, col2, self_dot_src.m128_f32[2]);
|
||||
DOT_F32_SSE(col3, col3, self_dot_src.m128_f32[3]);
|
||||
|
||||
register __m128 self_dot = _mm_load_ps(self_dot_src.m128_f32);
|
||||
|
||||
register __m128 max;
|
||||
FIND_MAX_F32_SSE(self_dot, max);
|
||||
//float norm = _mm_cvtss_f32(max);
|
||||
|
||||
// min on cross
|
||||
register __m128 col_shuff_1 = _mm_shuffle_ps(col1, col1, _MM_SHUFFLE(0, 3, 2, 1));
|
||||
register __m128 col_shuff_2 = _mm_shuffle_ps(col2, col2, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
register __m128 col_shuff_3 = _mm_shuffle_ps(col3, col3, _MM_SHUFFLE(2, 1, 0, 3));
|
||||
register __m128 min = _mm_min_ps(col0, col_shuff_1);
|
||||
register __m128 min0 = _mm_min_ps(col_shuff_2, col_shuff_3);
|
||||
min = _mm_min_ps(min, min0);
|
||||
|
||||
float quality = _mm_cvtss_f32(_mm_div_ps(min, _mm_add_ps(max, tiny)));
|
||||
ans->f32_x[0].m128_f32[0] = quality;
|
||||
if (quality < quality_threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
__m128 cross_dot_src[2];
|
||||
DOT_F32_SSE(col0, col1, cross_dot_src[0].m128_f32[0]);
|
||||
DOT_F32_SSE(col0, col2, cross_dot_src[0].m128_f32[1]);
|
||||
DOT_F32_SSE(col0, col3, cross_dot_src[0].m128_f32[2]);
|
||||
DOT_F32_SSE(col1, col2, cross_dot_src[0].m128_f32[3]);
|
||||
DOT_F32_SSE(col1, col3, cross_dot_src[1].m128_f32[0]);
|
||||
DOT_F32_SSE(col2, col3, cross_dot_src[1].m128_f32[1]);
|
||||
|
||||
register __m128 cross_dot_0 = _mm_load_ps(cross_dot_src[0].m128_f32);
|
||||
register __m128 cross_dot_1 = _mm_load_ps(cross_dot_src[1].m128_f32);
|
||||
|
||||
register __m128 base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(1, 0, 0, 0));
|
||||
register __m128 base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 2, 1));
|
||||
|
||||
register __m128 relation_0 =
|
||||
_mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
|
||||
|
||||
base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(2, 3, 2, 1));
|
||||
base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 3, 3));
|
||||
|
||||
register __m128 relation_1 =
|
||||
_mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
|
||||
|
||||
relation_0 = _mm_max_ps(relation_0, relation_1);
|
||||
FIND_MAX_F32_SSE(relation_0, relation_0);
|
||||
ans->f32_x[0].m128_f32[1] = _mm_cvtss_f32(relation_0);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,29 +1,29 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_internal.h"
|
||||
#include "sn_intrinsic.h"
|
||||
|
||||
INTERNAL __m128i facIncrement = { 4, 4, 4, 4 };
|
||||
INTERNAL __m128i facBase = { 1, 2, 3, 4 };
|
||||
|
||||
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n)
|
||||
{
|
||||
__m128i baseResult = facBase;
|
||||
__m128i facBaseLocal = facBase;
|
||||
const int cycle = (n - 3) >> 2;
|
||||
const uint32_t rest = (cycle << 2) + 1;
|
||||
for (size_t i = 0; i < cycle; i++) {
|
||||
facBaseLocal = _mm_add_epi32(facBaseLocal, facIncrement);
|
||||
baseResult = _mm_mullo_epi32(baseResult, facBaseLocal);
|
||||
}
|
||||
uint64_t result = (uint64_t)baseResult.m128i_i32[0] * (uint64_t)baseResult.m128i_i32[1] *
|
||||
(uint64_t)baseResult.m128i_i32[2] * (uint64_t)baseResult.m128i_i32[3];
|
||||
for (uint64_t i = rest; i < n; i++) {
|
||||
result *= i;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_internal.h"
|
||||
#include "sn_intrinsic.h"
|
||||
|
||||
INTERNAL __m128i facIncrement = { 4, 4, 4, 4 };
|
||||
INTERNAL __m128i facBase = { 1, 2, 3, 4 };
|
||||
|
||||
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n)
|
||||
{
|
||||
__m128i baseResult = facBase;
|
||||
__m128i facBaseLocal = facBase;
|
||||
const int cycle = (n - 3) >> 2;
|
||||
const uint32_t rest = (cycle << 2) + 1;
|
||||
for (size_t i = 0; i < cycle; i++) {
|
||||
facBaseLocal = _mm_add_epi32(facBaseLocal, facIncrement);
|
||||
baseResult = _mm_mullo_epi32(baseResult, facBaseLocal);
|
||||
}
|
||||
uint64_t result = (uint64_t)baseResult.m128i_i32[0] * (uint64_t)baseResult.m128i_i32[1] *
|
||||
(uint64_t)baseResult.m128i_i32[2] * (uint64_t)baseResult.m128i_i32[3];
|
||||
for (uint64_t i = rest; i < n; i++) {
|
||||
result *= i;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,51 +1,51 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include "StgSharpNative.h"
|
||||
|
||||
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length)
|
||||
{
|
||||
uint32_t hash = length, size = length << 1;
|
||||
byte *end = str + size, *ptr = str;
|
||||
|
||||
if (size < 4) {
|
||||
for (; ptr < end; ptr += 2) {
|
||||
hash = _mm_crc32_u16(hash, *ptr);
|
||||
}
|
||||
} else if (size < 16) {
|
||||
byte *last = end - 4;
|
||||
for (; ptr < end; ptr += 4) {
|
||||
hash = _mm_crc32_u32(hash, *(uint32_t *)ptr);
|
||||
}
|
||||
for (; ptr < end; ptr++) {
|
||||
hash = _mm_crc32_u16(hash, *ptr);
|
||||
}
|
||||
} else {
|
||||
byte *last_128 = end - 16;
|
||||
byte *last_32 = end - 4;
|
||||
uint64_t hash64 = size;
|
||||
for (; ptr < last_128; ptr += 16) {
|
||||
__m128i vec = _mm_loadu_si32((int *)ptr);
|
||||
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 0));
|
||||
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 1));
|
||||
}
|
||||
hash = _mm_crc32_u8(hash, hash64);
|
||||
hash = _mm_crc32_u8(hash, hash64 >> 32);
|
||||
for (; ptr < last_32; ptr += 4) {
|
||||
hash = _mm_crc32_u32(hash, *(uint32_t *)ptr);
|
||||
}
|
||||
for (; ptr < end; ptr += 2) {
|
||||
hash = _mm_crc32_u8(hash, *ptr);
|
||||
}
|
||||
}
|
||||
hash ^= hash >> 16;
|
||||
hash *= 0x85ebca6b;
|
||||
hash ^= hash >> 13;
|
||||
hash *= 0xc2b2ae35;
|
||||
hash ^= hash >> 16;
|
||||
return hash;
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include "StgSharpNative.h"
|
||||
|
||||
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length)
|
||||
{
|
||||
uint32_t hash = length, size = length << 1;
|
||||
byte *end = str + size, *ptr = str;
|
||||
|
||||
if (size < 4) {
|
||||
for (; ptr < end; ptr += 2) {
|
||||
hash = _mm_crc32_u16(hash, *ptr);
|
||||
}
|
||||
} else if (size < 16) {
|
||||
byte *last = end - 4;
|
||||
for (; ptr < end; ptr += 4) {
|
||||
hash = _mm_crc32_u32(hash, *(uint32_t *)ptr);
|
||||
}
|
||||
for (; ptr < end; ptr++) {
|
||||
hash = _mm_crc32_u16(hash, *ptr);
|
||||
}
|
||||
} else {
|
||||
byte *last_128 = end - 16;
|
||||
byte *last_32 = end - 4;
|
||||
uint64_t hash64 = size;
|
||||
for (; ptr < last_128; ptr += 16) {
|
||||
__m128i vec = _mm_loadu_si32((int *)ptr);
|
||||
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 0));
|
||||
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 1));
|
||||
}
|
||||
hash = _mm_crc32_u8(hash, hash64);
|
||||
hash = _mm_crc32_u8(hash, hash64 >> 32);
|
||||
for (; ptr < last_32; ptr += 4) {
|
||||
hash = _mm_crc32_u32(hash, *(uint32_t *)ptr);
|
||||
}
|
||||
for (; ptr < end; ptr += 2) {
|
||||
hash = _mm_crc32_u8(hash, *ptr);
|
||||
}
|
||||
}
|
||||
hash ^= hash >> 16;
|
||||
hash *= 0x85ebca6b;
|
||||
hash ^= hash >> 13;
|
||||
hash *= 0xc2b2ae35;
|
||||
hash ^= hash >> 16;
|
||||
return hash;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,77 +1,77 @@
|
||||
#ifndef STB_IMAGE_IMPLEMENTATION
|
||||
#define STB_IMAGE_IMPLEMENTATION
|
||||
#endif
|
||||
#include "StgSharpNative.h"
|
||||
#include "gl.h"
|
||||
#include "../lib/glfw/glfw3.h"
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
extern GladGLContext *currentContext;
|
||||
extern char infolog[512];
|
||||
|
||||
SN_API int SN_DECL glCheckShaderStat(GladGLContext *context, uint64_t shaderHandle, int key,
|
||||
char **logRef)
|
||||
{
|
||||
int stat = 0;
|
||||
context->GetShaderiv(shaderHandle, key, &stat);
|
||||
if (!stat) {
|
||||
context->GetShaderInfoLog(shaderHandle, 512, NULL, infolog);
|
||||
}
|
||||
*logRef = infolog;
|
||||
return stat;
|
||||
}
|
||||
|
||||
SN_API void SN_DECL initGL(int majorVersion, int minorVersion)
|
||||
{
|
||||
glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, majorVersion);
|
||||
glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, minorVersion);
|
||||
glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
|
||||
glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);
|
||||
}
|
||||
|
||||
SN_API GLFWglproc SN_DECL loadGlfuncDefault(char const *procName)
|
||||
{
|
||||
void *ret = glfwGetProcAddress(procName);
|
||||
// printf("%llu\n", (uint64_t)ret);
|
||||
if (ret == NULL) {
|
||||
strcpy_s(infolog, 30, "Failed to load OpenGL api:");
|
||||
for (size_t i = 0; i < sizeof(procName); i++) {
|
||||
infolog[30 + i] = procName[i];
|
||||
}
|
||||
//printf("%s\n", infolog);
|
||||
}
|
||||
return (GLFWglproc)ret;
|
||||
}
|
||||
|
||||
SN_API unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
|
||||
{
|
||||
context->LinkProgram(shaderProgram);
|
||||
|
||||
int success;
|
||||
context->GetProgramiv(shaderProgram, GL_LINK_STATUS, &success);
|
||||
if (!success) {
|
||||
context->GetProgramInfoLog(shaderProgram, 512, NULL, infolog);
|
||||
return 0;
|
||||
} else {
|
||||
return shaderProgram;
|
||||
}
|
||||
}
|
||||
|
||||
SN_API void SN_DECL loadImageData(char const *location, Image *out, imageLoader loader)
|
||||
{
|
||||
if (loader == NULL) {
|
||||
loader = stbi_load;
|
||||
}
|
||||
out->pixelPtr = loader(location, &(out->width), &(out->height), &(out->channel), 0);
|
||||
}
|
||||
SN_API void SN_DECL unloadImageData(Image *out)
|
||||
{
|
||||
free(out->pixelPtr);
|
||||
}
|
||||
|
||||
SN_API char *SN_DECL readLog()
|
||||
{
|
||||
infolog[511] = '\0';
|
||||
return infolog;
|
||||
}
|
||||
#ifndef STB_IMAGE_IMPLEMENTATION
|
||||
#define STB_IMAGE_IMPLEMENTATION
|
||||
#endif
|
||||
#include "StgSharpNative.h"
|
||||
#include "gl.h"
|
||||
#include "../lib/glfw/glfw3.h"
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
extern GladGLContext *currentContext;
|
||||
extern char infolog[512];
|
||||
|
||||
SN_API int SN_DECL glCheckShaderStat(GladGLContext *context, uint64_t shaderHandle, int key,
|
||||
char **logRef)
|
||||
{
|
||||
int stat = 0;
|
||||
context->GetShaderiv(shaderHandle, key, &stat);
|
||||
if (!stat) {
|
||||
context->GetShaderInfoLog(shaderHandle, 512, NULL, infolog);
|
||||
}
|
||||
*logRef = infolog;
|
||||
return stat;
|
||||
}
|
||||
|
||||
SN_API void SN_DECL initGL(int majorVersion, int minorVersion)
|
||||
{
|
||||
glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, majorVersion);
|
||||
glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, minorVersion);
|
||||
glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
|
||||
glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);
|
||||
}
|
||||
|
||||
SN_API GLFWglproc SN_DECL loadGlfuncDefault(char const *procName)
|
||||
{
|
||||
void *ret = glfwGetProcAddress(procName);
|
||||
// printf("%llu\n", (uint64_t)ret);
|
||||
if (ret == NULL) {
|
||||
strcpy_s(infolog, 30, "Failed to load OpenGL api:");
|
||||
for (size_t i = 0; i < sizeof(procName); i++) {
|
||||
infolog[30 + i] = procName[i];
|
||||
}
|
||||
//printf("%s\n", infolog);
|
||||
}
|
||||
return (GLFWglproc)ret;
|
||||
}
|
||||
|
||||
SN_API unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
|
||||
{
|
||||
context->LinkProgram(shaderProgram);
|
||||
|
||||
int success;
|
||||
context->GetProgramiv(shaderProgram, GL_LINK_STATUS, &success);
|
||||
if (!success) {
|
||||
context->GetProgramInfoLog(shaderProgram, 512, NULL, infolog);
|
||||
return 0;
|
||||
} else {
|
||||
return shaderProgram;
|
||||
}
|
||||
}
|
||||
|
||||
SN_API void SN_DECL loadImageData(char const *location, Image *out, imageLoader loader)
|
||||
{
|
||||
if (loader == NULL) {
|
||||
loader = stbi_load;
|
||||
}
|
||||
out->pixelPtr = loader(location, &(out->width), &(out->height), &(out->channel), 0);
|
||||
}
|
||||
SN_API void SN_DECL unloadImageData(Image *out)
|
||||
{
|
||||
free(out->pixelPtr);
|
||||
}
|
||||
|
||||
SN_API char *SN_DECL readLog()
|
||||
{
|
||||
infolog[511] = '\0';
|
||||
return infolog;
|
||||
}
|
||||
|
||||
@@ -1,54 +1,54 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
|
||||
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length)
|
||||
{
|
||||
int index = 0, result = 0, size = length, end = size - 8;
|
||||
__m128i s = _mm_set1_epi32(target);
|
||||
__m128i vec, com;
|
||||
|
||||
while (index < end) {
|
||||
vec = _mm_loadu_epi32(str + index);
|
||||
com = _mm_cmpeq_epi16(vec, s);
|
||||
result = _mm_movemask_epi8(com);
|
||||
if (result) {
|
||||
for (size_t i = 0; i < 7; i++) {
|
||||
if (((result & 0xF) == 0xF)) {
|
||||
return index + i;
|
||||
}
|
||||
if (result == 0) {
|
||||
break;
|
||||
}
|
||||
result >>= 2;
|
||||
}
|
||||
}
|
||||
index += 1;
|
||||
vec = _mm_loadu_epi32(str + index);
|
||||
com = _mm_cmpeq_epi16(vec, s);
|
||||
result = _mm_movemask_epi8(com);
|
||||
if (result) {
|
||||
for (size_t i = 0; i < 7; i++) {
|
||||
if (((result & 0xF) == 0xF)) {
|
||||
return index + i;
|
||||
}
|
||||
if (result == 0) {
|
||||
break;
|
||||
}
|
||||
result >>= 2;
|
||||
}
|
||||
}
|
||||
index += 7;
|
||||
}
|
||||
for (; (index < size - 1) && (!result); index += 2) {
|
||||
result = *(int *)(str + index) == target;
|
||||
}
|
||||
if (result) {
|
||||
return index;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
|
||||
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length)
|
||||
{
|
||||
int index = 0, result = 0, size = length, end = size - 8;
|
||||
__m128i s = _mm_set1_epi32(target);
|
||||
__m128i vec, com;
|
||||
|
||||
while (index < end) {
|
||||
vec = _mm_loadu_epi32(str + index);
|
||||
com = _mm_cmpeq_epi16(vec, s);
|
||||
result = _mm_movemask_epi8(com);
|
||||
if (result) {
|
||||
for (size_t i = 0; i < 7; i++) {
|
||||
if (((result & 0xF) == 0xF)) {
|
||||
return index + i;
|
||||
}
|
||||
if (result == 0) {
|
||||
break;
|
||||
}
|
||||
result >>= 2;
|
||||
}
|
||||
}
|
||||
index += 1;
|
||||
vec = _mm_loadu_epi32(str + index);
|
||||
com = _mm_cmpeq_epi16(vec, s);
|
||||
result = _mm_movemask_epi8(com);
|
||||
if (result) {
|
||||
for (size_t i = 0; i < 7; i++) {
|
||||
if (((result & 0xF) == 0xF)) {
|
||||
return index + i;
|
||||
}
|
||||
if (result == 0) {
|
||||
break;
|
||||
}
|
||||
result >>= 2;
|
||||
}
|
||||
}
|
||||
index += 7;
|
||||
}
|
||||
for (; (index < size - 1) && (!result); index += 2) {
|
||||
result = *(int *)(str + index) == target;
|
||||
}
|
||||
if (result) {
|
||||
return index;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,32 +1,32 @@
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_internal.h"
|
||||
|
||||
INTERNAL void SN_DECL f32_normalize(VEC(float) *restrict source, VEC(float) *restrict target)
|
||||
{
|
||||
register __m128 s = _mm_load_ps(&source->f32_x);
|
||||
register __m128 a_length_sq = _mm_mul_ps(s, s);
|
||||
a_length_sq = _mm_add_ps(a_length_sq,
|
||||
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
|
||||
a_length_sq = _mm_add_ss(a_length_sq,
|
||||
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
|
||||
__m128 a_inv_length = _mm_rsqrt_ps(a_length_sq);
|
||||
s = _mm_mul_ps(s, a_inv_length);
|
||||
|
||||
__m128 dot_product = _mm_mul_ps(target->f32_x[0], s);
|
||||
dot_product = _mm_add_ps(dot_product,
|
||||
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(2, 3, 0, 1)));
|
||||
dot_product = _mm_add_ss(dot_product,
|
||||
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
|
||||
__m128 b_orthogonal = _mm_sub_ps(target->f32_x[0], _mm_mul_ps(s, dot_product));
|
||||
|
||||
__m128 b_length_sq = _mm_mul_ps(b_orthogonal, b_orthogonal);
|
||||
b_length_sq = _mm_add_ps(b_length_sq,
|
||||
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
|
||||
b_length_sq = _mm_add_ss(b_length_sq,
|
||||
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
|
||||
__m128 b_inv_length = _mm_rsqrt_ps(b_length_sq);
|
||||
target->f32_x[0] = _mm_mul_ps(b_orthogonal, b_inv_length);
|
||||
}
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_internal.h"
|
||||
|
||||
INTERNAL void SN_DECL f32_normalize(VEC(float) *restrict source, VEC(float) *restrict target)
|
||||
{
|
||||
register __m128 s = _mm_load_ps(&source->f32_x);
|
||||
register __m128 a_length_sq = _mm_mul_ps(s, s);
|
||||
a_length_sq = _mm_add_ps(a_length_sq,
|
||||
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
|
||||
a_length_sq = _mm_add_ss(a_length_sq,
|
||||
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
|
||||
__m128 a_inv_length = _mm_rsqrt_ps(a_length_sq);
|
||||
s = _mm_mul_ps(s, a_inv_length);
|
||||
|
||||
__m128 dot_product = _mm_mul_ps(target->f32_x[0], s);
|
||||
dot_product = _mm_add_ps(dot_product,
|
||||
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(2, 3, 0, 1)));
|
||||
dot_product = _mm_add_ss(dot_product,
|
||||
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
|
||||
__m128 b_orthogonal = _mm_sub_ps(target->f32_x[0], _mm_mul_ps(s, dot_product));
|
||||
|
||||
__m128 b_length_sq = _mm_mul_ps(b_orthogonal, b_orthogonal);
|
||||
b_length_sq = _mm_add_ps(b_length_sq,
|
||||
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
|
||||
b_length_sq = _mm_add_ss(b_length_sq,
|
||||
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
|
||||
__m128 b_inv_length = _mm_rsqrt_ps(b_length_sq);
|
||||
target->f32_x[0] = _mm_mul_ps(b_orthogonal, b_inv_length);
|
||||
}
|
||||
|
||||
@@ -1,67 +1,67 @@
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_internal.h"
|
||||
|
||||
static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 };
|
||||
|
||||
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
__m128 const *vector, __m128 *ans)
|
||||
{
|
||||
*ans = _mm_dp_ps(transpose->f32_x[0], *vector, 0B11110001);
|
||||
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], *vector, 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 1));
|
||||
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], *vector, 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 3));
|
||||
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], *vector, 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
}
|
||||
|
||||
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
|
||||
__m256 tmp = _mm256_dp_ps(transpose->f32_y[0], _double, 0B11110001);
|
||||
tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->f32_y[0], _double, 0B11110010),
|
||||
0B00100010);
|
||||
*ans = _mm256_castps256_ps128(
|
||||
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
|
||||
}
|
||||
|
||||
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
ans[0] = _mm_dp_ps(transpose->f32_x[0], vector[0], 0B11110001);
|
||||
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], vector[0], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 2));
|
||||
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], vector[0], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 3));
|
||||
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], vector[0], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
|
||||
ans[1] = _mm_dp_ps(transpose->f32_x[0], vector[1], 0B11110001);
|
||||
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], vector[1], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 2));
|
||||
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], vector[1], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 3));
|
||||
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], vector[1], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
}
|
||||
|
||||
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
__m256 temp = transpose->f32_y[0];
|
||||
__m256 vector_stream = _mm256_set_m128(vector[1], vector[0]);
|
||||
|
||||
__m256 result = _mm256_dp_ps(temp, vector_stream, 0B11110001);
|
||||
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
|
||||
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010);
|
||||
|
||||
temp = transpose->f32_y[1];
|
||||
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100);
|
||||
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
|
||||
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000);
|
||||
|
||||
ans[0] = _mm256_castps256_ps128(result);
|
||||
ans[1] = _mm256_extractf128_ps(result, 1);
|
||||
ans[1] = _mm_shuffle_ps(ans[1], ans[1], _MM_SHUFFLE(2, 3, 0, 1));
|
||||
}
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_internal.h"
|
||||
|
||||
static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 };
|
||||
|
||||
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
__m128 const *vector, __m128 *ans)
|
||||
{
|
||||
*ans = _mm_dp_ps(transpose->f32_x[0], *vector, 0B11110001);
|
||||
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], *vector, 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 1));
|
||||
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], *vector, 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 3));
|
||||
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], *vector, 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
}
|
||||
|
||||
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
|
||||
__m256 tmp = _mm256_dp_ps(transpose->f32_y[0], _double, 0B11110001);
|
||||
tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->f32_y[0], _double, 0B11110010),
|
||||
0B00100010);
|
||||
*ans = _mm256_castps256_ps128(
|
||||
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
|
||||
}
|
||||
|
||||
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
ans[0] = _mm_dp_ps(transpose->f32_x[0], vector[0], 0B11110001);
|
||||
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], vector[0], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 2));
|
||||
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], vector[0], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 3));
|
||||
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], vector[0], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
|
||||
ans[1] = _mm_dp_ps(transpose->f32_x[0], vector[1], 0B11110001);
|
||||
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], vector[1], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 2));
|
||||
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], vector[1], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 3));
|
||||
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], vector[1], 0B11110001),
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
}
|
||||
|
||||
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
__m256 temp = transpose->f32_y[0];
|
||||
__m256 vector_stream = _mm256_set_m128(vector[1], vector[0]);
|
||||
|
||||
__m256 result = _mm256_dp_ps(temp, vector_stream, 0B11110001);
|
||||
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
|
||||
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010);
|
||||
|
||||
temp = transpose->f32_y[1];
|
||||
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100);
|
||||
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
|
||||
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000);
|
||||
|
||||
ans[0] = _mm256_castps256_ps128(result);
|
||||
ans[1] = _mm256_extractf128_ps(result, 1);
|
||||
ans[1] = _mm_shuffle_ps(ans[1], ans[1], _MM_SHUFFLE(2, 3, 0, 1));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user