139 lines
6.3 KiB
C
139 lines
6.3 KiB
C
#include "sn_target.h"
|
|
|
|
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
|
|
|
#include "StgSharpNative.h"
|
|
#include "sn_intrinsic.h"
|
|
#include "sn_intrinsic.std.h"
|
|
#include "sn_intrinsic.context.x86.h"
|
|
|
|
#if defined(_MSC_VER) && !defined(__clang__)
|
|
#include <intrin.h>
|
|
#include <isa_availability.h>
|
|
#else
|
|
#include <cpuid.h>
|
|
#endif // _MSC_VER
|
|
|
|
PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
|
|
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
|
|
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
|
|
|
|
SN_API void load_intrinsic_function(void *intrinsic_context, uint64_t id)
|
|
{
|
|
sn_null_assert(intrinsic_context);
|
|
|
|
SIMDID simdid;
|
|
if (id) {
|
|
simdid.mask = id;
|
|
} else {
|
|
simdid = sn_get_simd_level_global();
|
|
}
|
|
|
|
sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
|
|
|
|
switch (simdid.make_byte[1]) {
|
|
case SIMDID_MAIN_LVL_SSE:
|
|
load_intrinsic_sse(context, simdid);
|
|
break;
|
|
case SIMDID_MAIN_LVL_AVX2:
|
|
load_intrinsic_avx(context, simdid);
|
|
break;
|
|
case SIMDID_MAIN_LVL_AVX512:
|
|
case SIMDID_MAIN_LVL_AMX:
|
|
case SIMDID_MAIN_LVL_AVX10:
|
|
load_intrinsic_512(context, simdid);
|
|
break;
|
|
}
|
|
}
|
|
|
|
void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
|
|
{
|
|
context->city_hash_simplify = city_hash_simplify_sse;
|
|
context->factorial_simd = factorial_simd_sse;
|
|
context->index_pair = index_pair_sse;
|
|
|
|
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, sse, , add);
|
|
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, sse, , fill);
|
|
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, sse, , scalar_mul);
|
|
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, sse, , sub);
|
|
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, sse, , transpose);
|
|
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, sse, , fma);
|
|
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, sse, , tiled_fma);
|
|
context->mat[F32].pivot = NULL;
|
|
|
|
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, sse, , add);
|
|
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, sse, , fill);
|
|
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, sse, , scalar_mul);
|
|
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, sse, , sub);
|
|
context->mat[F64].buffer_transpose = NULL;
|
|
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, sse, , fma);
|
|
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, sse, , tiled_fma);
|
|
context->mat[F64].pivot = NULL;
|
|
|
|
context->vec_f32_normalize_3 = f32_normalize;
|
|
}
|
|
|
|
void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
|
|
{
|
|
char avx_seg = id.make_byte[5];
|
|
char is_fma = (avx_seg & SIMDID_AVX_FMA) != 0;
|
|
char is_f16c = (avx_seg & SIMDID_AVX_F16C) != 0;
|
|
|
|
context->city_hash_simplify = city_hash_simplify_sse;
|
|
context->factorial_simd = factorial_simd_sse;
|
|
context->index_pair = index_pair_sse;
|
|
|
|
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, avx, , add);
|
|
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, avx, , fill);
|
|
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, avx, , scalar_mul);
|
|
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, avx, , sub);
|
|
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, avx, , transpose);
|
|
context->mat[F32].ker_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , fma) :
|
|
STD_MK_PROC_NAME(float, avx, _fma, fma);
|
|
context->mat[F32].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , tiled_fma) :
|
|
STD_MK_PROC_NAME(float, avx, _fma, tiled_fma);
|
|
context->mat[F32].pivot = NULL;
|
|
|
|
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, avx, , add);
|
|
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, avx, , fill);
|
|
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, avx, , scalar_mul);
|
|
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, avx, , sub);
|
|
context->mat[F64].buffer_transpose = NULL;
|
|
context->mat[F64].ker_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , fma) :
|
|
STD_MK_PROC_NAME(double, avx, _fma, fma);
|
|
context->mat[F64].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , tiled_fma) :
|
|
STD_MK_PROC_NAME(double, avx, _fma, tiled_fma);
|
|
context->mat[F64].pivot = NULL;
|
|
|
|
context->vec_f32_normalize_3 = f32_normalize;
|
|
}
|
|
|
|
void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
|
|
{
|
|
context->city_hash_simplify = city_hash_simplify_sse;
|
|
context->factorial_simd = factorial_simd_sse;
|
|
context->index_pair = index_pair_sse;
|
|
|
|
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, 512, , add);
|
|
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, 512, , fill);
|
|
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, 512, , scalar_mul);
|
|
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, 512, , sub);
|
|
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, 512, , transpose);
|
|
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, 512, , fma);
|
|
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, 512, , tiled_fma);
|
|
context->mat[F32].pivot = NULL;
|
|
|
|
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, 512, , add);
|
|
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, 512, , fill);
|
|
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, 512, , scalar_mul);
|
|
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, 512, , sub);
|
|
context->mat[F64].buffer_transpose = NULL;
|
|
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, 512, , fma);
|
|
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, 512, , tiled_fma);
|
|
context->mat[F64].pivot = NULL;
|
|
|
|
context->vec_f32_normalize_3 = f32_normalize;
|
|
}
|
|
|
|
#endif
|