150 lines
6.3 KiB
C
150 lines
6.3 KiB
C
#ifndef SSC_INTRIN
|
|
#define SSC_INTRIN
|
|
|
|
#define SSC_INTRIN_HEAD
|
|
|
|
#include "StgSharpNative.h"
|
|
#include "sn_matkernel.h"
|
|
#include <immintrin.h>
|
|
#include <smmintrin.h>
|
|
|
|
#ifdef _MSC_VER
|
|
#include <intrin.h>
|
|
#include <isa_availability.h>
|
|
#endif
|
|
|
|
#define LASTTWO(x) (x & 3)
|
|
#define LAST(x) (x & 3)
|
|
#define get_align_begin(x) ((x + 15) & (~15))
|
|
|
|
#define SSE_INSERT(source, target, reset3, reset2, reset1, reset0) \
|
|
(LASTTWO(source - 1) << 6) | (LASTTWO(target - 1) << 4) | (LAST(reset3) << 3) | \
|
|
(LAST(reset2) << 2) | (LAST(reset1) << 1) | LAST(reset0)
|
|
#define SSE_INSERT_SIMPLE(source, target) SSE_INSERT(source, target, 0, 0, 0, 0)
|
|
#define M256_PERMUTE(s3, s2, s1, s0) (LASTTWO(s3) << 6) | (LASTTWO(s2) << 4) | (LASTTWO(s1) << 2) | (LASTTWO(s0)
|
|
#define BIT_CVRT(pack, index, T) *(T *)(pack->data + index)
|
|
#define zero_vec _mm_setzero_ps()
|
|
#define zero_vec_256 _mm256_setzero_ps()
|
|
|
|
typedef union uint64_u {
|
|
uint64_t u64;
|
|
uint32_t u32[2];
|
|
} uint64_u;
|
|
|
|
typedef struct mat_kernel mat_kernel;
|
|
|
|
typedef union scalar_pack {
|
|
uint64_t data[8];
|
|
} scalar_pack;
|
|
|
|
typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target);
|
|
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
|
|
typedef void(SN_DECL *MAT_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
|
|
typedef void(SN_DECL *MAT_PROC_LA)(mat_kernel *left, mat_kernel *ans);
|
|
typedef void(SN_DECL *MAT_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
|
|
typedef void(SN_DECL *MAT_PROC_LSA)(mat_kernel *matrix, scalar_pack *scalar, mat_kernel *ans);
|
|
typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
|
|
|
|
#pragma region matix function
|
|
|
|
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
|
|
|
|
INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
|
|
MAT_KERNEL(float) *restrict ans);
|
|
INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
|
|
MAT_KERNEL(float) *restrict ans);
|
|
INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
|
|
MAT_KERNEL(float) *restrict ans);
|
|
INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
|
|
MAT_KERNEL(float) *restrict ans);
|
|
INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
|
|
MAT_KERNEL(float) *restrict ans);
|
|
INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
|
|
MAT_KERNEL(float) *restrict ans);
|
|
INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source,
|
|
MAT_KERNEL(float) *restrict target);
|
|
INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source,
|
|
MAT_KERNEL(float) *restrict target);
|
|
INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source,
|
|
MAT_KERNEL(float) *restrict target);
|
|
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix,
|
|
MAT_KERNEL(float) *restrict ans,
|
|
scalar_pack const *scalar);
|
|
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix,
|
|
MAT_KERNEL(float) *restrict ans,
|
|
scalar_pack const *scalar);
|
|
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix,
|
|
MAT_KERNEL(float) *restrict ans,
|
|
scalar_pack const *scalar);
|
|
INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
|
|
INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
|
|
INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
|
|
#pragma endregion
|
|
|
|
#pragma region fma
|
|
|
|
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
|
|
MAT_KERNEL(float) const *restrict right,
|
|
MAT_KERNEL(float) *restrict ans);
|
|
INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left,
|
|
MAT_KERNEL(float) const *restrict right,
|
|
MAT_KERNEL(float) *restrict ans);
|
|
INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left,
|
|
MAT_KERNEL(float) const *restrict right,
|
|
MAT_KERNEL(float) *restrict ans);
|
|
|
|
#pragma endregion
|
|
|
|
#pragma region scaler_simd
|
|
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
|
|
#pragma endregion
|
|
|
|
#pragma region dot_product
|
|
|
|
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
|
__m128 const *vector, __m128 *ans);
|
|
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
|
__m128 *ans);
|
|
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
|
__m128 *ans);
|
|
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
|
__m128 *ans);
|
|
|
|
#pragma endregion
|
|
|
|
#pragma region string
|
|
|
|
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
|
|
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length);
|
|
|
|
#pragma endregion
|
|
|
|
typedef struct sn_intrinsic {
|
|
int (*city_hash_simplify)(byte *str, int length);
|
|
FACTORIALROC factorial_simd;
|
|
int (*index_pair)(short const *str, uint32_t target, int length);
|
|
MAT_PROC_LRA f32_add;
|
|
MAT_PROC_AS f32_fill;
|
|
MAT_PROC_LRA f32_fma;
|
|
MAT_PROC_LRA f32_scalar_mul;
|
|
MAT_PROC_LRA f32_sub;
|
|
MAT_PROC_LA f32_transpose;
|
|
VECTORNORMALIZEPROC f32_normalize_3;
|
|
} sn_intrinsic;
|
|
|
|
typedef enum most_advanced_instruction {
|
|
#ifdef _MSC_VER
|
|
SSE = __IA_SUPPORT_SSE42,
|
|
AVX256 = __IA_SUPPORT_VECTOR256,
|
|
AVX512 = __IA_SUPPORT_VECTOR512,
|
|
NEON
|
|
#else
|
|
SSE,
|
|
AVX256,
|
|
AVX512,
|
|
NEON
|
|
#endif
|
|
} most_advanced_instruction;
|
|
|
|
#endif
|