368 lines
11 KiB
C
368 lines
11 KiB
C
#define _GNU_SOURCE
|
|
|
|
#include "nfn_target.h"
|
|
|
|
#if NFN_IS_OS(NFN_OS_LINUX)
|
|
#include <errno.h>
|
|
#include <inttypes.h>
|
|
#include <limits.h>
|
|
#include <linux/mempolicy.h>
|
|
#include <pthread.h>
|
|
#include <sched.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/mman.h>
|
|
#include <sys/syscall.h>
|
|
#include <unistd.h>
|
|
|
|
typedef int (*nfn_linux_id_visitor)(int value, void *context);
|
|
|
|
typedef struct nfn_linux_counter_context {
|
|
int count;
|
|
} nfn_linux_counter_context;
|
|
|
|
typedef struct nfn_linux_max_context {
|
|
int max_value;
|
|
} nfn_linux_max_context;
|
|
|
|
typedef struct nfn_linux_fill_cpu_context {
|
|
uint32_t *processor_group_ids;
|
|
uint32_t *socket_ids;
|
|
uint32_t *cpu_indices;
|
|
uint32_t *numa_node_ids;
|
|
int numa_id;
|
|
int index;
|
|
} nfn_linux_fill_cpu_context;
|
|
|
|
typedef struct nfn_linux_fill_node_count_context {
|
|
int *count_of_each_numa;
|
|
} nfn_linux_fill_node_count_context;
|
|
|
|
static int nfn_linux_read_first_line(char const *path, char *buffer, size_t buffer_size)
|
|
{
|
|
FILE *file = fopen(path, "r");
|
|
if (file == NULL)
|
|
return 0;
|
|
|
|
char *line = fgets(buffer, (int)buffer_size, file);
|
|
fclose(file);
|
|
return line != NULL;
|
|
}
|
|
|
|
static int nfn_linux_parse_nonnegative_int(char const **cursor, int *value)
|
|
{
|
|
char *end = NULL;
|
|
uintmax_t parsed_value = 0;
|
|
|
|
if (**cursor == '-')
|
|
return 0;
|
|
|
|
errno = 0;
|
|
parsed_value = strtoumax(*cursor, &end, 10);
|
|
if (end == *cursor || errno == ERANGE || parsed_value > (uintmax_t)INT_MAX)
|
|
return 0;
|
|
|
|
*cursor = end;
|
|
*value = (int)parsed_value;
|
|
return 1;
|
|
}
|
|
|
|
static int nfn_linux_visit_id_list(char const *text, nfn_linux_id_visitor visitor, void *context)
|
|
{
|
|
char const *cursor = text;
|
|
|
|
while (*cursor != '\0') {
|
|
while (*cursor == ' ' || *cursor == '\t' || *cursor == '\n' || *cursor == ',')
|
|
cursor++;
|
|
|
|
if (*cursor == '\0')
|
|
break;
|
|
|
|
int begin = 0;
|
|
if (!nfn_linux_parse_nonnegative_int(&cursor, &begin))
|
|
return 0;
|
|
|
|
int finish = begin;
|
|
|
|
if (*cursor == '-') {
|
|
cursor++;
|
|
if (!nfn_linux_parse_nonnegative_int(&cursor, &finish))
|
|
return 0;
|
|
}
|
|
|
|
if (finish < begin)
|
|
return 0;
|
|
|
|
for (int value = begin;; value++) {
|
|
if (!visitor(value, context))
|
|
return 0;
|
|
|
|
if (value == finish)
|
|
break;
|
|
}
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
static int nfn_linux_count_id(int value, void *context)
|
|
{
|
|
(void)value;
|
|
((nfn_linux_counter_context *)context)->count++;
|
|
return 1;
|
|
}
|
|
|
|
static int nfn_linux_track_max_id(int value, void *context)
|
|
{
|
|
nfn_linux_max_context *max_context = (nfn_linux_max_context *)context;
|
|
if (value > max_context->max_value)
|
|
max_context->max_value = value;
|
|
return 1;
|
|
}
|
|
|
|
static int nfn_linux_read_socket_id(int cpu_id, uint32_t *socket_id)
|
|
{
|
|
char path[160];
|
|
char buffer[64];
|
|
char *end = NULL;
|
|
long parsed_value;
|
|
snprintf(path, sizeof(path),
|
|
"/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu_id);
|
|
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
|
|
return 0;
|
|
|
|
errno = 0;
|
|
parsed_value = strtol(buffer, &end, 10);
|
|
if (end == buffer || errno == ERANGE || parsed_value < 0 || (uintmax_t)parsed_value > UINT32_MAX)
|
|
return 0;
|
|
|
|
*socket_id = (uint32_t)parsed_value;
|
|
return 1;
|
|
}
|
|
|
|
static int nfn_linux_fill_cpu_id(int value, void *context)
|
|
{
|
|
nfn_linux_fill_cpu_context *fill_context = (nfn_linux_fill_cpu_context *)context;
|
|
uint32_t socket_id = 0;
|
|
if (!nfn_linux_read_socket_id(value, &socket_id))
|
|
return 0;
|
|
|
|
fill_context->processor_group_ids[fill_context->index] = 0u;
|
|
fill_context->socket_ids[fill_context->index] = socket_id;
|
|
fill_context->cpu_indices[fill_context->index] = (uint32_t)value;
|
|
fill_context->numa_node_ids[fill_context->index] = (uint32_t)fill_context->numa_id;
|
|
|
|
fill_context->index++;
|
|
return 1;
|
|
}
|
|
|
|
static int nfn_linux_count_cpu_list(char const *path)
|
|
{
|
|
char buffer[256];
|
|
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
|
|
return 0;
|
|
|
|
nfn_linux_counter_context counter = { 0 };
|
|
if (!nfn_linux_visit_id_list(buffer, nfn_linux_count_id, &counter))
|
|
return 0;
|
|
|
|
return counter.count;
|
|
}
|
|
|
|
static int nfn_linux_fill_node_cpu_count(int value, void *context)
|
|
{
|
|
char path[128];
|
|
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", value);
|
|
|
|
((nfn_linux_fill_node_count_context *)context)->count_of_each_numa[value] =
|
|
nfn_linux_count_cpu_list(path);
|
|
return 1;
|
|
}
|
|
|
|
static int nfn_linux_node_exists(int numa_id)
|
|
{
|
|
char path[128];
|
|
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d", numa_id);
|
|
return access(path, F_OK) == 0;
|
|
}
|
|
|
|
static size_t nfn_linux_round_up_to_page_size(size_t size)
|
|
{
|
|
int64_t page_size_value = (int64_t)sysconf(_SC_PAGESIZE);
|
|
size_t effective_page_size = (page_size_value > 0) ? (size_t)page_size_value : 4096u;
|
|
|
|
size_t rounded = size + effective_page_size - 1;
|
|
if (rounded < size)
|
|
return 0;
|
|
|
|
return rounded - (rounded % effective_page_size);
|
|
}
|
|
|
|
static int nfn_linux_bind_memory_to_node(void *address, size_t size, int numa_id, uint32_t flags)
|
|
{
|
|
#if defined(SYS_mbind)
|
|
size_t bits_per_word = sizeof(unsigned long) * 8u;
|
|
size_t word_count = ((size_t)numa_id / bits_per_word) + 1u;
|
|
unsigned long *node_mask = (unsigned long *)calloc(word_count, sizeof(unsigned long));
|
|
if (node_mask == NULL)
|
|
return 0;
|
|
|
|
node_mask[(size_t)numa_id / bits_per_word] = 1ul << ((size_t)numa_id % bits_per_word);
|
|
|
|
int syscall_result = (int)syscall(SYS_mbind, address, size, MPOL_BIND | MPOL_F_STATIC_NODES,
|
|
node_mask, (unsigned long)numa_id + 1ul, flags);
|
|
free(node_mask);
|
|
return syscall_result == 0;
|
|
#else
|
|
(void)address;
|
|
(void)size;
|
|
(void)numa_id;
|
|
(void)flags;
|
|
return 0;
|
|
#endif
|
|
}
|
|
|
|
static int nfn_linux_populate_writable_mapping(void *address, size_t size)
|
|
{
|
|
#if defined(MADV_POPULATE_WRITE)
|
|
if (madvise(address, size, MADV_POPULATE_WRITE) == 0)
|
|
return 1;
|
|
|
|
return errno == EINVAL;
|
|
#else
|
|
(void)address;
|
|
(void)size;
|
|
return 1;
|
|
#endif
|
|
}
|
|
|
|
/* Linux has no Windows-style processor groups; processor_group_id is ignored. */
|
|
NFN_API void NFN_DECL nfn_set_thread_affinity(int cpu_index, int processor_group_id)
|
|
{
|
|
(void)processor_group_id;
|
|
|
|
int cpu_count = (int)sysconf(_SC_NPROCESSORS_CONF);
|
|
if (cpu_index >= cpu_count)
|
|
cpu_count = cpu_index + 1;
|
|
if (cpu_count <= 0)
|
|
return;
|
|
|
|
cpu_set_t *set = CPU_ALLOC(cpu_count);
|
|
size_t size = CPU_ALLOC_SIZE(cpu_count);
|
|
if (set == NULL)
|
|
return;
|
|
|
|
CPU_ZERO_S(size, set);
|
|
|
|
if (cpu_index < 0) {
|
|
for (int i = 0; i < cpu_count; i++)
|
|
CPU_SET_S((int)i, size, set);
|
|
} else {
|
|
CPU_SET_S(cpu_index, size, set);
|
|
}
|
|
|
|
pthread_setaffinity_np(pthread_self(), size, set);
|
|
CPU_FREE(set);
|
|
}
|
|
|
|
NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
|
|
{
|
|
char buffer[256];
|
|
if (!nfn_linux_read_first_line("/sys/devices/system/node/online", buffer, sizeof(buffer)))
|
|
return 0;
|
|
|
|
nfn_linux_max_context max_context = { -1 };
|
|
if (!nfn_linux_visit_id_list(buffer, nfn_linux_track_max_id, &max_context) ||
|
|
max_context.max_value < 0)
|
|
return 0;
|
|
|
|
if (out_numa_count != NULL)
|
|
*out_numa_count = max_context.max_value + 1;
|
|
|
|
if (count_of_each_numa != NULL) {
|
|
for (int index = 0; index <= max_context.max_value; index++)
|
|
count_of_each_numa[index] = 0;
|
|
|
|
nfn_linux_fill_node_count_context fill_context = { count_of_each_numa };
|
|
if (!nfn_linux_visit_id_list(buffer, nfn_linux_fill_node_cpu_count, &fill_context))
|
|
return 0;
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
NFN_API int NFN_DECL nfn_get_numa_each(int numa_id,
|
|
uint32_t *processor_group_ids,
|
|
uint32_t *socket_ids,
|
|
uint32_t *cpu_indices,
|
|
uint32_t *numa_node_ids)
|
|
{
|
|
if (numa_id < 0 || processor_group_ids == NULL || socket_ids == NULL ||
|
|
cpu_indices == NULL || numa_node_ids == NULL)
|
|
return 0;
|
|
|
|
char path[128];
|
|
char buffer[256];
|
|
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa_id);
|
|
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
|
|
return 0;
|
|
|
|
nfn_linux_fill_cpu_context fill_context = {
|
|
processor_group_ids,
|
|
socket_ids,
|
|
cpu_indices,
|
|
numa_node_ids,
|
|
numa_id,
|
|
0
|
|
};
|
|
return nfn_linux_visit_id_list(buffer, nfn_linux_fill_cpu_id, &fill_context);
|
|
}
|
|
|
|
NFN_API void *NFN_DECL nfn_numa_alloc(size_t size, int numa_id)
|
|
{
|
|
size_t allocation_size = nfn_linux_round_up_to_page_size(size);
|
|
if (allocation_size == 0)
|
|
return NULL;
|
|
|
|
int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS;
|
|
#if defined(MAP_POPULATE)
|
|
if (numa_id >= 0)
|
|
mmap_flags |= MAP_POPULATE;
|
|
#endif
|
|
|
|
void *memory = mmap(NULL, allocation_size, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
|
|
if (memory == MAP_FAILED)
|
|
return NULL;
|
|
|
|
if (numa_id < 0)
|
|
return memory;
|
|
|
|
uint32_t bind_flags = 0u;
|
|
#if defined(MAP_POPULATE)
|
|
bind_flags = MPOL_MF_MOVE | MPOL_MF_STRICT;
|
|
#endif
|
|
|
|
if (!nfn_linux_node_exists(numa_id) ||
|
|
!nfn_linux_bind_memory_to_node(memory, allocation_size, numa_id, bind_flags) ||
|
|
!nfn_linux_populate_writable_mapping(memory, allocation_size)) {
|
|
munmap(memory, allocation_size);
|
|
return NULL;
|
|
}
|
|
|
|
return memory;
|
|
}
|
|
|
|
NFN_API void NFN_DECL nfn_numa_free(void *ptr, size_t size, int numa_id)
|
|
{
|
|
(void)numa_id;
|
|
|
|
if (ptr == NULL || size == 0)
|
|
return;
|
|
|
|
size_t allocation_size = nfn_linux_round_up_to_page_size(size);
|
|
if (allocation_size != 0)
|
|
munmap(ptr, allocation_size);
|
|
}
|
|
#endif
|