43#if N_REACTOR_AVAILABLE
48#include <sys/eventfd.h>
49#include <sys/socket.h>
56#define N_REACTOR_DEFAULT_MAX_FDS 256
61#define N_REACTOR_BATCH_SIZE 64
77 pthread_mutex_t registered_lock;
81 atomic_llong events_processed;
82 atomic_llong writes_partial;
83 atomic_llong reads_partial;
84 atomic_llong fds_registered;
85 atomic_llong fds_unregistered;
86 atomic_llong wake_signals;
90 atomic_llong wake_walks;
91 atomic_llong wake_walk_visits;
92 atomic_llong wake_walk_drains;
99 pthread_mutex_t dirty_lock;
104static int register_internal_efd(
int epoll_fd,
int efd, uint64_t tag) {
109 struct epoll_event ev;
112 if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, efd, &ev) != 0) {
113 n_log(
LOG_ERR,
"n_reactor: epoll_ctl ADD eventfd failed: %s",
125#define N_REACTOR_TAG_STOP 1ULL
126#define N_REACTOR_TAG_WAKE 2ULL
127#define N_REACTOR_TAG_INTERNAL_MAX 16ULL
132static long long drain_eventfd(
int efd) {
136 ssize_t n = read(efd, &buf,
sizeof(buf));
137 if (n ==
sizeof(buf)) {
138 total += (
long long)buf;
141 if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
break;
151static int set_nonblocking(
int fd) {
152 int flags = fcntl(fd, F_GETFL, 0);
153 if (flags < 0)
return 0;
154 if ((flags & O_NONBLOCK) == O_NONBLOCK)
return 1;
155 if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0)
return 0;
160static void reactor_recv_state_reset(
NETWORK*
netw) {
170static void reactor_send_state_reset(
NETWORK*
netw) {
183 struct epoll_event ev;
186 return epoll_ctl(r->epoll_fd, EPOLL_CTL_MOD,
netw->
link.
sock, &ev) == 0;
199static int reactor_send_state_load_next(
NETWORK*
netw) {
215 N_STR* zipped = NULL;
216 uint32_t flag_bit = 0;
224 if (zipped && zipped->
written > 0 &&
230 pkt_state |= flag_bit;
236 if (msg->
written > UINT32_MAX) {
237 n_log(
LOG_ERR,
"n_reactor: payload too large (%zu > UINT32_MAX); dropping",
242 size_t total = 8 + msg->
written;
249 uint32_t state_be = htonl(pkt_state);
250 uint32_t length_be = htonl((uint32_t)msg->
written);
251 memcpy(buf, &state_be, 4);
252 memcpy(buf + 4, &length_be, 4);
273 int r = reactor_send_state_load_next(
netw);
274 if (r < 0)
return -1;
275 if (r == 0)
return 1;
278 if (remaining == 0) {
292 uint32_t attempt = (remaining > UINT32_MAX) ? UINT32_MAX : (uint32_t)remaining;
301 atomic_fetch_add(&reactor->writes_partial, 1);
310 atomic_fetch_add(&reactor->writes_partial, 1);
323static void reactor_recv_dispatch_frame(
NETWORK*
netw,
335 msg->
data = pkt_payload;
336 msg->
length = (size_t)pkt_length + 1;
337 msg->
written = (size_t)pkt_length;
342 msg->
data[pkt_length] =
'\0';
347 if (want_zlib || want_lz4) {
354 "n_reactor: failed to decompress payload "
355 "(%" PRIu32
" bytes, codec=%s); dropping",
356 pkt_length, want_lz4 ?
"lz4" :
"zlib");
365 n_log(
LOG_ERR,
"n_reactor: recv_buf list_push failed; dropping frame");
384static void reactor_sweep_exit_asked(
n_reactor* reactor) {
385 NETWORK* batch[N_REACTOR_BATCH_SIZE];
388 pthread_mutex_lock(&reactor->registered_lock);
389 LIST_NODE* node = reactor->registered->start;
390 while (node && count < N_REACTOR_BATCH_SIZE) {
401 pthread_mutex_unlock(&reactor->registered_lock);
403 for (
int i = 0; i < count; i++) {
409 (void)reactor_drain_writes(n, reactor);
460 const char* p = chunk;
461 size_t rem = (size_t)got;
468 size_t take = (rem < need) ? rem : need;
476 atomic_fetch_add(&reactor->reads_partial, 1);
503 n_log(
LOG_ERR,
"n_reactor: alloc(empty payload) failed");
506 reactor_recv_dispatch_frame(
netw,
513 n_log(
LOG_ERR,
"n_reactor: alloc(%u-byte payload) failed",
526 size_t take = (rem < need) ? rem : need;
535 atomic_fetch_add(&reactor->reads_partial, 1);
542 reactor_recv_dispatch_frame(
netw,
571 r->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
572 if (r->epoll_fd < 0) {
573 n_log(
LOG_ERR,
"n_reactor_new: epoll_create1 failed: %s",
582 r->stop_efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
583 if (r->stop_efd < 0) {
584 n_log(
LOG_ERR,
"n_reactor_new: eventfd(stop) failed: %s",
588 r->wake_efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
589 if (r->wake_efd < 0) {
590 n_log(
LOG_ERR,
"n_reactor_new: eventfd(wake) failed: %s",
595 if (!register_internal_efd(r->epoll_fd, r->stop_efd, N_REACTOR_TAG_STOP) ||
596 !register_internal_efd(r->epoll_fd, r->wake_efd, N_REACTOR_TAG_WAKE)) {
601 if (!r->registered) {
602 n_log(
LOG_ERR,
"n_reactor_new: cannot allocate registered list");
605 pthread_mutex_init(&r->registered_lock, NULL);
607 atomic_store(&r->events_processed, 0);
608 atomic_store(&r->writes_partial, 0);
609 atomic_store(&r->reads_partial, 0);
610 atomic_store(&r->fds_registered, 0);
611 atomic_store(&r->fds_unregistered, 0);
612 atomic_store(&r->wake_signals, 0);
613 atomic_store(&r->wake_walks, 0);
614 atomic_store(&r->wake_walk_visits, 0);
615 atomic_store(&r->wake_walk_drains, 0);
616 r->stop_requested = 0;
621 if (!r->dirty_pending) {
622 n_log(
LOG_ERR,
"n_reactor_new: cannot allocate dirty_pending list");
625 pthread_mutex_init(&r->dirty_lock, NULL);
627 n_log(
LOG_INFO,
"n_reactor_new: created (epoll_fd=%d stop_efd=%d wake_efd=%d)",
628 r->epoll_fd, r->stop_efd, r->wake_efd);
633 if (r->wake_efd >= 0) close(r->wake_efd);
634 if (r->stop_efd >= 0) close(r->stop_efd);
635 if (r->epoll_fd >= 0) close(r->epoll_fd);
642 if (!reactor || !*reactor)
return;
648 if (r->wake_efd >= 0) close(r->wake_efd);
649 if (r->stop_efd >= 0) close(r->stop_efd);
650 if (r->epoll_fd >= 0) close(r->epoll_fd);
653 pthread_mutex_destroy(&r->registered_lock);
655 if (r->dirty_pending) {
657 pthread_mutex_destroy(&r->dirty_lock);
665 if (!reactor)
return;
667 struct epoll_event events[N_REACTOR_BATCH_SIZE];
669 while (!reactor->stop_requested) {
675 int n = epoll_wait(reactor->epoll_fd, events,
676 N_REACTOR_BATCH_SIZE, 1000);
678 if (errno == EINTR)
continue;
687 reactor_sweep_exit_asked(reactor);
691 for (
int i = 0; i < n; i++) {
692 uint64_t tag = events[i].data.u64;
693 atomic_fetch_add(&reactor->events_processed, 1);
695 if (tag == N_REACTOR_TAG_STOP) {
700 drain_eventfd(reactor->stop_efd);
701 reactor->stop_requested = 1;
704 if (tag == N_REACTOR_TAG_WAKE) {
710 long long drained = drain_eventfd(reactor->wake_efd);
711 atomic_fetch_add(&reactor->wake_signals, drained);
718 atomic_fetch_add(&reactor->wake_walks, 1);
719 long long visits_this_walk = 0;
720 long long drains_this_walk = 0;
722 LIST* walk_list = NULL;
724 pthread_mutex_t* walk_lock = NULL;
725 pthread_mutex_lock(&reactor->dirty_lock);
727 LIST* old = reactor->dirty_pending;
730 reactor->dirty_pending = fresh;
737 walk_lock = &reactor->dirty_lock;
741 pthread_mutex_unlock(&reactor->dirty_lock);
765 if (has_inflight || has_queued) {
767 int rc = reactor_drain_writes(
netw, reactor);
771 if (reactor_epoll_mod(reactor,
netw,
772 EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET)) {
787 if (walk_lock) pthread_mutex_unlock(walk_lock);
790 if (walk_lock) pthread_mutex_lock(walk_lock);
794 if (reactor_epoll_mod(reactor,
netw,
795 EPOLLIN | EPOLLRDHUP | EPOLLET)) {
808 if (walk_lock) pthread_mutex_unlock(walk_lock);
809 if (walk_owned && walk_list) {
814 atomic_fetch_add(&reactor->wake_walk_visits, visits_this_walk);
815 atomic_fetch_add(&reactor->wake_walk_drains, drains_this_walk);
826 uint32_t evmask = events[i].events;
827 if (evmask & (EPOLLERR | EPOLLHUP | EPOLLRDHUP)) {
839 if (evmask & EPOLLIN) {
840 if (!reactor_handle_readable(
netw, reactor)) {
852 int rc = reactor_drain_writes(
netw, reactor);
863 if (reactor_epoll_mod(reactor,
netw,
864 EPOLLIN | EPOLLRDHUP | EPOLLET)) {
868 if (reactor_epoll_mod(reactor,
netw,
869 EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET)) {
878 if (reactor_epoll_mod(reactor,
netw,
879 EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET)) {
884 if (evmask & EPOLLOUT) {
888 if (!reactor_handle_readable(
netw, reactor)) {
897 int rc = reactor_drain_writes(
netw, reactor);
911 if (reactor_epoll_mod(reactor,
netw,
912 EPOLLIN | EPOLLRDHUP | EPOLLET)) {
923 reactor_sweep_exit_asked(reactor);
927 (
long long)atomic_load(&reactor->events_processed));
931 if (!reactor || reactor->stop_efd < 0)
return;
936 ssize_t w = write(reactor->stop_efd, &one,
sizeof(one));
949 memset(out, 0,
sizeof(*out));
957 out->
wake_signals = atomic_load(&reactor->wake_signals);
958 out->
wake_walks = atomic_load(&reactor->wake_walks);
964 if (!reactor || !
netw)
return 0;
972 "n_reactor_register: socket %d has thread engine started; "
973 "reactor and thread mode are mutually exclusive",
978 n_log(
LOG_ERR,
"n_reactor_register: O_NONBLOCK on socket %d failed: %s",
991 reactor_recv_state_reset(
netw);
992 reactor_send_state_reset(
netw);
999 struct epoll_event ev;
1003 ev.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
1005 if (epoll_ctl(reactor->epoll_fd, EPOLL_CTL_ADD,
netw->
link.
sock, &ev) != 0) {
1006 n_log(
LOG_ERR,
"n_reactor_register: epoll_ctl ADD socket %d failed: %s",
1028 pthread_mutex_lock(&reactor->registered_lock);
1030 pthread_mutex_unlock(&reactor->registered_lock);
1032 atomic_fetch_add(&reactor->fds_registered, 1);
1038 if (!reactor || !
netw)
return;
1042 if (epoll_ctl(reactor->epoll_fd, EPOLL_CTL_DEL,
netw->
link.
sock, NULL) != 0) {
1043 if (errno != EBADF && errno != ENOENT) {
1045 "n_reactor_unregister: epoll_ctl DEL socket %d "
1050 reactor_recv_state_reset(
netw);
1051 reactor_send_state_reset(
netw);
1054 pthread_mutex_lock(&reactor->registered_lock);
1055 LIST_NODE* node = reactor->registered->start;
1057 if (node->
ptr == (
void*)
netw) {
1063 pthread_mutex_unlock(&reactor->registered_lock);
1070 pthread_mutex_lock(&reactor->dirty_lock);
1072 LIST_NODE* dn = reactor->dirty_pending->start;
1081 pthread_mutex_unlock(&reactor->dirty_lock);
1091 atomic_fetch_add(&reactor->fds_unregistered, 1);
1109 if (r->wake_efd < 0)
return;
1121 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) {
1122 pthread_mutex_lock(&r->dirty_lock);
1124 pthread_mutex_unlock(&r->dirty_lock);
1129 ssize_t w = write(r->wake_efd, &one,
sizeof(one));
1171 struct timespec ts = {0, 1000000L};
1172 nanosleep(&ts, NULL);
1177 size_t send_list_limit,
1178 size_t recv_list_limit,
1183 n_log(
LOG_ERR,
"netw_accept_into_reactor: NULL reactor");
1184 if (retval) *retval = EINVAL;
1188 recv_list_limit, blocking, retval);
1189 if (!
netw)
return NULL;
1194 n_log(
LOG_ERR,
"netw_accept_into_reactor: register failed for socket %d",
1197 if (retval) *retval = EIO;
1210 "n_reactor_new: epoll/eventfd not available on this "
1211 "platform; reactor mode unsupported (use thread mode)");
1216 if (reactor) *reactor = NULL;
1234 if (out) memset(out, 0,
sizeof(*out));
1257 size_t send_list_limit,
1258 size_t recv_list_limit,
1263 (void)send_list_limit;
1264 (void)recv_list_limit;
1267 if (retval) *retval = ENOSYS;
NETWORK * netw
Network for server mode, accepting incomming.
#define FreeNoLog(__ptr)
Free Handler without log.
#define Malloc(__ptr, __struct, __size)
Malloc Handler to get errors and set to 0.
#define __n_assert(__ptr, __ret)
macro to assert things
#define Free(__ptr)
Free Handler to get errors.
void * ptr
void pointer to store
LIST_NODE * start
pointer to the start of the list
size_t nb_items
number of item currently in the list
struct LIST_NODE * next
pointer to the next node
#define list_shift(__LIST_, __TYPE_)
Shift macro helper for void pointer casting.
int list_push(LIST *list, void *ptr, void(*destructor)(void *ptr))
Add a pointer to the end of the list.
#define remove_list_node(__LIST_, __NODE_, __TYPE_)
Remove macro helper for void pointer casting.
int list_destroy(LIST **list)
Empty and Free a list container.
LIST * new_generic_list(size_t max_items)
Initialiaze a generic list container to max_items pointers.
#define MAX_LIST_ITEMS
flag to pass to new_generic_list for the maximum possible number of item in a list
Structure of a generic LIST container.
Structure of a generic list node.
#define n_log(__LEVEL__,...)
Logging function wrapper to get line and func.
#define LOG_DEBUG
debug-level messages
#define LOG_ERR
error conditions
#define LOG_WARNING
warning conditions
#define LOG_INFO
informational
N_STR * zip4_nstr(N_STR *src)
Compress src with LZ4 block format.
N_STR * unzip4_nstr(N_STR *src)
Decompress an N_STR produced by zip4_nstr.
size_t written
number of meaningful bytes in data, excluding the null terminator; the size including the null termin...
size_t length
total allocation (in bytes) of the data buffer, padding included
void free_nstr_ptr(void *ptr)
Free a N_STR pointer structure.
#define free_nstr(__ptr)
free a N_STR structure and set the pointer to NULL
A box including a string and his lenght.
int reactor_recv_wants_write
TLS-over-reactor: recv returned NETW_IO_WANT_WRITE, the reactor re-runs the readable drain after the ...
N_SOCKET link
networking socket
int threaded_engine_status
Threaded network engine state for this network.
char * reactor_read_payload
malloc'd accumulator for payload bytes
int compress_mode
Per-packet compression mode, see NETW_COMPRESS_MODE.
int reactor_write_armed
1 = EPOLLOUT currently registered
size_t reactor_send_off
bytes already sent to socket
int reactor_read_hdr_have
bytes accumulated in reactor_read_hdr_buf
uint32_t reactor_read_pkt_state
state word for the in-flight frame
int reactor_read_phase
0=STATE, 1=LENGTH, 2=PAYLOAD
int in_dirty_list
Dirty-list membership flag.
int reactor_close_acked
Close handshake.
pthread_mutex_t recvbolt
mutex for threaded access of recv buf
pthread_mutex_t sendbolt
mutex for threaded access of send_buf
uint32_t reactor_read_pkt_length
payload length for the in-flight frame
size_t reactor_read_payload_have
bytes accumulated into reactor_read_payload
char * reactor_send_buf
malloc'd framed bytes, NULL when idle
SOCKET sock
a normal socket
netw_func recv_data_once
single-attempt recv, same contract as send_data_once.
LIST * recv_buf
reveicing buffer (for incomming usage)
size_t reactor_send_len
total bytes in reactor_send_buf
int reactor_registered
Reactor close-handshake latch.
netw_func send_data_once
single-attempt send (non-blocking / reactor use).
unsigned char reactor_read_hdr_buf[4]
accumulator for header words
int reactor_send_wants_read
TLS-over-reactor: the in-flight send returned NETW_IO_WANT_READ (renegotiation), the reactor retries ...
LIST * send_buf
sending buffer (for outgoing queuing )
#define NETW_IO_WANT_READ
single-attempt I/O (send_data_once / recv_data_once): the operation cannot progress until the socket ...
#define NETW_COMPRESS_THRESHOLD
Opportunistic compression policy.
#define NETW_COMPRESS_MIN_RATIO
ssize_t recv_data_once(void *netw, char *buf, uint32_t n)
single-attempt recv for non-blocking sockets (reactor use).
ssize_t send_data_once(void *netw, char *buf, uint32_t n)
single-attempt send for non-blocking sockets (reactor use).
#define netw_atomic_write_reactor_handle(netw, val)
#define netw_atomic_read_reactor_mode(netw)
Lock-free atomic read of the reactor_mode flag.
NETWORK * netw_accept_from_ex(NETWORK *from, size_t send_list_limit, size_t recv_list_limit, int blocking, int *retval)
make a normal 'accept' .
#define netw_atomic_write_reactor_mode(netw, val)
#define NETW_SOCKET_DISCONNECTED
Code for a disconnected recv.
int netw_set(NETWORK *netw, int flag)
Restart or reset the specified network ability.
int netw_get_state(NETWORK *netw, uint32_t *state, int *thr_engine_status)
Get the state of a network.
#define netw_atomic_read_reactor_handle(netw)
Same contract for the back-pointer to the reactor.
#define NETW_IO_WANT_WRITE
single-attempt I/O: the operation cannot progress until the socket is WRITABLE.
int netw_close(NETWORK **netw)
Closing a specified Network, destroy queues, free the structure.
@ NETW_COMPRESS_NONE
no automatic compression on send, still decompresses inbound
@ NETW_COMPRESS_LZ4
compress on send with LZ4, decompress either on recv
@ NETW_THR_ENGINE_STARTED
N_STR * unzip_nstr(N_STR *src)
return an uncompressed version of src
N_STR * zip_nstr(N_STR *src)
return a compressed version of src
Common headers and low-level functions & define.
List structures and definitions.
LZ4 block-compression handler.
void * n_reactor_run_thread_entry(void *arg)
pthread_create-compatible entry point that calls n_reactor_run on the reactor passed via arg.
void n_reactor_run(n_reactor *reactor)
Run the epoll loop on the calling thread.
void n_reactor_get_stats(const n_reactor *reactor, n_reactor_stats *out)
Read current stats counters into *out.
n_reactor * n_reactor_new(int max_fds_hint)
Create a new reactor.
NETWORK * netw_accept_into_reactor(NETWORK *listener, size_t send_list_limit, size_t recv_list_limit, int blocking, n_reactor *reactor, int *retval)
Accept a connection on listener and register it with reactor instead of starting per-connection threa...
void n_reactor_unregister(n_reactor *reactor, NETWORK *netw)
Unregister a NETWORK from the reactor.
int n_reactor_register(n_reactor *reactor, NETWORK *netw)
Register a NETWORK with the reactor.
void n_reactor_notify_send(NETWORK *netw)
Producer-side wake-up after a netw_add_msg.
void n_reactor_close_netw_sync(NETWORK *netw)
Synchronously close a reactor-registered NETWORK from the game thread.
void n_reactor_stop(n_reactor *reactor)
Signal the run loop to exit at the next iteration.
void n_reactor_destroy(n_reactor **reactor)
Tear down a reactor.
Single-threaded epoll reactor for n_network connections.
long long fds_registered
lifetime register call count
long long writes_partial
EAGAIN on send -> re-armed EPOLLOUT.
long long reads_partial
EAGAIN on recv -> kept accumulator.
long long events_processed
total epoll events dispatched
long long wake_signals
eventfd wake events processed
long long wake_walk_visits
long long wake_walk_drains
struct n_reactor n_reactor
Opaque reactor handle.
long long fds_unregistered
lifetime unregister call count
Counters for the dashboard / profile_server.sh.
N_STR and string function declaration.
ZLIB compression handler.