In interrupt and legacy modes an idle worker arms its Rx queues and sleeps in rte_epoll_wait() with a 10 ms timeout. The timeout lets the worker wake periodically to re-check the quit flag, but it also hides a broken interrupt path: with no interrupt delivered the worker still wakes every 10 ms and polls, so traffic keeps flowing and interrupt mode looks like it works when it does not.
Block indefinitely instead, so only a real Rx interrupt can wake the worker. To still stop the workers on SIGINT, add an eventfd to each worker epoll set; the signal handler writes it to wake every worker, which then observes the quit flag and leaves its loop. Both loops share this sleep path, so the eventfd is created for the interrupt and legacy modes; it is skipped in the wakeup log so it is not reported as an Rx interrupt. Signed-off-by: Maxime Leroy <[email protected]> --- examples/l3fwd-power/main.c | 40 +++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c index c423b0dba5..73ee2d2def 100644 --- a/examples/l3fwd-power/main.c +++ b/examples/l3fwd-power/main.c @@ -9,6 +9,8 @@ #include <sys/types.h> #include <string.h> #include <sys/queue.h> +#include <sys/epoll.h> +#include <sys/eventfd.h> #include <stdarg.h> #include <errno.h> #include <getopt.h> @@ -143,6 +145,9 @@ static int promiscuous_on = 0; /* NUMA is enabled by default. */ static int numa_on = 1; volatile bool quit_signal; +/* eventfd to wake idle workers out of a blocking rte_epoll_wait() at exit */ +static int wakeup_fd = -1; +static struct rte_epoll_event wakeup_event[RTE_MAX_LCORE]; /* timer to update telemetry every 500ms */ static struct rte_timer telemetry_timer; @@ -420,10 +425,17 @@ static int is_done(void) static void signal_exit_now(int sigtype) { + uint64_t one = 1; + ssize_t nbytes; - if (sigtype == SIGINT) + if (sigtype == SIGINT) { quit_signal = true; - + /* interrupt mode only: wake a worker blocked in rte_epoll_wait() */ + if (wakeup_fd >= 0) { + nbytes = write(wakeup_fd, &one, sizeof(one)); + RTE_SET_USED(nbytes); + } + } } /* Frequency scale down timer callback */ @@ -835,7 +847,7 @@ sleep_until_rx_interrupt(int num, int lcore) static alignas(RTE_CACHE_LINE_SIZE) struct { bool wakeup; } status[RTE_MAX_LCORE]; - struct rte_epoll_event event[num]; + struct rte_epoll_event event[num + 1]; int n, i; uint16_t port_id; uint16_t queue_id; @@ -847,8 +859,10 @@ sleep_until_rx_interrupt(int num, int lcore) rte_lcore_id()); } - n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num, 10); + n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num + 1, -1); for (i = 0; i < n; i++) { + if (event[i].fd == wakeup_fd) + continue; data = event[i].epdata.data; port_id = ((uintptr_t)data) >> (sizeof(uint16_t) * CHAR_BIT); queue_id = ((uintptr_t)data) & @@ -921,6 +935,7 @@ rx_intr_enable_all(struct lcore_conf *qconf) static int event_register(struct lcore_conf *qconf) { struct lcore_rx_queue *rx_queue; + struct rte_epoll_event *wev; uint16_t queueid; uint16_t portid; uint32_t data; @@ -946,6 +961,16 @@ static int event_register(struct lcore_conf *qconf) return ret; } + /* also watch the shutdown eventfd so a blocking wait wakes at exit */ + if (wakeup_fd >= 0) { + wev = &wakeup_event[rte_lcore_id()]; + wev->epdata.event = EPOLLIN | EPOLLPRI; + ret = rte_epoll_ctl(RTE_EPOLL_PER_THREAD, EPOLL_CTL_ADD, + wakeup_fd, wev); + if (ret && ret != -EEXIST) + return ret; + } + return 0; } @@ -2989,6 +3014,13 @@ main(int argc, char **argv) check_all_ports_link_status(enabled_port_mask); + if (app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_INTERRUPT) { + wakeup_fd = eventfd(0, EFD_NONBLOCK); + if (wakeup_fd < 0) + rte_exit(EXIT_FAILURE, "eventfd() failed: %s\n", + strerror(errno)); + } + /* launch per-lcore init on every lcore */ if (app_mode == APP_MODE_LEGACY) { rte_eal_mp_remote_launch(main_legacy_loop, NULL, CALL_MAIN); -- 2.43.0
