The branch main has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=a58590631ccc0fa5bdbbdf88021c6878d644d128
commit a58590631ccc0fa5bdbbdf88021c6878d644d128 Author: Mark Johnston <[email protected]> AuthorDate: 2026-07-08 17:12:54 +0000 Commit: Mark Johnston <[email protected]> CommitDate: 2026-07-09 13:41:27 +0000 taskqueue: Avoid unbounded epoch read sections The taskqueue thread loop tries to avoid entering and exiting net epoch read sections for every task. This reduces the overhead of net epoch integration, but the implementation wasn't bounding the length of the read section, so a busy taskqueue thread could hold an epoch open for an unbounded period. This is easy to achieve with the epair task, for instance. Bound the number of tasks that we'll execute without observing the global epoch, and provide a sysctl to control it. Let the default bound be eight. Reviewed by: glebius MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D58031 --- sys/kern/subr_gtaskqueue.c | 20 ++++++++++++-------- sys/kern/subr_taskqueue.c | 34 ++++++++++++++++++++++++++-------- sys/sys/taskqueue.h | 2 ++ 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/sys/kern/subr_gtaskqueue.c b/sys/kern/subr_gtaskqueue.c index e13721ffe1f2..4b0600101b64 100644 --- a/sys/kern/subr_gtaskqueue.c +++ b/sys/kern/subr_gtaskqueue.c @@ -342,14 +342,14 @@ gtaskqueue_run_locked(struct gtaskqueue *queue) struct epoch_tracker et; struct gtaskqueue_busy tb; struct gtask *gtask; - bool in_net_epoch; + unsigned int epochtasks; KASSERT(queue != NULL, ("tq is NULL")); TQ_ASSERT_LOCKED(queue); tb.tb_running = NULL; LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link); - in_net_epoch = false; + epochtasks = 0; while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) { STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); gtask->ta_flags &= ~TASK_ENQUEUED; @@ -358,19 +358,23 @@ gtaskqueue_run_locked(struct gtaskqueue *queue) TQ_UNLOCK(queue); KASSERT(gtask->ta_func != NULL, ("task->ta_func is NULL")); - if (!in_net_epoch && TASK_IS_NET(gtask)) { - in_net_epoch = true; - NET_EPOCH_ENTER(et); - } else if (in_net_epoch && !TASK_IS_NET(gtask)) { + if (TASK_IS_NET(gtask)) { + if (epochtasks++ == 0) + NET_EPOCH_ENTER(et); + } else if (epochtasks > 0) { NET_EPOCH_EXIT(et); - in_net_epoch = false; + epochtasks = 0; } gtask->ta_func(gtask->ta_context); + if (epochtasks > net_epoch_task_limit) { + NET_EPOCH_EXIT(et); + epochtasks = 0; + } TQ_LOCK(queue); wakeup(gtask); } - if (in_net_epoch) + if (epochtasks > 0) NET_EPOCH_EXIT(et); LIST_REMOVE(&tb, tb_link); } diff --git a/sys/kern/subr_taskqueue.c b/sys/kern/subr_taskqueue.c index f17e59a441a6..b337aa83d69d 100644 --- a/sys/kern/subr_taskqueue.c +++ b/sys/kern/subr_taskqueue.c @@ -43,6 +43,7 @@ #include <sys/sched.h> #include <sys/smp.h> #include <sys/stdarg.h> +#include <sys/sysctl.h> #include <sys/taskqueue.h> #include <sys/unistd.h> @@ -78,6 +79,19 @@ struct taskqueue { void *tq_cb_contexts[TASKQUEUE_NUM_CALLBACKS]; }; +static SYSCTL_NODE(_kern, OID_AUTO, taskqueue, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, + "taskqueue information"); + +/* + * Limit on the number of tasks that may be run in a single epoch section. + * It's profitable to batch tasks together, but there must be a bound in order + * to maintain system liveness. + */ +unsigned int net_epoch_task_limit = 8; +SYSCTL_UINT(_kern_taskqueue, OID_AUTO, net_epoch_task_limit, CTLFLAG_RWTUN, + &net_epoch_task_limit, 0, + "Maximum number of tasks to run in an epoch section"); + #define TQ_FLAGS_ACTIVE (1 << 0) #define TQ_FLAGS_BLOCKED (1 << 1) #define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2) @@ -486,15 +500,15 @@ taskqueue_run_locked(struct taskqueue *queue) struct epoch_tracker et; struct taskqueue_busy tb; struct task *task; - bool in_net_epoch; + unsigned int epochtasks; int pending; KASSERT(queue != NULL, ("tq is NULL")); TQ_ASSERT_LOCKED(queue); tb.tb_running = NULL; LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link); - in_net_epoch = false; + epochtasks = 0; while ((task = STAILQ_FIRST(&queue->tq_queue)) != NULL) { STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); if (queue->tq_hint == task) @@ -507,19 +521,23 @@ taskqueue_run_locked(struct taskqueue *queue) TQ_UNLOCK(queue); KASSERT(task->ta_func != NULL, ("task->ta_func is NULL")); - if (!in_net_epoch && TASK_IS_NET(task)) { - in_net_epoch = true; - NET_EPOCH_ENTER(et); - } else if (in_net_epoch && !TASK_IS_NET(task)) { + if (TASK_IS_NET(task)) { + if (epochtasks++ == 0) + NET_EPOCH_ENTER(et); + } else if (epochtasks > 0) { NET_EPOCH_EXIT(et); - in_net_epoch = false; + epochtasks = 0; } task->ta_func(task->ta_context, pending); + if (epochtasks > net_epoch_task_limit) { + NET_EPOCH_EXIT(et); + epochtasks = 0; + } TQ_LOCK(queue); wakeup(task); } - if (in_net_epoch) + if (epochtasks > 0) NET_EPOCH_EXIT(et); LIST_REMOVE(&tb, tb_link); } diff --git a/sys/sys/taskqueue.h b/sys/sys/taskqueue.h index 9da364a1a716..b9d4ebe98be4 100644 --- a/sys/sys/taskqueue.h +++ b/sys/sys/taskqueue.h @@ -221,4 +221,6 @@ struct taskqueue *taskqueue_create_fast(const char *name, int mflags, */ TASKQUEUE_DECLARE(bus); +extern unsigned int net_epoch_task_limit; + #endif /* !_SYS_TASKQUEUE_H_ */
