svn commit: r256613 - head/sys/kern

[Alexander Motin]

Author: mav
Date: Wed Oct 16 09:52:59 2013
New Revision: 256613
URL: http://svnweb.freebsd.org/changeset/base/256613

Log:
  MFprojects/camlock r254763:
  Move tq_enqueue() call out of the queue lock for known handlers (actually
  I have found no others in the base system).  This reduces queue lock hold
  time and congestion spinning under active multithreaded enqueuing.

Modified:
  head/sys/kern/subr_taskqueue.c

Modified: head/sys/kern/subr_taskqueue.c
==============================================================================
--- head/sys/kern/subr_taskqueue.c      Wed Oct 16 09:48:23 2013        
(r256612)
+++ head/sys/kern/subr_taskqueue.c      Wed Oct 16 09:52:59 2013        
(r256613)
@@ -47,6 +47,9 @@ __FBSDID("$FreeBSD$");
 static MALLOC_DEFINE(M_TASKQUEUE, "taskqueue", "Task Queues");
 static void    *taskqueue_giant_ih;
 static void    *taskqueue_ih;
+static void     taskqueue_fast_enqueue(void *);
+static void     taskqueue_swi_enqueue(void *);
+static void     taskqueue_swi_giant_enqueue(void *);
 
 struct taskqueue_busy {
        struct task     *tb_running;
@@ -70,6 +73,7 @@ struct taskqueue {
 
 #define        TQ_FLAGS_ACTIVE         (1 << 0)
 #define        TQ_FLAGS_BLOCKED        (1 << 1)
+#define        TQ_FLAGS_UNLOCKED_ENQUEUE       (1 << 2)
 
 #define        DT_CALLOUT_ARMED        (1 << 0)
 
@@ -97,7 +101,8 @@ _timeout_task_init(struct taskqueue *que
 {
 
        TASK_INIT(&timeout_task->t, priority, func, context);
-       callout_init_mtx(&timeout_task->c, &queue->tq_mutex, 0);
+       callout_init_mtx(&timeout_task->c, &queue->tq_mutex,
+           CALLOUT_RETURNUNLOCKED);
        timeout_task->q = queue;
        timeout_task->f = 0;
 }
@@ -128,6 +133,11 @@ _taskqueue_create(const char *name __unu
        queue->tq_context = context;
        queue->tq_spin = (mtxflags & MTX_SPIN) != 0;
        queue->tq_flags |= TQ_FLAGS_ACTIVE;
+       if (enqueue == taskqueue_fast_enqueue ||
+           enqueue == taskqueue_swi_enqueue ||
+           enqueue == taskqueue_swi_giant_enqueue ||
+           enqueue == taskqueue_thread_enqueue)
+               queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE;
        mtx_init(&queue->tq_mutex, mtxname, NULL, mtxflags);
 
        return queue;
@@ -197,6 +207,7 @@ taskqueue_enqueue_locked(struct taskqueu
        if (task->ta_pending) {
                if (task->ta_pending < USHRT_MAX)
                        task->ta_pending++;
+               TQ_UNLOCK(queue);
                return (0);
        }
 
@@ -220,8 +231,12 @@ taskqueue_enqueue_locked(struct taskqueu
        }
 
        task->ta_pending = 1;
+       if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) != 0)
+               TQ_UNLOCK(queue);
        if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0)
                queue->tq_enqueue(queue->tq_context);
+       if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) == 0)
+               TQ_UNLOCK(queue);
 
        return (0);
 }
@@ -232,7 +247,6 @@ taskqueue_enqueue(struct taskqueue *queu
 
        TQ_LOCK(queue);
        res = taskqueue_enqueue_locked(queue, task);
-       TQ_UNLOCK(queue);
 
        return (res);
 }
@@ -278,8 +292,8 @@ taskqueue_enqueue_timeout(struct taskque
                        callout_reset(&timeout_task->c, ticks,
                            taskqueue_timeout_func, timeout_task);
                }
+               TQ_UNLOCK(queue);
        }
-       TQ_UNLOCK(queue);
        return (res);
 }
 
@@ -566,7 +580,6 @@ taskqueue_thread_enqueue(void *context)
        tqp = context;
        tq = *tqp;
 
-       TQ_ASSERT_LOCKED(tq);
        wakeup_one(tq);
 }
 
_______________________________________________
svn-src-head@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"