svn commit: r293666 - stable/10/sys/dev/nvd

[Jim Harris]

Author: jimharris
Date: Mon Jan 11 17:24:18 2016
New Revision: 293666
URL: https://svnweb.freebsd.org/changeset/base/293666

Log:
  MFC r293323:
  
    nvd: submit bios directly when BIO_ORDERED not set or in flight
  
    This significantly improves parallelism in the most common case.
    The taskqueue is still used whenever BIO_ORDERED bios are in flight.
  
    This patch is based heavily on a patch from gallatin@.

Modified:
  stable/10/sys/dev/nvd/nvd.c

Modified: stable/10/sys/dev/nvd/nvd.c
==============================================================================
--- stable/10/sys/dev/nvd/nvd.c Mon Jan 11 17:22:57 2016        (r293665)
+++ stable/10/sys/dev/nvd/nvd.c Mon Jan 11 17:24:18 2016        (r293666)
@@ -73,6 +73,7 @@ struct nvd_disk {
        struct nvme_namespace   *ns;
 
        uint32_t                cur_depth;
+       uint32_t                ordered_in_flight;
 
        TAILQ_ENTRY(nvd_disk)   global_tailq;
        TAILQ_ENTRY(nvd_disk)   ctrlr_tailq;
@@ -160,6 +161,8 @@ nvd_bio_submit(struct nvd_disk *ndisk, s
        err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done);
        if (err) {
                atomic_add_int(&ndisk->cur_depth, -1);
+               if (__predict_false(bp->bio_flags & BIO_ORDERED))
+                       atomic_add_int(&ndisk->ordered_in_flight, -1);
                bp->bio_error = err;
                bp->bio_flags |= BIO_ERROR;
                bp->bio_resid = bp->bio_bcount;
@@ -177,6 +180,18 @@ nvd_strategy(struct bio *bp)
 
        ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1;
 
+       if (__predict_false(bp->bio_flags & BIO_ORDERED))
+               atomic_add_int(&ndisk->ordered_in_flight, 1);
+
+       if (__predict_true(ndisk->ordered_in_flight == 0)) {
+               nvd_bio_submit(ndisk, bp);
+               return;
+       }
+
+       /*
+        * There are ordered bios in flight, so we need to submit
+        *  bios through the task queue to enforce ordering.
+        */
        mtx_lock(&ndisk->bioqlock);
        bioq_insert_tail(&ndisk->bioq, bp);
        mtx_unlock(&ndisk->bioqlock);
@@ -208,6 +223,8 @@ nvd_done(void *arg, const struct nvme_co
        ndisk = bp->bio_disk->d_drv1;
 
        atomic_add_int(&ndisk->cur_depth, -1);
+       if (__predict_false(bp->bio_flags & BIO_ORDERED))
+               atomic_add_int(&ndisk->ordered_in_flight, -1);
 
        biodone(bp);
 }
@@ -316,6 +333,7 @@ nvd_new_disk(struct nvme_namespace *ns, 
        ndisk->ns = ns;
        ndisk->disk = disk;
        ndisk->cur_depth = 0;
+       ndisk->ordered_in_flight = 0;
 
        mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF);
        bioq_init(&ndisk->bioq);
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"