Prepare pios earlier in preparation to try to execute them earlier. Convert more places to use lock less lists.
https://virtuozzo.atlassian.net/browse/VSTOR-91820 Signed-off-by: Alexander Atanasov <alexander.atana...@virtuozzo.com> --- drivers/md/dm-ploop-map.c | 95 ++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/drivers/md/dm-ploop-map.c b/drivers/md/dm-ploop-map.c index 8d165bd4fa9d..475494b951a3 100644 --- a/drivers/md/dm-ploop-map.c +++ b/drivers/md/dm-ploop-map.c @@ -292,11 +292,15 @@ static struct pio *ploop_split_and_chain_pio(struct ploop *ploop, ALLOW_ERROR_INJECTION(ploop_split_and_chain_pio, NULL); static int ploop_split_pio_to_list(struct ploop *ploop, struct pio *pio, - struct list_head *ret_list) + struct llist_head *ret_list) { u32 clu_size = CLU_SIZE(ploop); + struct llist_node *pos, *t; struct pio *split; - LIST_HEAD(list); + LLIST_HEAD(llist); + struct llist_node *lltmp; + + WARN_ON(!pio->bi_iter.bi_size); while (1) { loff_t start = to_bytes(pio->bi_iter.bi_sector); @@ -314,17 +318,24 @@ static int ploop_split_pio_to_list(struct ploop *ploop, struct pio *pio, if (!split) goto err; - list_add_tail(&split->list, &list); + llist_add((struct llist_node *)(&split->list), &llist); } - list_splice_tail(&list, ret_list); - list_add_tail(&pio->list, ret_list); + INIT_LIST_HEAD(&pio->list); + llist_add((struct llist_node *)(&pio->list), &llist); + lltmp = llist_reverse_order(llist_del_all(&llist)); + pio->list.next = NULL; + llist_add_batch(lltmp, (struct llist_node *)(&pio->list), ret_list); + return 0; err: - while ((pio = ploop_pio_list_pop(&list)) != NULL) { + llist_for_each_safe(pos, t, llist.first) { + pio = list_entry((struct list_head *)pos, typeof(*pio), list); pio->bi_status = BLK_STS_RESOURCE; + INIT_LIST_HEAD(&pio->list); ploop_pio_endio(pio); } + return -ENOMEM; } ALLOW_ERROR_INJECTION(ploop_split_pio_to_list, ERRNO); @@ -341,7 +352,7 @@ static void ploop_schedule_work(struct ploop *ploop) static void ploop_dispatch_pio(struct ploop *ploop, struct pio *pio) { - struct llist_head *list = (struct llist_head *)&ploop->pios[pio->queue_list_id]; + struct llist_head *list = &ploop->pios[pio->queue_list_id]; lockdep_assert_not_held(&ploop->deferred_lock); WARN_ON_ONCE(pio->queue_list_id >= PLOOP_LIST_COUNT); @@ -1622,12 +1633,11 @@ ALLOW_ERROR_INJECTION(ploop_create_bvec_from_rq, NULL); static void ploop_prepare_one_embedded_pio(struct ploop *ploop, struct pio *pio, - struct list_head *deferred_pios) + struct llist_head *lldeferred_pios) { struct ploop_rq *prq = pio->endio_cb_data; struct request *rq = prq->rq; struct bio_vec *bvec = NULL; - LIST_HEAD(list); int ret; if (rq->bio != rq->biotail) { @@ -1646,16 +1656,18 @@ static void ploop_prepare_one_embedded_pio(struct ploop *ploop, pio->bi_iter.bi_idx = 0; pio->bi_iter.bi_bvec_done = 0; } else { - /* Single bio already provides bvec array */ + /* Single bio already provides bvec array + * bvec is updated to the correct on submit + * it is different after partial IO + */ bvec = rq->bio->bi_io_vec; - pio->bi_iter = rq->bio->bi_iter; } pio->bi_iter.bi_sector = ploop_rq_pos(ploop, rq); pio->bi_io_vec = bvec; pio->queue_list_id = PLOOP_LIST_DEFERRED; - ret = ploop_split_pio_to_list(ploop, pio, deferred_pios); + ret = ploop_split_pio_to_list(ploop, pio, lldeferred_pios); if (ret) goto err_nomem; @@ -1667,7 +1679,7 @@ static void ploop_prepare_one_embedded_pio(struct ploop *ploop, static void ploop_prepare_embedded_pios(struct ploop *ploop, struct llist_node *pios, - struct list_head *deferred_pios) + struct llist_head *deferred_pios) { struct pio *pio; struct llist_node *pos, *t; @@ -1684,12 +1696,17 @@ static void ploop_prepare_embedded_pios(struct ploop *ploop, } static void ploop_process_deferred_pios(struct ploop *ploop, - struct list_head *pios) + struct llist_head *pios) { struct pio *pio; - while ((pio = ploop_pio_list_pop(pios)) != NULL) + struct llist_node *pos, *t; + + llist_for_each_safe(pos, t, pios->first) { + pio = list_entry((struct list_head *)pos, typeof(*pio), list); + INIT_LIST_HEAD(&pio->list); /* until type is changed */ ploop_process_one_deferred_bio(ploop, pio); + } } static void ploop_process_one_discard_pio(struct ploop *ploop, struct pio *pio) @@ -1787,19 +1804,13 @@ static void ploop_submit_metadata_writeback(struct ploop *ploop) } } -static void process_ploop_fsync_work(struct ploop *ploop) +static void process_ploop_fsync_work(struct ploop *ploop, struct llist_node *llflush_pios) { struct file *file; struct pio *pio; int ret; - struct llist_node *llflush_pios; struct llist_node *pos, *t; - llflush_pios = llist_del_all(&ploop->pios[PLOOP_LIST_FLUSH]); - - if (!llflush_pios) - return; - file = ploop_top_delta(ploop)->file; /* All flushes are done as one */ ret = vfs_fsync(file, 0); @@ -1818,38 +1829,39 @@ static void process_ploop_fsync_work(struct ploop *ploop) void do_ploop_run_work(struct ploop *ploop) { - LIST_HEAD(deferred_pios); + LLIST_HEAD(deferred_pios); struct llist_node *llembedded_pios; struct llist_node *lldeferred_pios; struct llist_node *lldiscard_pios; struct llist_node *llcow_pios; struct llist_node *llresubmit; + struct llist_node *llflush_pios; unsigned int old_flags = current->flags; current->flags |= PF_IO_THREAD|PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO; llembedded_pios = llist_del_all(&ploop->pios[PLOOP_LIST_PREPARE]); - smp_wmb(); /* */ - lldeferred_pios = llist_del_all(&ploop->pios[PLOOP_LIST_DEFERRED]); + smp_wmb(); /* */ + llresubmit = llist_del_all(&ploop->llresubmit_pios); lldiscard_pios = llist_del_all(&ploop->pios[PLOOP_LIST_DISCARD]); llcow_pios = llist_del_all(&ploop->pios[PLOOP_LIST_COW]); - llresubmit = llist_del_all(&ploop->llresubmit_pios); /* add old deferred back to the list */ if (lldeferred_pios) { struct llist_node *pos, *t; struct pio *pio; - + /* Add one by one we need last for batch add */ llist_for_each_safe(pos, t, lldeferred_pios) { - pio = list_entry((struct list_head *)pos, typeof(*pio), list); - INIT_LIST_HEAD(&pio->list); - list_add(&pio->list, &deferred_pios); + llist_add(pos, &deferred_pios); } } ploop_prepare_embedded_pios(ploop, llembedded_pios, &deferred_pios); + llflush_pios = llist_del_all(&ploop->pios[PLOOP_LIST_FLUSH]); + smp_wmb(); /* */ + if (llresubmit) ploop_process_resubmit_pios(ploop, llist_reverse_order(llresubmit)); @@ -1863,8 +1875,9 @@ void do_ploop_run_work(struct ploop *ploop) ploop_submit_metadata_writeback(ploop); - if (!llist_empty(&ploop->pios[PLOOP_LIST_FLUSH])) - process_ploop_fsync_work(ploop); + if (llflush_pios) + process_ploop_fsync_work(ploop, llist_reverse_order(llflush_pios)); + current->flags = old_flags; } @@ -1913,6 +1926,10 @@ static void ploop_submit_embedded_pio(struct ploop *ploop, struct pio *pio) struct ploop_rq *prq = pio->endio_cb_data; struct request *rq = prq->rq; bool queue = true; + LLIST_HEAD(deferred_pios); + int ret = 0; + struct pio *spio, *stmp; + struct llist_node *pos, *t; if (blk_rq_bytes(rq)) { pio->queue_list_id = PLOOP_LIST_PREPARE; @@ -1928,19 +1945,27 @@ static void ploop_submit_embedded_pio(struct ploop *ploop, struct pio *pio) } ploop_inc_nr_inflight(ploop, pio); - llist_add((struct llist_node *)(&pio->list), &ploop->pios[PLOOP_LIST_PREPARE]); + ploop_prepare_one_embedded_pio(ploop, pio, &deferred_pios); + llist_for_each_safe(pos, t, deferred_pios.first) { + spio = list_entry((struct list_head *)pos, typeof(*pio), list); + llist_add((struct llist_node *)(&spio->list), &ploop->pios[PLOOP_LIST_DEFERRED]); + } out: if (queue) ploop_schedule_work(ploop); } -void ploop_submit_embedded_pios(struct ploop *ploop, struct list_head *list) +void ploop_submit_embedded_pios(struct ploop *ploop, struct llist_node *list) { struct pio *pio; + struct llist_node *pos, *t; - while ((pio = ploop_pio_list_pop(list)) != NULL) + llist_for_each_safe(pos, t, list) { + pio = list_entry((struct list_head *)pos, typeof(*pio), list); + INIT_LIST_HEAD(&pio->list); ploop_submit_embedded_pio(ploop, pio); + } } int ploop_clone_and_map(struct dm_target *ti, struct request *rq, -- 2.43.0 _______________________________________________ Devel mailing list Devel@openvz.org https://lists.openvz.org/mailman/listinfo/devel
