Prepare pios earlier in preparation to try to execute them earlier. Convert more places to use lock less lists.
https://virtuozzo.atlassian.net/browse/VSTOR-91820 Signed-off-by: Alexander Atanasov <alexander.atana...@virtuozzo.com> --- drivers/md/dm-ploop-cmd.c | 13 +---- drivers/md/dm-ploop-map.c | 101 ++++++++++++++++++++++---------------- drivers/md/dm-ploop.h | 2 +- 3 files changed, 62 insertions(+), 54 deletions(-) diff --git a/drivers/md/dm-ploop-cmd.c b/drivers/md/dm-ploop-cmd.c index 3d9818003a08..1cd215800a16 100644 --- a/drivers/md/dm-ploop-cmd.c +++ b/drivers/md/dm-ploop-cmd.c @@ -123,7 +123,6 @@ ALLOW_ERROR_INJECTION(ploop_inflight_bios_ref_switch, ERRNO); static void ploop_resume_submitting_pios(struct ploop *ploop) { - LIST_HEAD(list); struct llist_node *suspended_pending; spin_lock_irq(&ploop->deferred_lock); @@ -132,16 +131,8 @@ static void ploop_resume_submitting_pios(struct ploop *ploop) spin_unlock_irq(&ploop->deferred_lock); suspended_pending = llist_del_all(&ploop->llsuspended_pios); - if (suspended_pending) { - struct llist_node *pos, *t; - - llist_for_each_safe(pos, t, suspended_pending) { - struct pio *pio = list_entry((struct list_head *)pos, typeof(*pio), list); - - list_add(&pio->list, &list); - } - ploop_submit_embedded_pios(ploop, &list); - } + if (suspended_pending) + ploop_submit_embedded_pios(ploop, suspended_pending); } static int ploop_suspend_submitting_pios(struct ploop *ploop) diff --git a/drivers/md/dm-ploop-map.c b/drivers/md/dm-ploop-map.c index 76ef88d563a3..897b4cb04746 100644 --- a/drivers/md/dm-ploop-map.c +++ b/drivers/md/dm-ploop-map.c @@ -143,18 +143,11 @@ static void ploop_init_prq_and_embedded_pio(struct ploop *ploop, void ploop_enospc_timer(struct timer_list *timer) { struct ploop *ploop = from_timer(ploop, timer, enospc_timer); - LIST_HEAD(list); - struct llist_node *pos, *t; struct llist_node *enospc_pending = llist_del_all(&ploop->enospc_pios); if (enospc_pending) { enospc_pending = llist_reverse_order(enospc_pending); - llist_for_each_safe(pos, t, enospc_pending) { - struct pio *pio = list_entry((struct list_head *)pos, typeof(*pio), list); - - list_add(&pio->list, &list); - } - ploop_submit_embedded_pios(ploop, &list); + ploop_submit_embedded_pios(ploop, enospc_pending); } } @@ -303,11 +296,15 @@ static struct pio *ploop_split_and_chain_pio(struct ploop *ploop, ALLOW_ERROR_INJECTION(ploop_split_and_chain_pio, NULL); static int ploop_split_pio_to_list(struct ploop *ploop, struct pio *pio, - struct list_head *ret_list) + struct llist_head *ret_list) { u32 clu_size = CLU_SIZE(ploop); + struct llist_node *pos, *t; struct pio *split; - LIST_HEAD(list); + LLIST_HEAD(llist); + struct llist_node *lltmp; + + WARN_ON(!pio->bi_iter.bi_size); while (1) { loff_t start = to_bytes(pio->bi_iter.bi_sector); @@ -325,17 +322,24 @@ static int ploop_split_pio_to_list(struct ploop *ploop, struct pio *pio, if (!split) goto err; - list_add_tail(&split->list, &list); + llist_add((struct llist_node *)(&split->list), &llist); } - list_splice_tail(&list, ret_list); - list_add_tail(&pio->list, ret_list); + INIT_LIST_HEAD(&pio->list); + llist_add((struct llist_node *)(&pio->list), &llist); + lltmp = llist_reverse_order(llist_del_all(&llist)); + pio->list.next = NULL; + llist_add_batch(lltmp, (struct llist_node *)(&pio->list), ret_list); + return 0; err: - while ((pio = ploop_pio_list_pop(&list)) != NULL) { + llist_for_each_safe(pos, t, llist.first) { + pio = list_entry((struct list_head *)pos, typeof(*pio), list); pio->bi_status = BLK_STS_RESOURCE; + INIT_LIST_HEAD(&pio->list); ploop_pio_endio(pio); } + return -ENOMEM; } ALLOW_ERROR_INJECTION(ploop_split_pio_to_list, ERRNO); @@ -347,7 +351,7 @@ static void ploop_schedule_work(struct ploop *ploop) static void ploop_dispatch_pio(struct ploop *ploop, struct pio *pio) { - struct llist_head *list = (struct llist_head *)&ploop->pios[pio->queue_list_id]; + struct llist_head *list = &ploop->pios[pio->queue_list_id]; WARN_ON_ONCE(pio->queue_list_id >= PLOOP_LIST_COUNT); @@ -1632,12 +1636,11 @@ ALLOW_ERROR_INJECTION(ploop_create_bvec_from_rq, NULL); static void ploop_prepare_one_embedded_pio(struct ploop *ploop, struct pio *pio, - struct list_head *deferred_pios) + struct llist_head *lldeferred_pios) { struct ploop_rq *prq = pio->endio_cb_data; struct request *rq = prq->rq; struct bio_vec *bvec = NULL; - LIST_HEAD(list); int ret; if (rq->bio != rq->biotail) { @@ -1656,16 +1659,18 @@ static void ploop_prepare_one_embedded_pio(struct ploop *ploop, pio->bi_iter.bi_idx = 0; pio->bi_iter.bi_bvec_done = 0; } else { - /* Single bio already provides bvec array */ + /* Single bio already provides bvec array + * bvec is updated to the correct on submit + * it is different after partial IO + */ bvec = rq->bio->bi_io_vec; - pio->bi_iter = rq->bio->bi_iter; } pio->bi_iter.bi_sector = ploop_rq_pos(ploop, rq); pio->bi_io_vec = bvec; pio->queue_list_id = PLOOP_LIST_DEFERRED; - ret = ploop_split_pio_to_list(ploop, pio, deferred_pios); + ret = ploop_split_pio_to_list(ploop, pio, lldeferred_pios); if (ret) goto err_nomem; @@ -1677,7 +1682,7 @@ static void ploop_prepare_one_embedded_pio(struct ploop *ploop, static void ploop_prepare_embedded_pios(struct ploop *ploop, struct llist_node *pios, - struct list_head *deferred_pios) + struct llist_head *deferred_pios) { struct pio *pio; struct llist_node *pos, *t; @@ -1694,12 +1699,17 @@ static void ploop_prepare_embedded_pios(struct ploop *ploop, } static void ploop_process_deferred_pios(struct ploop *ploop, - struct list_head *pios) + struct llist_head *pios) { struct pio *pio; - while ((pio = ploop_pio_list_pop(pios)) != NULL) + struct llist_node *pos, *t; + + llist_for_each_safe(pos, t, pios->first) { + pio = list_entry((struct list_head *)pos, typeof(*pio), list); + INIT_LIST_HEAD(&pio->list); /* until type is changed */ ploop_process_one_deferred_bio(ploop, pio); + } } static void ploop_process_one_discard_pio(struct ploop *ploop, struct pio *pio) @@ -1798,19 +1808,13 @@ static void ploop_submit_metadata_writeback(struct ploop *ploop) } } -static void process_ploop_fsync_work(struct ploop *ploop) +static void process_ploop_fsync_work(struct ploop *ploop, struct llist_node *llflush_pios) { struct file *file; struct pio *pio; int ret; - struct llist_node *llflush_pios; struct llist_node *pos, *t; - llflush_pios = llist_del_all(&ploop->pios[PLOOP_LIST_FLUSH]); - - if (!llflush_pios) - return; - file = ploop_top_delta(ploop)->file; /* All flushes are done as one */ ret = vfs_fsync(file, 0); @@ -1829,12 +1833,13 @@ static void process_ploop_fsync_work(struct ploop *ploop) void do_ploop_run_work(struct ploop *ploop) { - LIST_HEAD(deferred_pios); + LLIST_HEAD(deferred_pios); struct llist_node *llembedded_pios; struct llist_node *lldeferred_pios; struct llist_node *lldiscard_pios; struct llist_node *llcow_pios; struct llist_node *llresubmit; + struct llist_node *llflush_pios; unsigned int old_flags = current->flags; current->flags |= PF_IO_THREAD|PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO; @@ -1842,24 +1847,23 @@ void do_ploop_run_work(struct ploop *ploop) llembedded_pios = llist_del_all(&ploop->pios[PLOOP_LIST_PREPARE]); lldeferred_pios = llist_del_all(&ploop->pios[PLOOP_LIST_DEFERRED]); + llresubmit = llist_del_all(&ploop->llresubmit_pios); lldiscard_pios = llist_del_all(&ploop->pios[PLOOP_LIST_DISCARD]); llcow_pios = llist_del_all(&ploop->pios[PLOOP_LIST_COW]); - llresubmit = llist_del_all(&ploop->llresubmit_pios); /* add old deferred back to the list */ if (lldeferred_pios) { struct llist_node *pos, *t; - struct pio *pio; - + /* Add one by one we need last for batch add */ llist_for_each_safe(pos, t, lldeferred_pios) { - pio = list_entry((struct list_head *)pos, typeof(*pio), list); - INIT_LIST_HEAD(&pio->list); - list_add(&pio->list, &deferred_pios); + llist_add(pos, &deferred_pios); } } ploop_prepare_embedded_pios(ploop, llembedded_pios, &deferred_pios); + llflush_pios = llist_del_all(&ploop->pios[PLOOP_LIST_FLUSH]); + if (llresubmit) ploop_process_resubmit_pios(ploop, llist_reverse_order(llresubmit)); @@ -1873,8 +1877,9 @@ void do_ploop_run_work(struct ploop *ploop) ploop_submit_metadata_writeback(ploop); - if (!llist_empty(&ploop->pios[PLOOP_LIST_FLUSH])) - process_ploop_fsync_work(ploop); + if (llflush_pios) + process_ploop_fsync_work(ploop, llist_reverse_order(llflush_pios)); + current->flags = old_flags; } @@ -1917,6 +1922,9 @@ static void ploop_submit_embedded_pio(struct ploop *ploop, struct pio *pio) { struct ploop_rq *prq = pio->endio_cb_data; struct request *rq = prq->rq; + LLIST_HEAD(deferred_pios); + struct pio *spio; + struct llist_node *pos, *t; if (blk_rq_bytes(rq)) { pio->queue_list_id = PLOOP_LIST_PREPARE; @@ -1931,17 +1939,26 @@ static void ploop_submit_embedded_pio(struct ploop *ploop, struct pio *pio) } ploop_inc_nr_inflight(ploop, pio); - llist_add((struct llist_node *)(&pio->list), &ploop->pios[PLOOP_LIST_PREPARE]); + ploop_prepare_one_embedded_pio(ploop, pio, &deferred_pios); + + llist_for_each_safe(pos, t, deferred_pios.first) { + spio = list_entry((struct list_head *)pos, typeof(*pio), list); + llist_add((struct llist_node *)(&spio->list), &ploop->pios[PLOOP_LIST_DEFERRED]); + } ploop_schedule_work(ploop); } -void ploop_submit_embedded_pios(struct ploop *ploop, struct list_head *list) +void ploop_submit_embedded_pios(struct ploop *ploop, struct llist_node *list) { struct pio *pio; + struct llist_node *pos, *t; - while ((pio = ploop_pio_list_pop(list)) != NULL) + llist_for_each_safe(pos, t, list) { + pio = list_entry((struct list_head *)pos, typeof(*pio), list); + INIT_LIST_HEAD(&pio->list); ploop_submit_embedded_pio(ploop, pio); + } } int ploop_clone_and_map(struct dm_target *ti, struct request *rq, diff --git a/drivers/md/dm-ploop.h b/drivers/md/dm-ploop.h index 64d8eac4ef84..1e5927365d81 100644 --- a/drivers/md/dm-ploop.h +++ b/drivers/md/dm-ploop.h @@ -569,7 +569,7 @@ extern int ploop_add_delta(struct ploop *ploop, u32 level, extern int ploop_check_delta_length(struct ploop *ploop, struct file *file, loff_t *file_size); extern void ploop_submit_embedded_pios(struct ploop *ploop, - struct list_head *list); + struct llist_node *list); extern void ploop_dispatch_pios(struct ploop *ploop, struct pio *pio, struct list_head *pio_list); extern void do_ploop_work(struct work_struct *ws); -- 2.43.0 _______________________________________________ Devel mailing list Devel@openvz.org https://lists.openvz.org/mailman/listinfo/devel
