pios can get reordered, fix in the next commit.
https://virtuozzo.atlassian.net/browse/VSTOR-91820
Signed-off-by: Alexander Atanasov <alexander.atana...@virtuozzo.com>
---
drivers/md/dm-ploop-map.c | 119 ++++++++++++++++++++++-------------
drivers/md/dm-ploop-target.c | 6 +-
drivers/md/dm-ploop.h | 4 +-
3 files changed, 79 insertions(+), 50 deletions(-)
diff --git a/drivers/md/dm-ploop-map.c b/drivers/md/dm-ploop-map.c
index d93e96857d02..6b929c478dd0 100644
--- a/drivers/md/dm-ploop-map.c
+++ b/drivers/md/dm-ploop-map.c
@@ -340,8 +340,7 @@ ALLOW_ERROR_INJECTION(ploop_split_pio_to_list, ERRNO);
static void ploop_dispatch_pio(struct ploop *ploop, struct pio *pio,
bool *is_data, bool *is_flush)
{
- struct list_head *list = &ploop->pios[pio->queue_list_id];
- unsigned long flags;
+ struct llist_head *list = (struct llist_head
*)&ploop->pios[pio->queue_list_id];
lockdep_assert_not_held(&ploop->deferred_lock);
WARN_ON_ONCE(pio->queue_list_id >= PLOOP_LIST_COUNT);
@@ -351,9 +350,7 @@ static void ploop_dispatch_pio(struct ploop *ploop, struct
pio *pio,
else
*is_data = true;
- spin_lock_irqsave(&ploop->deferred_lock, flags);
- list_add_tail(&pio->list, list);
- spin_unlock_irqrestore(&ploop->deferred_lock, flags);
+ llist_add((struct llist_node *)(&pio->list), list);
}
void ploop_dispatch_pios(struct ploop *ploop, struct pio *pio,
@@ -812,7 +809,7 @@ static void ploop_advance_local_after_bat_wb(struct ploop
*ploop,
wait_llist_pending = llist_reverse_order(wait_llist_pending);
llist_for_each_safe(pos, t, wait_llist_pending) {
pio = list_entry((struct list_head *)pos, typeof(*pio),
list);
- list_add(&pio->list, &list);
+ list_add_tail(&pio->list, &list);
}
}
@@ -1148,13 +1145,10 @@ static void ploop_attach_end_action(struct pio *pio,
struct ploop_index_wb *piwb
static void ploop_queue_resubmit(struct pio *pio)
{
struct ploop *ploop = pio->ploop;
- unsigned long flags;
pio->queue_list_id = PLOOP_LIST_INVALID;
- spin_lock_irqsave(&ploop->deferred_lock, flags);
- list_add_tail(&pio->list, &ploop->resubmit_pios);
- spin_unlock_irqrestore(&ploop->deferred_lock, flags);
+ llist_add((struct llist_node *)(&pio->list), &ploop->llresubmit_pios);
queue_work(ploop->wq, &ploop->worker);
}
@@ -1409,20 +1403,21 @@ static void ploop_submit_cow_index_wb(struct ploop_cow
*cow)
}
static void ploop_process_delta_cow(struct ploop *ploop,
- struct list_head *cow_list)
+ struct llist_node *cow_llist)
{
struct ploop_cow *cow;
struct pio *aux_pio;
- if (list_empty(cow_list))
- return;
-
- while ((aux_pio = ploop_pio_list_pop(cow_list)) != NULL) {
+ struct llist_node *pos, *t;
+ llist_for_each_safe(pos, t, cow_llist) {
+ aux_pio = list_entry((struct list_head *)pos, typeof(*aux_pio),
list);
cow = aux_pio->endio_cb_data;
if (unlikely(aux_pio->bi_status != BLK_STS_OK)) {
ploop_complete_cow(cow, aux_pio->bi_status);
continue;
}
+ /* until type is changed */
+ INIT_LIST_HEAD(&aux_pio->list);
if (cow->dst_clu == BAT_ENTRY_NONE) {
/*
@@ -1678,13 +1673,17 @@ static void ploop_prepare_one_embedded_pio(struct ploop
*ploop,
}
static void ploop_prepare_embedded_pios(struct ploop *ploop,
- struct list_head *pios,
+ struct llist_node *pios,
struct list_head *deferred_pios)
{
struct pio *pio;
+ struct llist_node *pos, *t;
- while ((pio = ploop_pio_list_pop(pios)) != NULL)
+ llist_for_each_safe(pos, t, pios) {
+ pio = list_entry((struct list_head *)pos, typeof(*pio), list);
+ INIT_LIST_HEAD(&pio->list); /* until type is changed */
ploop_prepare_one_embedded_pio(ploop, pio, deferred_pios);
+ }
}
static void ploop_process_deferred_pios(struct ploop *ploop,
@@ -1746,21 +1745,28 @@ static void ploop_process_one_discard_pio(struct ploop
*ploop, struct pio *pio)
}
static void ploop_process_discard_pios(struct ploop *ploop,
- struct list_head *pios)
+ struct llist_node *discpios)
{
struct pio *pio;
+ struct llist_node *pos, *t;
- while ((pio = ploop_pio_list_pop(pios)) != NULL)
+ llist_for_each_safe(pos, t, discpios) {
+ pio = list_entry((struct list_head *)pos, typeof(*pio), list);
+ INIT_LIST_HEAD(&pio->list);
ploop_process_one_discard_pio(ploop, pio);
+ }
}
static void ploop_process_resubmit_pios(struct ploop *ploop,
- struct list_head *pios)
+ struct llist_node *llpios)
{
struct pio *pio;
+ struct llist_node *pos, *t;
- while ((pio = ploop_pio_list_pop(pios)) != NULL) {
+ llist_for_each_safe(pos, t, llpios) {
+ pio = list_entry((struct list_head *)pos, typeof(*pio), list);
pio->queue_list_id = PLOOP_LIST_INVALID;
+ INIT_LIST_HEAD(&pio->list); /* Until type is changed */
ploop_submit_rw_mapped(ploop, pio);
}
}
@@ -1785,18 +1791,21 @@ static void ploop_submit_metadata_writeback(struct
ploop *ploop)
static void process_ploop_fsync_work(struct ploop *ploop)
{
- LIST_HEAD(flush_pios);
struct file *file;
struct pio *pio;
int ret;
- spin_lock_irq(&ploop->deferred_lock);
- list_splice_init(&ploop->pios[PLOOP_LIST_FLUSH], &flush_pios);
- spin_unlock_irq(&ploop->deferred_lock);
+ struct llist_node *llflush_pios;
+ struct llist_node *pos, *t;
+
+ llflush_pios = llist_del_all(&ploop->pios[PLOOP_LIST_FLUSH]);
file = ploop_top_delta(ploop)->file;
+ /* All flushes are done as one */
ret = vfs_fsync(file, 0);
- while ((pio = ploop_pio_list_pop(&flush_pios)) != NULL) {
+ llist_for_each_safe(pos, t, llflush_pios) {
+ pio = list_entry((struct list_head *)pos, typeof(*pio), list);
+ INIT_LIST_HEAD(&pio->list); /* Until type is changed */
if (unlikely(ret)) {
pio->bi_status = errno_to_blk_status(ret);
if (static_branch_unlikely(&ploop_standby_check))
@@ -1809,32 +1818,54 @@ static void process_ploop_fsync_work(struct ploop
*ploop)
void do_ploop_work(struct work_struct *ws)
{
struct ploop *ploop = container_of(ws, struct ploop, worker);
- LIST_HEAD(embedded_pios);
LIST_HEAD(deferred_pios);
- LIST_HEAD(discard_pios);
- LIST_HEAD(cow_pios);
- LIST_HEAD(resubmit_pios);
+ struct llist_node *llembedded_pios;
+ struct llist_node *lldeferred_pios;
+ struct llist_node *lldiscard_pios;
+ struct llist_node *llcow_pios;
+ struct llist_node *llresubmit;
bool do_fsync = false;
unsigned int old_flags = current->flags;
current->flags |= PF_IO_THREAD|PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO;
spin_lock_irq(&ploop->deferred_lock);
- list_splice_init(&ploop->pios[PLOOP_LIST_PREPARE], &embedded_pios);
- list_splice_init(&ploop->pios[PLOOP_LIST_DEFERRED], &deferred_pios);
- list_splice_init(&ploop->pios[PLOOP_LIST_DISCARD], &discard_pios);
- list_splice_init(&ploop->pios[PLOOP_LIST_COW], &cow_pios);
- list_splice_init(&ploop->resubmit_pios, &resubmit_pios);
- if (!list_empty(&ploop->pios[PLOOP_LIST_FLUSH]))
+ llembedded_pios = llist_del_all(&ploop->pios[PLOOP_LIST_PREPARE]);
+ lldeferred_pios = llist_del_all(&ploop->pios[PLOOP_LIST_DEFERRED]);
+ lldiscard_pios = llist_del_all(&ploop->pios[PLOOP_LIST_DISCARD]);
+ llcow_pios = llist_del_all(&ploop->pios[PLOOP_LIST_COW]);
+ llresubmit = llist_del_all(&ploop->llresubmit_pios);
+
+ if (!llist_empty(&ploop->pios[PLOOP_LIST_FLUSH]))
do_fsync = true;
+
spin_unlock_irq(&ploop->deferred_lock);
- ploop_prepare_embedded_pios(ploop, &embedded_pios, &deferred_pios);
+ /* add old deferred to the list */
+ if (lldeferred_pios) {
+ struct llist_node *pos, *t;
+ struct pio *pio;
+
+ llist_for_each_safe(pos, t,
llist_reverse_order(lldeferred_pios)) {
+ pio = list_entry((struct list_head *)pos, typeof(*pio),
list);
+ INIT_LIST_HEAD(&pio->list);
+ list_add_tail(&pio->list, &deferred_pios);
+ }
+ }
+
+ ploop_prepare_embedded_pios(ploop, llembedded_pios, &deferred_pios);
+
+ if (llresubmit)
+ ploop_process_resubmit_pios(ploop,
llist_reverse_order(llresubmit));
+
- ploop_process_resubmit_pios(ploop, &resubmit_pios);
ploop_process_deferred_pios(ploop, &deferred_pios);
- ploop_process_discard_pios(ploop, &discard_pios);
- ploop_process_delta_cow(ploop, &cow_pios);
+
+ if (lldiscard_pios)
+ ploop_process_discard_pios(ploop,
llist_reverse_order(lldiscard_pios));
+
+ if (llcow_pios)
+ ploop_process_delta_cow(ploop, llist_reverse_order(llcow_pios));
ploop_submit_metadata_writeback(ploop);
@@ -1868,18 +1899,16 @@ static void ploop_submit_embedded_pio(struct ploop
*ploop, struct pio *pio)
worker = &ploop->fsync_worker;
}
- spin_lock_irqsave(&ploop->deferred_lock, flags);
if (unlikely(ploop->stop_submitting_pios)) {
llist_add((struct llist_node *)(&pio->list),
&ploop->llsuspended_pios);
queue = false;
- goto unlock;
+ goto out;
}
ploop_inc_nr_inflight(ploop, pio);
- list_add_tail(&pio->list, &ploop->pios[pio->queue_list_id]);
-unlock:
- spin_unlock_irqrestore(&ploop->deferred_lock, flags);
+ llist_add((struct llist_node *)(&pio->list),
&ploop->pios[pio->queue_list_id]);
+out:
if (queue)
queue_work(ploop->wq, worker);
}
diff --git a/drivers/md/dm-ploop-target.c b/drivers/md/dm-ploop-target.c
index 94d6f661f50c..f12c6912f8d0 100644
--- a/drivers/md/dm-ploop-target.c
+++ b/drivers/md/dm-ploop-target.c
@@ -140,7 +140,7 @@ static bool ploop_has_pending_activity(struct ploop *ploop)
spin_lock_irq(&ploop->deferred_lock);
for (i = 0; i < PLOOP_LIST_COUNT; i++)
- has |= !list_empty(&ploop->pios[i]);
+ has |= !llist_empty(&ploop->pios[i]);
spin_unlock_irq(&ploop->deferred_lock);
return has;
@@ -373,9 +373,9 @@ static int ploop_ctr(struct dm_target *ti, unsigned int
argc, char **argv)
init_llist_head(&ploop->llsuspended_pios);
for (i = 0; i < PLOOP_LIST_COUNT; i++)
- INIT_LIST_HEAD(&ploop->pios[i]);
+ init_llist_head(&ploop->pios[i]);
- INIT_LIST_HEAD(&ploop->resubmit_pios);
+ init_llist_head(&ploop->llresubmit_pios);
init_llist_head(&ploop->enospc_pios);
INIT_LIST_HEAD(&ploop->cluster_lk_list);
diff --git a/drivers/md/dm-ploop.h b/drivers/md/dm-ploop.h
index 7e7f82aa1402..66c7ce20dd60 100644
--- a/drivers/md/dm-ploop.h
+++ b/drivers/md/dm-ploop.h
@@ -196,9 +196,9 @@ struct ploop {
spinlock_t inflight_lock;
spinlock_t deferred_lock;
- struct list_head pios[PLOOP_LIST_COUNT];
+ struct llist_head pios[PLOOP_LIST_COUNT];
- struct list_head resubmit_pios; /* After partial IO */
+ struct llist_head llresubmit_pios; /* After partial IO */
struct llist_head enospc_pios; /* Delayed after ENOSPC */
atomic_t service_pios;
--
2.43.0
_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel
