The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=92e40a9b9241313b3d16543819ccd8d642bb11a5
commit 92e40a9b9241313b3d16543819ccd8d642bb11a5 Author: John Baldwin <[email protected]> AuthorDate: 2022-04-21 19:01:55 +0000 Commit: John Baldwin <[email protected]> CommitDate: 2022-04-21 19:01:55 +0000 busdma_bounce: Batch bounce page free operations when possible. Reviewed by: imp Differential Revision: https://reviews.freebsd.org/D34968 --- sys/kern/subr_busdma_bounce.c | 69 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/sys/kern/subr_busdma_bounce.c b/sys/kern/subr_busdma_bounce.c index 243da8e9487f..f3699cf2ad27 100644 --- a/sys/kern/subr_busdma_bounce.c +++ b/sys/kern/subr_busdma_bounce.c @@ -382,55 +382,54 @@ add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, } static void -free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) +free_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map) { - struct bus_dmamap *map; + struct bounce_page *bpage; struct bounce_zone *bz; bool schedule_thread; + u_int count; + + if (STAILQ_EMPTY(&map->bpages)) + return; bz = dmat->bounce_zone; - bpage->datavaddr = 0; - bpage->datacount = 0; - if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) { - /* - * Reset the bounce page to start at offset 0. Other uses - * of this bounce page may need to store a full page of - * data and/or assume it starts on a page boundary. - */ - bpage->vaddr &= ~PAGE_MASK; - bpage->busaddr &= ~PAGE_MASK; + count = 0; + schedule_thread = false; + STAILQ_FOREACH(bpage, &map->bpages, links) { + bpage->datavaddr = 0; + bpage->datacount = 0; + + if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) { + /* + * Reset the bounce page to start at offset 0. + * Other uses of this bounce page may need to + * store a full page of data and/or assume it + * starts on a page boundary. + */ + bpage->vaddr &= ~PAGE_MASK; + bpage->busaddr &= ~PAGE_MASK; + } + count++; } - schedule_thread = false; mtx_lock(&bounce_lock); - STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); - bz->free_bpages++; - bz->active_bpages--; - if ((map = STAILQ_FIRST(&bz->bounce_map_waitinglist)) != NULL) { - if (reserve_bounce_pages(map->dmat, map, 1) == 0) { - STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links); - STAILQ_INSERT_TAIL(&bounce_map_callbacklist, - map, links); - bz->total_deferred++; - schedule_thread = true; - } + STAILQ_CONCAT(&bz->bounce_page_list, &map->bpages); + bz->free_bpages += count; + bz->active_bpages -= count; + while ((map = STAILQ_FIRST(&bz->bounce_map_waitinglist)) != NULL) { + if (reserve_bounce_pages(map->dmat, map, 1) != 0) + break; + + STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links); + STAILQ_INSERT_TAIL(&bounce_map_callbacklist, map, links); + bz->total_deferred++; + schedule_thread = true; } mtx_unlock(&bounce_lock); if (schedule_thread) wakeup(&bounce_map_callbacklist); } -static void -free_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map) -{ - struct bounce_page *bpage; - - while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { - STAILQ_REMOVE_HEAD(&map->bpages, links); - free_bounce_page(dmat, bpage); - } -} - static void busdma_thread(void *dummy __unused) {
