Author: cem Date: Tue Jul 5 20:51:52 2016 New Revision: 302352 URL: https://svnweb.freebsd.org/changeset/base/302352
Log: ioat(4): Split timer into poll and shrink functions Poll should happen quickly, while shrink should happen infrequently. Protect is_completion_pending with submit_lock. Reviewed by: markj Approved by: re Sponsored by: EMC / Isilon Storage Division Differential Revision: https://reviews.freebsd.org/D7097 Modified: head/sys/dev/ioat/ioat.c head/sys/dev/ioat/ioat_internal.h Modified: head/sys/dev/ioat/ioat.c ============================================================================== --- head/sys/dev/ioat/ioat.c Tue Jul 5 20:48:59 2016 (r302351) +++ head/sys/dev/ioat/ioat.c Tue Jul 5 20:51:52 2016 (r302352) @@ -61,8 +61,8 @@ __FBSDID("$FreeBSD$"); #ifndef BUS_SPACE_MAXADDR_40BIT #define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL #endif -#define IOAT_INTR_TIMO (hz / 10) #define IOAT_REFLK (&ioat->submit_lock) +#define IOAT_SHRINK_PERIOD (10 * hz) static int ioat_probe(device_t device); static int ioat_attach(device_t device); @@ -96,7 +96,8 @@ static int ring_grow(struct ioat_softc * static int ring_shrink(struct ioat_softc *, uint32_t oldorder, struct ioat_descriptor **); static void ioat_halted_debug(struct ioat_softc *, uint32_t); -static void ioat_timer_callback(void *arg); +static void ioat_poll_timer_callback(void *arg); +static void ioat_shrink_timer_callback(void *arg); static void dump_descriptor(void *hw_desc); static void ioat_submit_single(struct ioat_softc *ioat); static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, @@ -332,7 +333,8 @@ ioat_detach(device_t device) mtx_unlock(IOAT_REFLK); ioat_teardown_intr(ioat); - callout_drain(&ioat->timer); + callout_drain(&ioat->poll_timer); + callout_drain(&ioat->shrink_timer); pci_disable_busmaster(device); @@ -428,7 +430,8 @@ ioat3_attach(device_t device) mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF); - callout_init(&ioat->timer, 1); + callout_init(&ioat->poll_timer, 1); + callout_init(&ioat->shrink_timer, 1); TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat); /* Establish lock order for Witness */ @@ -634,6 +637,7 @@ ioat_process_events(struct ioat_softc *i struct bus_dmadesc *dmadesc; uint64_t comp_update, status; uint32_t completed, chanerr; + boolean_t pending; int error; mtx_lock(&ioat->cleanup_lock); @@ -668,19 +672,33 @@ ioat_process_events(struct ioat_softc *i } ioat->last_seen = desc->hw_desc_bus_addr; - - if (ioat->head == ioat->tail) { - ioat->is_completion_pending = FALSE; - callout_reset(&ioat->timer, IOAT_INTR_TIMO, - ioat_timer_callback, ioat); - } - ioat->stats.descriptors_processed += completed; out: ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); + + /* Perform a racy check first; only take the locks if it passes. */ + pending = (ioat_get_active(ioat) != 0); + if (!pending && ioat->is_completion_pending) { + mtx_unlock(&ioat->cleanup_lock); + mtx_lock(&ioat->submit_lock); + mtx_lock(&ioat->cleanup_lock); + + pending = (ioat_get_active(ioat) != 0); + if (!pending && ioat->is_completion_pending) { + ioat->is_completion_pending = FALSE; + callout_reset(&ioat->shrink_timer, IOAT_SHRINK_PERIOD, + ioat_shrink_timer_callback, ioat); + callout_stop(&ioat->poll_timer); + } + mtx_unlock(&ioat->submit_lock); + } mtx_unlock(&ioat->cleanup_lock); + if (pending) + callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, + ioat); + if (completed != 0) { ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF); wakeup(&ioat->tail); @@ -1602,7 +1620,18 @@ ioat_halted_debug(struct ioat_softc *ioa } static void -ioat_timer_callback(void *arg) +ioat_poll_timer_callback(void *arg) +{ + struct ioat_softc *ioat; + + ioat = arg; + ioat_log_message(3, "%s\n", __func__); + + ioat_process_events(ioat); +} + +static void +ioat_shrink_timer_callback(void *arg) { struct ioat_descriptor **newring; struct ioat_softc *ioat; @@ -1611,11 +1640,6 @@ ioat_timer_callback(void *arg) ioat = arg; ioat_log_message(1, "%s\n", __func__); - if (ioat->is_completion_pending) { - ioat_process_events(ioat); - return; - } - /* Slowly scale the ring down if idle. */ mtx_lock(&ioat->submit_lock); order = ioat->ring_size_order; @@ -1641,8 +1665,8 @@ ioat_timer_callback(void *arg) out: if (ioat->ring_size_order > IOAT_MIN_ORDER) - callout_reset(&ioat->timer, 10 * hz, - ioat_timer_callback, ioat); + callout_reset(&ioat->poll_timer, IOAT_SHRINK_PERIOD, + ioat_shrink_timer_callback, ioat); } /* @@ -1658,8 +1682,9 @@ ioat_submit_single(struct ioat_softc *io if (!ioat->is_completion_pending) { ioat->is_completion_pending = TRUE; - callout_reset(&ioat->timer, IOAT_INTR_TIMO, - ioat_timer_callback, ioat); + callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, + ioat); + callout_stop(&ioat->shrink_timer); } ioat->stats.descriptors_submitted++; @@ -2126,12 +2151,19 @@ DB_SHOW_COMMAND(ioat, db_show_ioat) db_printf(" cached_intrdelay: %u\n", sc->cached_intrdelay); db_printf(" *comp_update: 0x%jx\n", (uintmax_t)*sc->comp_update); - db_printf(" timer:\n"); - db_printf(" c_time: %ju\n", (uintmax_t)sc->timer.c_time); - db_printf(" c_arg: %p\n", sc->timer.c_arg); - db_printf(" c_func: %p\n", sc->timer.c_func); - db_printf(" c_lock: %p\n", sc->timer.c_lock); - db_printf(" c_flags: 0x%x\n", (unsigned)sc->timer.c_flags); + db_printf(" poll_timer:\n"); + db_printf(" c_time: %ju\n", (uintmax_t)sc->poll_timer.c_time); + db_printf(" c_arg: %p\n", sc->poll_timer.c_arg); + db_printf(" c_func: %p\n", sc->poll_timer.c_func); + db_printf(" c_lock: %p\n", sc->poll_timer.c_lock); + db_printf(" c_flags: 0x%x\n", (unsigned)sc->poll_timer.c_flags); + + db_printf(" shrink_timer:\n"); + db_printf(" c_time: %ju\n", (uintmax_t)sc->shrink_timer.c_time); + db_printf(" c_arg: %p\n", sc->shrink_timer.c_arg); + db_printf(" c_func: %p\n", sc->shrink_timer.c_func); + db_printf(" c_lock: %p\n", sc->shrink_timer.c_lock); + db_printf(" c_flags: 0x%x\n", (unsigned)sc->shrink_timer.c_flags); db_printf(" quiescing: %d\n", (int)sc->quiescing); db_printf(" destroying: %d\n", (int)sc->destroying); Modified: head/sys/dev/ioat/ioat_internal.h ============================================================================== --- head/sys/dev/ioat/ioat_internal.h Tue Jul 5 20:48:59 2016 (r302351) +++ head/sys/dev/ioat/ioat_internal.h Tue Jul 5 20:51:52 2016 (r302352) @@ -480,13 +480,14 @@ struct ioat_softc { uint64_t *comp_update; bus_addr_t comp_update_bus_addr; - struct callout timer; + struct callout poll_timer; + struct callout shrink_timer; struct task reset_task; boolean_t quiescing; boolean_t destroying; boolean_t is_resize_pending; - boolean_t is_completion_pending; + boolean_t is_completion_pending; /* submit_lock */ boolean_t is_reset_pending; boolean_t is_channel_running; boolean_t intrdelay_supported; _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
