The branch main has been updated by np:
URL:
https://cgit.FreeBSD.org/src/commit/?id=9ba8670a8b175de79ea087688f51595b4f2db862
commit 9ba8670a8b175de79ea087688f51595b4f2db862
Author: Navdeep Parhar <n...@freebsd.org>
AuthorDate: 2024-09-29 23:29:41 +0000
Commit: Navdeep Parhar <n...@freebsd.org>
CommitDate: 2024-09-30 00:38:11 +0000
cxgbe(4): Allow t4_tom to be unloaded safely.
* Disable IFCAP_TOE automatically on all ifnets on all adapters during
unload. This is user-friendly and avoids panics due to stale ifnet
state after t4_tom is unloaded.
* Do not allow unload if tids are in use by the TOE on any adapter.
Reported by: Bimal Abraham @ Chelsio
MFC after: 1 week
Sponsored by: Chelsio Communications
---
sys/dev/cxgbe/adapter.h | 1 +
sys/dev/cxgbe/t4_main.c | 22 +++++++++++-----------
sys/dev/cxgbe/tom/t4_tom.c | 31 +++++++++++++++++++++++--------
3 files changed, 35 insertions(+), 19 deletions(-)
diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 2769277e8411..862a90e8a441 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -1394,6 +1394,7 @@ void cxgbe_media_status(if_t, struct ifmediareq *);
void t4_os_cim_err(struct adapter *);
int suspend_adapter(struct adapter *);
int resume_adapter(struct adapter *);
+int toe_capability(struct vi_info *, bool);
#ifdef KERN_TLS
/* t6_kern_tls.c */
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 307b9c27111a..508f3f08f22a 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -869,7 +869,6 @@ static int stop_lld(struct adapter *);
static inline int restart_adapter(struct adapter *);
static int restart_lld(struct adapter *);
#ifdef TCP_OFFLOAD
-static int toe_capability(struct vi_info *, bool);
static int deactivate_all_uld(struct adapter *);
static void stop_all_uld(struct adapter *);
static void restart_all_uld(struct adapter *);
@@ -12386,7 +12385,7 @@ t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t
data, int fflag,
}
#ifdef TCP_OFFLOAD
-static int
+int
toe_capability(struct vi_info *vi, bool enable)
{
int rc;
@@ -12452,6 +12451,7 @@ toe_capability(struct vi_info *vi, bool enable)
if (isset(&sc->offload_map, pi->port_id)) {
/* TOE is enabled on another VI of this port. */
+ MPASS(pi->uld_vis > 0);
pi->uld_vis++;
return (0);
}
@@ -12477,17 +12477,17 @@ toe_capability(struct vi_info *vi, bool enable)
if (!uld_active(sc, ULD_ISCSI))
(void) t4_activate_uld(sc, ULD_ISCSI);
- pi->uld_vis++;
- setbit(&sc->offload_map, pi->port_id);
+ if (pi->uld_vis++ == 0)
+ setbit(&sc->offload_map, pi->port_id);
} else {
- pi->uld_vis--;
-
- if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0)
+ if ((if_getcapenable(vi->ifp) & IFCAP_TOE) == 0) {
+ /* TOE is already disabled. */
return (0);
-
- KASSERT(uld_active(sc, ULD_TOM),
- ("%s: TOM never initialized?", __func__));
- clrbit(&sc->offload_map, pi->port_id);
+ }
+ MPASS(isset(&sc->offload_map, pi->port_id));
+ MPASS(pi->uld_vis > 0);
+ if (--pi->uld_vis == 0)
+ clrbit(&sc->offload_map, pi->port_id);
}
return (0);
diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c
index d5b6b6ee2ce0..97cb380d0e71 100644
--- a/sys/dev/cxgbe/tom/t4_tom.c
+++ b/sys/dev/cxgbe/tom/t4_tom.c
@@ -1946,20 +1946,29 @@ done:
static int
t4_tom_deactivate(struct adapter *sc)
{
- int rc = 0;
+ int rc = 0, i, v;
struct tom_data *td = sc->tom_softc;
+ struct vi_info *vi;
ASSERT_SYNCHRONIZED_OP(sc);
if (td == NULL)
return (0); /* XXX. KASSERT? */
- if (sc->offload_map != 0)
- return (EBUSY); /* at least one port has IFCAP_TOE enabled */
-
if (uld_active(sc, ULD_IWARP) || uld_active(sc, ULD_ISCSI))
return (EBUSY); /* both iWARP and iSCSI rely on the TOE. */
+ if (sc->offload_map != 0) {
+ for_each_port(sc, i) {
+ for_each_vi(sc->port[i], v, vi) {
+ toe_capability(vi, false);
+ if_setcapenablebit(vi->ifp, 0, IFCAP_TOE);
+ SETTOEDEV(vi->ifp, NULL);
+ }
+ }
+ MPASS(sc->offload_map == 0);
+ }
+
mtx_lock(&td->toep_list_lock);
if (!TAILQ_EMPTY(&td->toep_list))
rc = EBUSY;
@@ -2243,14 +2252,16 @@ t4_tom_mod_load(void)
}
static void
-tom_uninit(struct adapter *sc, void *arg __unused)
+tom_uninit(struct adapter *sc, void *arg)
{
+ bool *ok_to_unload = arg;
+
if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4tomun"))
return;
/* Try to free resources (works only if no port has IFCAP_TOE) */
- if (uld_active(sc, ULD_TOM))
- t4_deactivate_uld(sc, ULD_TOM);
+ if (uld_active(sc, ULD_TOM) && t4_deactivate_uld(sc, ULD_TOM) != 0)
+ *ok_to_unload = false;
end_synchronized_op(sc, 0);
}
@@ -2258,7 +2269,11 @@ tom_uninit(struct adapter *sc, void *arg __unused)
static int
t4_tom_mod_unload(void)
{
- t4_iterate(tom_uninit, NULL);
+ bool ok_to_unload = true;
+
+ t4_iterate(tom_uninit, &ok_to_unload);
+ if (!ok_to_unload)
+ return (EBUSY);
if (t4_unregister_uld(&tom_uld_info, ULD_TOM) == EBUSY)
return (EBUSY);
