The branch main has been updated by kib:
URL:
https://cgit.FreeBSD.org/src/commit/?id=614087c65e997fcdedcd60f368a035a4b09d106d
commit 614087c65e997fcdedcd60f368a035a4b09d106d
Author: Konstantin Belousov <k...@freebsd.org>
AuthorDate: 2025-06-24 02:51:52 +0000
Commit: Konstantin Belousov <k...@freebsd.org>
CommitDate: 2025-06-24 15:58:53 +0000
sysctl net.inet.tcp.ktlslist: do not rely on global generation for ktls
sessions
Disallow parallel executions for the sysctl, which makes it possible to
have the generation count for the sysctl requests itself instead of for the
sessions. When the first pass over inpcbs is done, assign them the
request' gen count. On the second pass, only externalize the inpcbs
with ktls sessions which gen count is equal to the current request.
This way, we can be sure that the second pass does not copy out more
inpcbs than was counted for in the first pass, while eliminating the
global atomic op during ktls session creation.
Requested by: gallatin
Reviewed by: gallatin, markj
Sponsored by: NVidia networking
Differential revision: https://reviews.freebsd.org/D51000
---
sys/kern/uipc_ktls.c | 10 ++--------
sys/netinet/tcp_subr.c | 41 +++++++++++++++++++++++++++++------------
sys/sys/ktls.h | 7 -------
3 files changed, 31 insertions(+), 27 deletions(-)
diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c
index 1cbaa7db2e84..ce09042abdac 100644
--- a/sys/kern/uipc_ktls.c
+++ b/sys/kern/uipc_ktls.c
@@ -595,8 +595,6 @@ start:
return (error);
}
-uint64_t ktls_glob_gen = 1;
-
static int
ktls_create_session(struct socket *so, struct tls_enable *en,
struct ktls_session **tlsp, int direction)
@@ -821,8 +819,7 @@ ktls_create_session(struct socket *so, struct tls_enable
*en,
arc4rand(tls->params.iv + 8, sizeof(uint64_t), 0);
}
- atomic_thread_fence_rel();
- tls->gen = atomic_fetchadd_64(&ktls_glob_gen, 1);
+ tls->gen = 0;
*tlsp = tls;
return (0);
}
@@ -865,8 +862,7 @@ ktls_clone_session(struct ktls_session *tls, int direction)
memcpy(tls_new->params.cipher_key, tls->params.cipher_key,
tls->params.cipher_key_len);
- atomic_thread_fence_rel();
- tls_new->gen = atomic_fetchadd_64(&ktls_glob_gen, 1);
+ tls_new->gen = 0;
return (tls_new);
}
@@ -1946,8 +1942,6 @@ ktls_destroy(struct ktls_session *tls)
MPASS(tls->refcount == 0);
- atomic_add_acq_64(&ktls_glob_gen, 1);
-
inp = tls->inp;
if (tls->tx) {
wlocked = INP_WLOCKED(inp);
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index 7cd2168d262b..4542c5ccb8bb 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -2669,8 +2669,13 @@ SYSCTL_PROC(_net_inet_tcp, TCPCTL_PCBLIST, pcblist,
#define SND_TAG_STATUS_MAXLEN 128
#ifdef KERN_TLS
+
+static struct sx ktlslist_lock;
+SX_SYSINIT(ktlslistlock, &ktlslist_lock, "ktlslist");
+static uint64_t ktls_glob_gen = 1;
+
static int
-tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
+tcp_ktlslist_locked(SYSCTL_HANDLER_ARGS, bool export_keys)
{
struct xinpgen xig;
struct inpcb *inp;
@@ -2684,6 +2689,7 @@ tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
int error;
bool ek, p;
+ sx_assert(&ktlslist_lock, SA_XLOCKED);
if (req->newptr != NULL)
return (EPERM);
@@ -2692,7 +2698,7 @@ tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
ipi_gencnt = V_tcbinfo.ipi_gencnt;
bzero(&xig, sizeof(xig));
xig.xig_len = sizeof(xig);
- xig.xig_gen = atomic_load_acq_64(&ktls_glob_gen);
+ xig.xig_gen = ktls_glob_gen++;
xig.xig_sogen = so_gencnt;
struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
@@ -2708,7 +2714,8 @@ tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
ek = export_keys && cr_canexport_ktlskeys(
req->td, inp);
ksr = so->so_rcv.sb_tls_info;
- if (ktls_session_genvis(ksr, xig.xig_gen)) {
+ if (ksr != NULL) {
+ ksr->gen = xig.xig_gen;
p = true;
if (ek) {
sz = SIZE_T_MAX;
@@ -2726,7 +2733,8 @@ tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
}
}
kss = so->so_snd.sb_tls_info;
- if (ktls_session_genvis(kss, xig.xig_gen)) {
+ if (kss != NULL) {
+ kss->gen = xig.xig_gen;
p = true;
if (ek) {
sz = SIZE_T_MAX;
@@ -2783,11 +2791,11 @@ tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
ksr = so->so_rcv.sb_tls_info;
kss = so->so_snd.sb_tls_info;
xktls = (struct xktls_session *)buf;
- if (ktls_session_genvis(ksr, xig.xig_gen)) {
+ if (ksr != NULL && ksr->gen == xig.xig_gen) {
p = true;
ktls_session_to_xktls_onedir(ksr, ek, &xktls->rcv);
}
- if (ktls_session_genvis(kss, xig.xig_gen)) {
+ if (kss != NULL && kss->gen == xig.xig_gen) {
p = true;
ktls_session_to_xktls_onedir(kss, ek, &xktls->snd);
}
@@ -2798,7 +2806,7 @@ tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
xktls->so_pcb = (kvaddr_t)inp;
memcpy(&xktls->coninf, &inp->inp_inc, sizeof(xktls->coninf));
len = sizeof(*xktls);
- if (ktls_session_genvis(ksr, xig.xig_gen)) {
+ if (ksr != NULL && ksr->gen == xig.xig_gen) {
if (ek) {
sz = buflen - len;
ktls_session_copy_keys(ksr, buf + len, &sz);
@@ -2815,7 +2823,7 @@ tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
len += sz;
}
}
- if (ktls_session_genvis(kss, xig.xig_gen)) {
+ if (kss != NULL && kss->gen == xig.xig_gen) {
if (ek) {
sz = buflen - len;
ktls_session_copy_keys(kss, buf + len, &sz);
@@ -2845,8 +2853,6 @@ tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
}
if (error == 0) {
- atomic_thread_fence_rel();
- xig.xig_gen = atomic_load_64(&ktls_glob_gen);
xig.xig_sogen = so_gencnt;
xig.xig_count = cnt;
error = SYSCTL_OUT(req, &xig, sizeof(xig));
@@ -2856,16 +2862,27 @@ tcp_ktlslist(SYSCTL_HANDLER_ARGS, bool export_keys)
return (error);
}
+static int
+tcp_ktlslist1(SYSCTL_HANDLER_ARGS, bool export_keys)
+{
+ int res;
+
+ sx_xlock(&ktlslist_lock);
+ res = tcp_ktlslist_locked(oidp, arg1, arg2, req, export_keys);
+ sx_xunlock(&ktlslist_lock);
+ return (res);
+}
+
static int
tcp_ktlslist_nokeys(SYSCTL_HANDLER_ARGS)
{
- return (tcp_ktlslist(oidp, arg1, arg2, req, false));
+ return (tcp_ktlslist1(oidp, arg1, arg2, req, false));
}
static int
tcp_ktlslist_wkeys(SYSCTL_HANDLER_ARGS)
{
- return (tcp_ktlslist(oidp, arg1, arg2, req, true));
+ return (tcp_ktlslist1(oidp, arg1, arg2, req, true));
}
SYSCTL_PROC(_net_inet_tcp, TCPCTL_KTLSLIST, ktlslist,
diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h
index 0f9e5c5ed87b..a940bcfaba25 100644
--- a/sys/sys/ktls.h
+++ b/sys/sys/ktls.h
@@ -233,7 +233,6 @@ struct ktls_session {
} __aligned(CACHE_LINE_SIZE);
extern unsigned int ktls_ifnet_max_rexmit_pct;
-extern uint64_t ktls_glob_gen;
typedef enum {
KTLS_MBUF_CRYPTO_ST_MIXED = 0,
@@ -283,12 +282,6 @@ ktls_free(struct ktls_session *tls)
ktls_destroy(tls);
}
-static inline bool
-ktls_session_genvis(const struct ktls_session *ks, uint64_t gen)
-{
- return (ks != NULL && ks->gen <= gen);
-}
-
void ktls_session_to_xktls_onedir(const struct ktls_session *ks,
bool export_keys, struct xktls_session_onedir *xktls_od);
void ktls_session_copy_keys(const struct ktls_session *ktls,
