Author: glebius Date: Sat May 2 23:38:13 2020 New Revision: 360573 URL: https://svnweb.freebsd.org/changeset/base/360573
Log: Step 2.1: Build TLS workqueue from mbufs, not struct mbuf_ext_pgs. Reviewed by: gallatin Differential Revision: https://reviews.freebsd.org/D24598 Modified: head/sys/kern/kern_mbuf.c head/sys/kern/uipc_ktls.c head/sys/sys/ktls.h head/sys/sys/mbuf.h Modified: head/sys/kern/kern_mbuf.c ============================================================================== --- head/sys/kern/kern_mbuf.c Sat May 2 22:56:22 2020 (r360572) +++ head/sys/kern/kern_mbuf.c Sat May 2 23:38:13 2020 (r360573) @@ -1246,7 +1246,6 @@ mb_free_ext(struct mbuf *m) break; case EXT_PGS: { #ifdef KERN_TLS - struct mbuf_ext_pgs *pgs; struct ktls_session *tls; #endif @@ -1254,11 +1253,10 @@ mb_free_ext(struct mbuf *m) ("%s: ext_free not set", __func__)); mref->m_ext.ext_free(mref); #ifdef KERN_TLS - pgs = &mref->m_ext_pgs; - tls = pgs->tls; + tls = mref->m_ext_pgs.tls; if (tls != NULL && !refcount_release_if_not_last(&tls->refcount)) - ktls_enqueue_to_free(pgs); + ktls_enqueue_to_free(mref); else #endif uma_zfree(zone_mbuf, mref); Modified: head/sys/kern/uipc_ktls.c ============================================================================== --- head/sys/kern/uipc_ktls.c Sat May 2 22:56:22 2020 (r360572) +++ head/sys/kern/uipc_ktls.c Sat May 2 23:38:13 2020 (r360573) @@ -79,7 +79,7 @@ __FBSDID("$FreeBSD$"); struct ktls_wq { struct mtx mtx; - STAILQ_HEAD(, mbuf_ext_pgs) head; + STAILQ_HEAD(, mbuf) head; bool running; } __aligned(CACHE_LINE_SIZE); @@ -1430,16 +1430,19 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, } void -ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs) +ktls_enqueue_to_free(struct mbuf *m) { + struct mbuf_ext_pgs *pgs; struct ktls_wq *wq; bool running; + pgs = &m->m_ext_pgs; + /* Mark it for freeing. */ pgs->flags |= EPG_FLAG_2FREE; wq = &ktls_wq[pgs->tls->wq_index]; mtx_lock(&wq->mtx); - STAILQ_INSERT_TAIL(&wq->head, pgs, stailq); + STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq); running = wq->running; mtx_unlock(&wq->mtx); if (!running) @@ -1472,7 +1475,7 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int pa wq = &ktls_wq[pgs->tls->wq_index]; mtx_lock(&wq->mtx); - STAILQ_INSERT_TAIL(&wq->head, pgs, stailq); + STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq); running = wq->running; mtx_unlock(&wq->mtx); if (!running) @@ -1481,11 +1484,12 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int pa } static __noinline void -ktls_encrypt(struct mbuf_ext_pgs *pgs) +ktls_encrypt(struct mbuf *top) { struct ktls_session *tls; struct socket *so; - struct mbuf *m, *top; + struct mbuf *m; + struct mbuf_ext_pgs *pgs; vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; @@ -1493,15 +1497,14 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs) int error, i, len, npages, off, total_pages; bool is_anon; - so = pgs->so; - tls = pgs->tls; - top = __containerof(pgs, struct mbuf, m_ext_pgs); - KASSERT(tls != NULL, ("tls = NULL, top = %p, pgs = %p\n", top, pgs)); - KASSERT(so != NULL, ("so = NULL, top = %p, pgs = %p\n", top, pgs)); + so = top->m_ext_pgs.so; + tls = top->m_ext_pgs.tls; + KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top)); + KASSERT(so != NULL, ("so = NULL, top = %p\n", top)); #ifdef INVARIANTS - pgs->so = NULL; + top->m_ext_pgs.so = NULL; #endif - total_pages = pgs->enc_cnt; + total_pages = top->m_ext_pgs.enc_cnt; npages = 0; /* @@ -1631,10 +1634,8 @@ static void ktls_work_thread(void *ctx) { struct ktls_wq *wq = ctx; - struct mbuf_ext_pgs *p, *n; - struct ktls_session *tls; - struct mbuf *m; - STAILQ_HEAD(, mbuf_ext_pgs) local_head; + struct mbuf *m, *n; + STAILQ_HEAD(, mbuf) local_head; #if defined(__aarch64__) || defined(__amd64__) || defined(__i386__) fpu_kern_thread(0); @@ -1651,14 +1652,12 @@ ktls_work_thread(void *ctx) STAILQ_CONCAT(&local_head, &wq->head); mtx_unlock(&wq->mtx); - STAILQ_FOREACH_SAFE(p, &local_head, stailq, n) { - if (p->flags & EPG_FLAG_2FREE) { - tls = p->tls; - ktls_free(tls); - m = __containerof(p, struct mbuf, m_ext_pgs); + STAILQ_FOREACH_SAFE(m, &local_head, m_ext_pgs.stailq, n) { + if (m->m_ext_pgs.flags & EPG_FLAG_2FREE) { + ktls_free(m->m_ext_pgs.tls); uma_zfree(zone_mbuf, m); } else { - ktls_encrypt(p); + ktls_encrypt(m); counter_u64_add(ktls_cnt_on, -1); } } Modified: head/sys/sys/ktls.h ============================================================================== --- head/sys/sys/ktls.h Sat May 2 22:56:22 2020 (r360572) +++ head/sys/sys/ktls.h Sat May 2 23:38:13 2020 (r360573) @@ -169,7 +169,6 @@ struct iovec; struct ktls_session; struct m_snd_tag; struct mbuf; -struct mbuf_ext_pgs; struct sockbuf; struct socket; @@ -212,7 +211,7 @@ void ktls_frame(struct mbuf *m, struct ktls_session *t uint8_t record_type); void ktls_seq(struct sockbuf *sb, struct mbuf *m); void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count); -void ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs); +void ktls_enqueue_to_free(struct mbuf *m); int ktls_get_rx_mode(struct socket *so); int ktls_set_tx_mode(struct socket *so, int mode); int ktls_get_tx_mode(struct socket *so); Modified: head/sys/sys/mbuf.h ============================================================================== --- head/sys/sys/mbuf.h Sat May 2 22:56:22 2020 (r360572) +++ head/sys/sys/mbuf.h Sat May 2 23:38:13 2020 (r360573) @@ -372,7 +372,7 @@ struct mbuf { struct ktls_session *tls; struct socket *so; uint64_t seqno; - STAILQ_ENTRY(mbuf_ext_pgs) stailq; + STAILQ_ENTRY(mbuf) stailq; } m_ext_pgs; }; union { _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
