The branch main has been updated by jhb:
URL:
https://cgit.FreeBSD.org/src/commit/?id=1c8f4b3c9f9e8ca5823d153d3b117246b3d18db4
commit 1c8f4b3c9f9e8ca5823d153d3b117246b3d18db4
Author: John Baldwin <j...@freebsd.org>
AuthorDate: 2021-05-25 23:59:18 +0000
Commit: John Baldwin <j...@freebsd.org>
CommitDate: 2021-05-25 23:59:18 +0000
Support unmapped mbufs in crypto buffers.
Reviewed by: markj
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D30134
---
sys/opencrypto/criov.c | 121 +++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 111 insertions(+), 10 deletions(-)
diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c
index b18a01faf26a..bf7965032084 100644
--- a/sys/opencrypto/criov.c
+++ b/sys/opencrypto/criov.c
@@ -239,6 +239,99 @@ cvm_page_copydata(vm_page_t *pages, int off, int len,
caddr_t cp)
}
#endif /* CRYPTO_MAY_HAVE_VMPAGE */
+/*
+ * Given a starting page in an m_epg, determine the length of the
+ * current physically contiguous segment.
+ */
+static __inline size_t
+m_epg_pages_extent(struct mbuf *m, int idx, u_int pglen)
+{
+ size_t len;
+ u_int i;
+
+ len = pglen;
+ for (i = idx + 1; i < m->m_epg_npgs; i++) {
+ if (m->m_epg_pa[i - 1] + PAGE_SIZE != m->m_epg_pa[i])
+ break;
+ len += m_epg_pagelen(m, i, 0);
+ }
+ return (len);
+}
+
+static __inline void *
+m_epg_segbase(struct mbuf *m, size_t offset)
+{
+ u_int i, pglen, pgoff;
+
+ offset += mtod(m, vm_offset_t);
+ if (offset < m->m_epg_hdrlen)
+ return (m->m_epg_hdr + offset);
+ offset -= m->m_epg_hdrlen;
+ pgoff = m->m_epg_1st_off;
+ for (i = 0; i < m->m_epg_npgs; i++) {
+ pglen = m_epg_pagelen(m, i, pgoff);
+ if (offset < pglen)
+ return ((void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff +
+ offset));
+ offset -= pglen;
+ pgoff = 0;
+ }
+ KASSERT(offset <= m->m_epg_trllen, ("%s: offset beyond trailer",
+ __func__));
+ return (m->m_epg_trail + offset);
+}
+
+static __inline size_t
+m_epg_seglen(struct mbuf *m, size_t offset)
+{
+ u_int i, pglen, pgoff;
+
+ offset += mtod(m, vm_offset_t);
+ if (offset < m->m_epg_hdrlen)
+ return (m->m_epg_hdrlen - offset);
+ offset -= m->m_epg_hdrlen;
+ pgoff = m->m_epg_1st_off;
+ for (i = 0; i < m->m_epg_npgs; i++) {
+ pglen = m_epg_pagelen(m, i, pgoff);
+ if (offset < pglen)
+ return (m_epg_pages_extent(m, i, pglen) - offset);
+ offset -= pglen;
+ pgoff = 0;
+ }
+ KASSERT(offset <= m->m_epg_trllen, ("%s: offset beyond trailer",
+ __func__));
+ return (m->m_epg_trllen - offset);
+}
+
+static __inline void *
+m_epg_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
+{
+ u_int i, pglen, pgoff;
+
+ skip += mtod(m, vm_offset_t);
+ if (skip < m->m_epg_hdrlen) {
+ if (len > m->m_epg_hdrlen - skip)
+ return (NULL);
+ return (m->m_epg_hdr + skip);
+ }
+ skip -= m->m_epg_hdrlen;
+ pgoff = m->m_epg_1st_off;
+ for (i = 0; i < m->m_epg_npgs; i++) {
+ pglen = m_epg_pagelen(m, i, pgoff);
+ if (skip < pglen) {
+ if (len > m_epg_pages_extent(m, i, pglen) - skip)
+ return (NULL);
+ return ((void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff +
+ skip));
+ }
+ skip -= pglen;
+ pgoff = 0;
+ }
+ KASSERT(skip <= m->m_epg_trllen && len <= m->m_epg_trllen - skip,
+ ("%s: segment beyond trailer", __func__));
+ return (m->m_epg_trail + skip);
+}
+
void
crypto_cursor_init(struct crypto_buffer_cursor *cc,
const struct crypto_buffer *cb)
@@ -345,8 +438,8 @@ crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
case CRYPTO_BUF_MBUF:
if (cc->cc_mbuf == NULL)
return (NULL);
- KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
- ("%s: not supported for unmapped mbufs", __func__));
+ if (cc->cc_mbuf->m_flags & M_EXTPG)
+ return (m_epg_segbase(cc->cc_mbuf, cc->cc_offset));
return (mtod(cc->cc_mbuf, char *) + cc->cc_offset);
case CRYPTO_BUF_VMPAGE:
return ((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
@@ -372,6 +465,8 @@ crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
case CRYPTO_BUF_MBUF:
if (cc->cc_mbuf == NULL)
return (0);
+ if (cc->cc_mbuf->m_flags & M_EXTPG)
+ return (m_epg_seglen(cc->cc_mbuf, cc->cc_offset));
return (cc->cc_mbuf->m_len - cc->cc_offset);
case CRYPTO_BUF_UIO:
return (cc->cc_iov->iov_len - cc->cc_offset);
@@ -401,12 +496,14 @@ crypto_cursor_copyback(struct crypto_buffer_cursor *cc,
int size,
break;
case CRYPTO_BUF_MBUF:
for (;;) {
- KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
- ("%s: not supported for unmapped mbufs", __func__));
- dst = mtod(cc->cc_mbuf, char *) + cc->cc_offset;
+ /*
+ * This uses m_copyback() for individual
+ * mbufs so that cc_mbuf and cc_offset are
+ * updated.
+ */
remain = cc->cc_mbuf->m_len - cc->cc_offset;
todo = MIN(remain, size);
- memcpy(dst, src, todo);
+ m_copyback(cc->cc_mbuf, cc->cc_offset, todo, src);
src += todo;
if (todo < remain) {
cc->cc_offset += todo;
@@ -482,12 +579,14 @@ crypto_cursor_copydata(struct crypto_buffer_cursor *cc,
int size, void *vdst)
break;
case CRYPTO_BUF_MBUF:
for (;;) {
- KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
- ("%s: not supported for unmapped mbufs", __func__));
- src = mtod(cc->cc_mbuf, const char *) + cc->cc_offset;
+ /*
+ * This uses m_copydata() for individual
+ * mbufs so that cc_mbuf and cc_offset are
+ * updated.
+ */
remain = cc->cc_mbuf->m_len - cc->cc_offset;
todo = MIN(remain, size);
- memcpy(dst, src, todo);
+ m_copydata(cc->cc_mbuf, cc->cc_offset, todo, dst);
dst += todo;
if (todo < remain) {
cc->cc_offset += todo;
@@ -715,6 +814,8 @@ m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t
len)
if (skip + len > m->m_len)
return (NULL);
+ if (m->m_flags & M_EXTPG)
+ return (m_epg_contiguous_subsegment(m, skip, len));
return (mtod(m, char*) + skip);
}
_______________________________________________
dev-commits-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/dev-commits-src-all
To unsubscribe, send any mail to "dev-commits-src-all-unsubscr...@freebsd.org"
