The branch stable/13 has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=2ba824366ca60679c36296991582054116d2d4a7
commit 2ba824366ca60679c36296991582054116d2d4a7 Author: John Baldwin <[email protected]> AuthorDate: 2021-05-25 23:59:18 +0000 Commit: John Baldwin <[email protected]> CommitDate: 2021-10-21 15:51:25 +0000 Extend m_apply() to support unmapped mbufs. m_apply() invokes the callback function separately on each segment of an unmapped mbuf: the TLS header, individual pages, and the TLS trailer. Reviewed by: markj Sponsored by: Netflix Differential Revision: https://reviews.freebsd.org/D30132 (cherry picked from commit 3c7a01d773ac2d128eabb596eed7098f76966cc5) --- sys/kern/uipc_mbuf.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c index f525c25d1431..8b5b83337d56 100644 --- a/sys/kern/uipc_mbuf.c +++ b/sys/kern/uipc_mbuf.c @@ -1239,6 +1239,62 @@ m_append(struct mbuf *m0, int len, c_caddr_t cp) return (remainder == 0); } +static int +m_apply_extpg_one(struct mbuf *m, int off, int len, + int (*f)(void *, void *, u_int), void *arg) +{ + void *p; + u_int i, count, pgoff, pglen; + int rval; + + KASSERT(PMAP_HAS_DMAP, + ("m_apply_extpg_one does not support unmapped mbufs")); + off += mtod(m, vm_offset_t); + if (off < m->m_epg_hdrlen) { + count = min(m->m_epg_hdrlen - off, len); + rval = f(arg, m->m_epg_hdr + off, count); + if (rval) + return (rval); + len -= count; + off = 0; + } else + off -= m->m_epg_hdrlen; + pgoff = m->m_epg_1st_off; + for (i = 0; i < m->m_epg_npgs && len > 0; i++) { + pglen = m_epg_pagelen(m, i, pgoff); + if (off < pglen) { + count = min(pglen - off, len); + p = (void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff); + rval = f(arg, p, count); + if (rval) + return (rval); + len -= count; + off = 0; + } else + off -= pglen; + pgoff = 0; + } + if (len > 0) { + KASSERT(off < m->m_epg_trllen, + ("m_apply_extpg_one: offset beyond trailer")); + KASSERT(len <= m->m_epg_trllen - off, + ("m_apply_extpg_one: length beyond trailer")); + return (f(arg, m->m_epg_trail + off, len)); + } + return (0); +} + +/* Apply function f to the data in a single mbuf. */ +static int +m_apply_one(struct mbuf *m, int off, int len, + int (*f)(void *, void *, u_int), void *arg) +{ + if ((m->m_flags & M_EXTPG) != 0) + return (m_apply_extpg_one(m, off, len, f, arg)); + else + return (f(arg, mtod(m, caddr_t) + off, len)); +} + /* * Apply function f to the data in an mbuf chain starting "off" bytes from * the beginning, continuing for "len" bytes. @@ -1262,7 +1318,7 @@ m_apply(struct mbuf *m, int off, int len, while (len > 0) { KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain")); count = min(m->m_len - off, len); - rval = (*f)(arg, mtod(m, caddr_t) + off, count); + rval = m_apply_one(m, off, count, f, arg); if (rval) return (rval); len -= count;
