[dpdk-dev] [PATCH] rte_mbuf: scattered pktmbufs freeing optimization
From: "vadim.suraev at gmail.com" new function - rte_pktmbuf_free_bulk makes freeing long scattered (chained) pktmbufs belonging to the same pool more optimal using rte_mempool_put_bulk rather than calling rte_mempool_put for each segment. Inlike rte_pktmbuf_free, which calls rte_pktmbuf_free_seg, this function calls __rte_pktmbuf_prefree_seg. If non-NULL returned, the pointer is placed in an array. When array is filled or when the last segment is processed, rte_mempool_put_bulk is called. In case of multiple producers, performs 3 times better. Signed-off-by: vadim.suraev at gmail.com --- lib/librte_mbuf/rte_mbuf.h | 55 1 file changed, 55 insertions(+) diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h index 17ba791..1d6f848 100644 --- a/lib/librte_mbuf/rte_mbuf.h +++ b/lib/librte_mbuf/rte_mbuf.h @@ -824,6 +824,61 @@ static inline void rte_pktmbuf_free(struct rte_mbuf *m) } } +/* This macro defines the size of max bulk of mbufs to free for rte_pktmbuf_free_bulk */ +#define MAX_MBUF_FREE_SIZE 32 + +/* If RTE_LIBRTE_MBUF_DEBUG is enabled, checks if all mbufs must belong to the same mempool */ +#ifdef RTE_LIBRTE_MBUF_DEBUG + +#define RTE_MBUF_MEMPOOL_CHECK1(m) struct rte_mempool *first_buffers_mempool = (m) ? (m)->pool : NULL + +#define RTE_MBUF_MEMPOOL_CHECK2(m) RTE_MBUF_ASSERT(first_buffers_mempool == (m)->pool) + +#else + +#define RTE_MBUF_MEMPOOL_CHECK1(m) + +#define RTE_MBUF_MEMPOOL_CHECK2(m) + +#endif + +/** + * Free chained (scattered) mbuf into its original mempool. + * + * All the mbufs in the chain must belong to the same mempool. + * + * @param head + * The head of mbufs to be freed chain + */ + +static inline void __attribute__((always_inline)) +rte_pktmbuf_free_bulk(struct rte_mbuf *head) +{ +void *mbufs[MAX_MBUF_FREE_SIZE]; +unsigned mbufs_count = 0; +struct rte_mbuf *next; + +RTE_MBUF_MEMPOOL_CHECK1(head); + +while(head) { +next = head->next; +head->next = NULL; +if(__rte_pktmbuf_prefree_seg(head)) { +RTE_MBUF_ASSERT(rte_mbuf_refcnt_read(head) == 0); +RTE_MBUF_MEMPOOL_CHECK2(head); +mbufs[mbufs_count++] = head; +} +head = next; +if(mbufs_count == MAX_MBUF_FREE_SIZE) { +rte_mempool_put_bulk(((struct rte_mbuf *)mbufs[0])->pool,mbufs,mbufs_count); +mbufs_count = 0; +} +} +if(mbufs_count > 0) { +rte_mempool_put_bulk(((struct rte_mbuf *)mbufs[0])->pool,mbufs,mbufs_count); +} +} + /** * Creates a "clone" of the given packet mbuf. * -- 1.7.9.5
[dpdk-dev] [PATCH] rte_mbuf: bulk allocation and freeing functions + unittest
From: "vadim.suraev at gmail.com" - an API function to allocate a bulk of rte_mbufs - an API function to free a bulk of rte_mbufs - an API function to free a chained rte_mbuf - unittest for aboce Signed-off-by: vadim.suraev at gmail.com --- app/test/test_mbuf.c | 73 lib/librte_mbuf/rte_mbuf.h | 101 2 files changed, 174 insertions(+) diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c index 1ff66cb..a557705 100644 --- a/app/test/test_mbuf.c +++ b/app/test/test_mbuf.c @@ -405,6 +405,67 @@ test_pktmbuf_pool(void) return ret; } +/* test pktmbuf bulk allocation and freeing +*/ +static int +test_pktmbuf_pool_bulk(void) +{ + unsigned i; + unsigned mbufs_to_allocate = NB_MBUF - 32; /* size of mempool - size of local cache, otherwise may fail */ + struct rte_mbuf *m[mbufs_to_allocate]; + int ret = 0; + unsigned mbuf_count_before_allocation = rte_mempool_count(pktmbuf_pool); + + for (i=0; inext = m[i + 1]; + m[0]->nb_segs++; + } + /* free them */ + rte_pktmbuf_free_chain(m[0]); + + if (rte_mempool_count(pktmbuf_pool) != mbuf_count_before_allocation) { + printf("mempool count %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), mbuf_count_before_allocation); + return -1; + } + return ret; +} + /* * test that the pointer to the data on a packet mbuf is set properly */ @@ -790,6 +851,18 @@ test_mbuf(void) return -1; } + /* test bulk allocation and freeing */ + if (test_pktmbuf_pool_bulk() < 0) { + printf("test_pktmbuf_pool_bulk() failed\n"); + return -1; + } + + /* once again to ensure all mbufs were freed */ + if (test_pktmbuf_pool_bulk() < 0) { + printf("test_pktmbuf_pool_bulk() failed\n"); + return -1; + } + /* test that the pointer to the data on a packet mbuf is set properly */ if (test_pktmbuf_pool_ptr() < 0) { printf("test_pktmbuf_pool_ptr() failed\n"); diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h index 17ba791..482920e 100644 --- a/lib/librte_mbuf/rte_mbuf.h +++ b/lib/librte_mbuf/rte_mbuf.h @@ -825,6 +825,107 @@ static inline void rte_pktmbuf_free(struct rte_mbuf *m) } /** + * Allocates a bulk of mbufs, initiates refcnt and resets + * + * @param pool + *memory pool to allocate from + * @param mbufs + *Array of pointers to mbuf + * @param count + *Array size + */ +static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count) +{ + unsigned idx; + int rc = 0; + + if (unlikely(rc = rte_mempool_get_bulk(pool, (void **)mbufs, count))) { + return rc; + } + + for (idx = 0; idx < count; idx++) { + RTE_MBUF_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0); + rte_mbuf_refcnt_set(mbufs[idx], 1); + rte_pktmbuf_reset(mbufs[idx]); + } + return rc; +} + +/** + * Frees a bulk of mbufs into its original mempool. + * It is responsibility of caller to update and check refcnt + * as well as to check for attached mbufs to be freed + * + * @param mbufs + *Array of pointers to mbuf + * @param count + *Array size + */ + +static inline void rte_pktmbuf_bulk_raw_free(struct rte_mbuf **mbufs, unsigned count) +{ + if (unlikely(count == 0)) + return; + rte_mempool_put_bulk(mbufs[0]->pool, (void **)mbufs, count); +} + +/** + * Frees a bulk of mbufs into its original mempool. + * This function assumes refcnt has been already decremented + * as well as the freed mbufs are direct + * + * @param mbufs + *Array of pointers to mbuf + * @param count + *Array size + */ + +static inline void rte_pktmbuf_bulk_free(struct rte_mbuf **mbufs, unsigned count) +{ + if (unlikely(count == 0)) + return; + unsigned idx; + + for(idx = 0; idx < count; idx++) { + rte_mbuf_refcnt_update(mbufs[idx], -1); + RTE_MBUF_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0); + } + rte_mempool_put_bulk(mbufs[0]->pool, (void **)mbufs, count); +} + +/** + * Frees chained (scattered) mbufs into its original mempool(s). + * + * @param head + *The head of mbufs to be freed chain + */ + +static inline void rte_pktmbuf_free_chain(struct rte_mbuf *head) +{ + if (likely(head != NULL)) { + struct rte_mbuf *mbufs[head->nb_segs]; + unsigned mbufs_count = 0; + struct rte_mbuf *next; + + while (head) { + next = head->next; + head->next = NULL; + if (likely(NULL != (head = __rte_pktmbuf_prefree_seg(head { + RTE_MBUF_ASSERT(rte_mbuf_re
[dpdk-dev] [PATCH] rte_mbuf: mbuf bulk alloc/free functions added + unittest
From: "vadim.suraev at gmail.com" - an API function to allocate a bulk of rte_mbufs - an API function to free a bulk of rte_mbufs - an API function to free a chained rte_mbuf - unittest for above Signed-off-by: vadim.suraev at gmail.com --- app/test/test_mbuf.c | 94 +++- lib/librte_mbuf/rte_mbuf.h | 89 + 2 files changed, 182 insertions(+), 1 deletion(-) diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c index 1ff66cb..b20c6a4 100644 --- a/app/test/test_mbuf.c +++ b/app/test/test_mbuf.c @@ -77,6 +77,7 @@ #define REFCNT_RING_SIZE(REFCNT_MBUF_NUM * REFCNT_MAX_REF) #define MAKE_STRING(x) # x +#define MBUF_POOL_LOCAL_CACHE_SIZE 32 static struct rte_mempool *pktmbuf_pool = NULL; @@ -405,6 +406,84 @@ test_pktmbuf_pool(void) return ret; } +/* test pktmbuf bulk allocation and freeing +*/ +static int +test_pktmbuf_pool_bulk(void) +{ + unsigned i; + /* size of mempool - size of local cache, otherwise may fail */ + unsigned mbufs_to_allocate = NB_MBUF - MBUF_POOL_LOCAL_CACHE_SIZE; + struct rte_mbuf *m[mbufs_to_allocate]; + int ret = 0; + unsigned mbuf_count_before_allocation = rte_mempool_count(pktmbuf_pool); + + for (i = 0; i < mbufs_to_allocate; i++) + m[i] = NULL; + /* alloc NB_MBUF-MBUF_POOL_LOCAL_CACHE_SIZE mbufs */ + ret = rte_pktmbuf_alloc_bulk(pktmbuf_pool, m, mbufs_to_allocate); + if (ret) { + printf("cannot allocate %d mbufs bulk mempool_cnt=%d ret=%d\n", + mbufs_to_allocate, + rte_mempool_count(pktmbuf_pool), + ret); + return -1; + } + if ((rte_mempool_count(pktmbuf_pool) + mbufs_to_allocate) != + mbuf_count_before_allocation) { + printf("mempool count %d + allocated %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbufs_to_allocate, + mbuf_count_before_allocation); + return -1; + } + /* free them */ + rte_pktmbuf_bulk_free(m, mbufs_to_allocate); + + if (rte_mempool_count(pktmbuf_pool) != mbuf_count_before_allocation) { + printf("mempool count %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbuf_count_before_allocation); + return -1; + } + for (i = 0; i < mbufs_to_allocate; i++) + m[i] = NULL; + + /* alloc NB_MBUF-MBUF_POOL_LOCAL_CACHE_SIZE mbufs */ + ret = rte_pktmbuf_alloc_bulk(pktmbuf_pool, m, mbufs_to_allocate); + if (ret) { + printf("cannot allocate %d mbufs bulk mempool_cnt=%d ret=%d\n", + mbufs_to_allocate, + rte_mempool_count(pktmbuf_pool), + ret); + return -1; + } + if ((rte_mempool_count(pktmbuf_pool) + mbufs_to_allocate) != + mbuf_count_before_allocation) { + printf("mempool count %d + allocated %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbufs_to_allocate, + mbuf_count_before_allocation); + return -1; + } + + /* chain it */ + for (i = 0; i < mbufs_to_allocate - 1; i++) { + m[i]->next = m[i + 1]; + m[0]->nb_segs++; + } + /* free them */ + rte_pktmbuf_free_chain(m[0]); + + if (rte_mempool_count(pktmbuf_pool) != mbuf_count_before_allocation) { + printf("mempool count %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbuf_count_before_allocation); + return -1; + } + return ret; +} + /* * test that the pointer to the data on a packet mbuf is set properly */ @@ -766,7 +845,8 @@ test_mbuf(void) if (pktmbuf_pool == NULL) { pktmbuf_pool = rte_mempool_create("test_pktmbuf_pool", NB_MBUF, - MBUF_SIZE, 32, + MBUF_SIZE, + MBUF_POOL_LOCAL_CACHE_SIZE, sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, NULL, @@ -790,6 +870,18 @@ test_mbuf(void) return -1; } + /* test bulk allocation and freeing */ + if (test_pktmbuf_pool_bulk() < 0) { + printf("test_pktmbuf_pool_bulk() failed\n"); + return -1; + } + + /* once again to ensure all mbufs were freed */ + if (test_pktmbuf_pool_bulk() < 0) { +
[dpdk-dev] [PATCH v2] rte_mbuf: mbuf bulk alloc/free functions added + unittest
From: "vadim.suraev at gmail.com" This patch adds mbuf bulk allocation/freeing functions and unittest Signed-off-by: Vadim Suraev --- New in v2: - function rte_pktmbuf_alloc_bulk added - function rte_pktmbuf_bulk_free added - function rte_pktmbuf_free_chain added - applied reviewers' comments app/test/test_mbuf.c | 94 +++- lib/librte_mbuf/rte_mbuf.h | 89 + 2 files changed, 182 insertions(+), 1 deletion(-) diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c index 1ff66cb..b20c6a4 100644 --- a/app/test/test_mbuf.c +++ b/app/test/test_mbuf.c @@ -77,6 +77,7 @@ #define REFCNT_RING_SIZE(REFCNT_MBUF_NUM * REFCNT_MAX_REF) #define MAKE_STRING(x) # x +#define MBUF_POOL_LOCAL_CACHE_SIZE 32 static struct rte_mempool *pktmbuf_pool = NULL; @@ -405,6 +406,84 @@ test_pktmbuf_pool(void) return ret; } +/* test pktmbuf bulk allocation and freeing +*/ +static int +test_pktmbuf_pool_bulk(void) +{ + unsigned i; + /* size of mempool - size of local cache, otherwise may fail */ + unsigned mbufs_to_allocate = NB_MBUF - MBUF_POOL_LOCAL_CACHE_SIZE; + struct rte_mbuf *m[mbufs_to_allocate]; + int ret = 0; + unsigned mbuf_count_before_allocation = rte_mempool_count(pktmbuf_pool); + + for (i = 0; i < mbufs_to_allocate; i++) + m[i] = NULL; + /* alloc NB_MBUF-MBUF_POOL_LOCAL_CACHE_SIZE mbufs */ + ret = rte_pktmbuf_alloc_bulk(pktmbuf_pool, m, mbufs_to_allocate); + if (ret) { + printf("cannot allocate %d mbufs bulk mempool_cnt=%d ret=%d\n", + mbufs_to_allocate, + rte_mempool_count(pktmbuf_pool), + ret); + return -1; + } + if ((rte_mempool_count(pktmbuf_pool) + mbufs_to_allocate) != + mbuf_count_before_allocation) { + printf("mempool count %d + allocated %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbufs_to_allocate, + mbuf_count_before_allocation); + return -1; + } + /* free them */ + rte_pktmbuf_bulk_free(m, mbufs_to_allocate); + + if (rte_mempool_count(pktmbuf_pool) != mbuf_count_before_allocation) { + printf("mempool count %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbuf_count_before_allocation); + return -1; + } + for (i = 0; i < mbufs_to_allocate; i++) + m[i] = NULL; + + /* alloc NB_MBUF-MBUF_POOL_LOCAL_CACHE_SIZE mbufs */ + ret = rte_pktmbuf_alloc_bulk(pktmbuf_pool, m, mbufs_to_allocate); + if (ret) { + printf("cannot allocate %d mbufs bulk mempool_cnt=%d ret=%d\n", + mbufs_to_allocate, + rte_mempool_count(pktmbuf_pool), + ret); + return -1; + } + if ((rte_mempool_count(pktmbuf_pool) + mbufs_to_allocate) != + mbuf_count_before_allocation) { + printf("mempool count %d + allocated %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbufs_to_allocate, + mbuf_count_before_allocation); + return -1; + } + + /* chain it */ + for (i = 0; i < mbufs_to_allocate - 1; i++) { + m[i]->next = m[i + 1]; + m[0]->nb_segs++; + } + /* free them */ + rte_pktmbuf_free_chain(m[0]); + + if (rte_mempool_count(pktmbuf_pool) != mbuf_count_before_allocation) { + printf("mempool count %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbuf_count_before_allocation); + return -1; + } + return ret; +} + /* * test that the pointer to the data on a packet mbuf is set properly */ @@ -766,7 +845,8 @@ test_mbuf(void) if (pktmbuf_pool == NULL) { pktmbuf_pool = rte_mempool_create("test_pktmbuf_pool", NB_MBUF, - MBUF_SIZE, 32, + MBUF_SIZE, + MBUF_POOL_LOCAL_CACHE_SIZE, sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, NULL, @@ -790,6 +870,18 @@ test_mbuf(void) return -1; } + /* test bulk allocation and freeing */ + if (test_pktmbuf_pool_bulk() < 0) { + printf("test_pktmbuf_pool_bulk() failed\n"); + return -1; + } + + /* once again to ensure all
[dpdk-dev] [PATCH v2] rte_mbuf: mbuf bulk alloc/free functions added + unittest
From: "vadim.suraev at gmail.com" This patch adds mbuf bulk allocation/freeing functions and unittest Signed-off-by: Vadim Suraev --- New in v2: - function rte_pktmbuf_alloc_bulk added - function rte_pktmbuf_bulk_free added - function rte_pktmbuf_free_chain added - applied reviewers' comments app/test/test_mbuf.c | 94 +++- lib/librte_mbuf/rte_mbuf.h | 91 ++ 2 files changed, 184 insertions(+), 1 deletion(-) diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c index 1ff66cb..b20c6a4 100644 --- a/app/test/test_mbuf.c +++ b/app/test/test_mbuf.c @@ -77,6 +77,7 @@ #define REFCNT_RING_SIZE(REFCNT_MBUF_NUM * REFCNT_MAX_REF) #define MAKE_STRING(x) # x +#define MBUF_POOL_LOCAL_CACHE_SIZE 32 static struct rte_mempool *pktmbuf_pool = NULL; @@ -405,6 +406,84 @@ test_pktmbuf_pool(void) return ret; } +/* test pktmbuf bulk allocation and freeing +*/ +static int +test_pktmbuf_pool_bulk(void) +{ + unsigned i; + /* size of mempool - size of local cache, otherwise may fail */ + unsigned mbufs_to_allocate = NB_MBUF - MBUF_POOL_LOCAL_CACHE_SIZE; + struct rte_mbuf *m[mbufs_to_allocate]; + int ret = 0; + unsigned mbuf_count_before_allocation = rte_mempool_count(pktmbuf_pool); + + for (i = 0; i < mbufs_to_allocate; i++) + m[i] = NULL; + /* alloc NB_MBUF-MBUF_POOL_LOCAL_CACHE_SIZE mbufs */ + ret = rte_pktmbuf_alloc_bulk(pktmbuf_pool, m, mbufs_to_allocate); + if (ret) { + printf("cannot allocate %d mbufs bulk mempool_cnt=%d ret=%d\n", + mbufs_to_allocate, + rte_mempool_count(pktmbuf_pool), + ret); + return -1; + } + if ((rte_mempool_count(pktmbuf_pool) + mbufs_to_allocate) != + mbuf_count_before_allocation) { + printf("mempool count %d + allocated %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbufs_to_allocate, + mbuf_count_before_allocation); + return -1; + } + /* free them */ + rte_pktmbuf_bulk_free(m, mbufs_to_allocate); + + if (rte_mempool_count(pktmbuf_pool) != mbuf_count_before_allocation) { + printf("mempool count %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbuf_count_before_allocation); + return -1; + } + for (i = 0; i < mbufs_to_allocate; i++) + m[i] = NULL; + + /* alloc NB_MBUF-MBUF_POOL_LOCAL_CACHE_SIZE mbufs */ + ret = rte_pktmbuf_alloc_bulk(pktmbuf_pool, m, mbufs_to_allocate); + if (ret) { + printf("cannot allocate %d mbufs bulk mempool_cnt=%d ret=%d\n", + mbufs_to_allocate, + rte_mempool_count(pktmbuf_pool), + ret); + return -1; + } + if ((rte_mempool_count(pktmbuf_pool) + mbufs_to_allocate) != + mbuf_count_before_allocation) { + printf("mempool count %d + allocated %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbufs_to_allocate, + mbuf_count_before_allocation); + return -1; + } + + /* chain it */ + for (i = 0; i < mbufs_to_allocate - 1; i++) { + m[i]->next = m[i + 1]; + m[0]->nb_segs++; + } + /* free them */ + rte_pktmbuf_free_chain(m[0]); + + if (rte_mempool_count(pktmbuf_pool) != mbuf_count_before_allocation) { + printf("mempool count %d != initial %d\n", + rte_mempool_count(pktmbuf_pool), + mbuf_count_before_allocation); + return -1; + } + return ret; +} + /* * test that the pointer to the data on a packet mbuf is set properly */ @@ -766,7 +845,8 @@ test_mbuf(void) if (pktmbuf_pool == NULL) { pktmbuf_pool = rte_mempool_create("test_pktmbuf_pool", NB_MBUF, - MBUF_SIZE, 32, + MBUF_SIZE, + MBUF_POOL_LOCAL_CACHE_SIZE, sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, NULL, @@ -790,6 +870,18 @@ test_mbuf(void) return -1; } + /* test bulk allocation and freeing */ + if (test_pktmbuf_pool_bulk() < 0) { + printf("test_pktmbuf_pool_bulk() failed\n"); + return -1; + } + + /* once again to ensure al