From: "Artem V. Andreev" <artem.andr...@oktetlabs.ru>
Clustered allocation is required to simplify packaging objects into
buckets and search of the bucket control structure by an object.
Signed-off-by: Artem V. Andreev <artem.andr...@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybche...@solarflare.com>
---
lib/librte_mempool/rte_mempool.c | 39 +++++++++++++++++++++++++++++++++++----
lib/librte_mempool/rte_mempool.h | 23 +++++++++++++++++++++--
test/test/test_mempool.c | 2 +-
3 files changed, 57 insertions(+), 7 deletions(-)
diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c
index d50dba4..43455a3 100644
--- a/lib/librte_mempool/rte_mempool.c
+++ b/lib/librte_mempool/rte_mempool.c
@@ -239,7 +239,8 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
*/
size_t
rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift,
- unsigned int flags)
+ unsigned int flags,
+ const struct rte_mempool_info *info)
{
size_t obj_per_page, pg_num, pg_sz;
unsigned int mask;
@@ -252,6 +253,17 @@ rte_mempool_xmem_size(uint32_t elt_num, size_t
total_elt_sz, uint32_t pg_shift,
if (total_elt_sz == 0)
return 0;
+ if (flags & MEMPOOL_F_CAPA_ALLOCATE_IN_CLUSTERS) {
+ unsigned int align_shift =
+ rte_bsf32(
+ rte_align32pow2(total_elt_sz *
+ info->cluster_size));
+ if (pg_shift < align_shift) {
+ return ((elt_num / info->cluster_size) + 2)
+ << align_shift;
+ }
+ }
+
if (pg_shift == 0)
return total_elt_sz * elt_num;
@@ -362,6 +374,7 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char
*vaddr,
void *opaque)
{
unsigned total_elt_sz;
+ unsigned int page_align_size = 0;
unsigned i = 0;
size_t off;
struct rte_mempool_memhdr *memhdr;
@@ -407,7 +420,11 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char
*vaddr,
memhdr->free_cb = free_cb;
memhdr->opaque = opaque;
- if (mp->flags & MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS)
+ if (mp->flags & MEMPOOL_F_CAPA_ALLOCATE_IN_CLUSTERS) {
+ page_align_size = rte_align32pow2(total_elt_sz *
+ mp->info.cluster_size);
+ off = RTE_PTR_ALIGN_CEIL(vaddr, page_align_size) - vaddr;
+ } else if (mp->flags & MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS)
/* align object start address to a multiple of total_elt_sz */
off = total_elt_sz - ((uintptr_t)vaddr % total_elt_sz);
else if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN)
@@ -424,6 +441,10 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char
*vaddr,
mempool_add_elem(mp, (char *)vaddr + off, iova + off);
off += mp->elt_size + mp->trailer_size;
i++;
+ if ((mp->flags & MEMPOOL_F_CAPA_ALLOCATE_IN_CLUSTERS) &&
+ (i % mp->info.cluster_size) == 0)
+ off = RTE_PTR_ALIGN_CEIL((char *)vaddr + off,
+ page_align_size) - vaddr;
}
/* not enough room to store one object */
@@ -579,6 +600,16 @@ rte_mempool_populate_default(struct rte_mempool *mp)
if ((ret < 0) && (ret != -ENOTSUP))
return ret;
+ ret = rte_mempool_ops_get_info(mp, &mp->info);
+ if ((ret < 0) && (ret != -ENOTSUP))
+ return ret;
+ if (ret == -ENOTSUP)
+ mp->info.cluster_size = 0;
+
+ if ((mp->info.cluster_size == 0) &&
+ (mp_flags & MEMPOOL_F_CAPA_ALLOCATE_IN_CLUSTERS))
+ return -EINVAL;
+
/* update mempool capabilities */
mp->flags |= mp_flags;
@@ -595,7 +626,7 @@ rte_mempool_populate_default(struct rte_mempool *mp)
total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) {
size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift,
- mp->flags);
+ mp->flags, &mp->info);
ret = snprintf(mz_name, sizeof(mz_name),
RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id);
@@ -653,7 +684,7 @@ get_anon_size(const struct rte_mempool *mp)
pg_shift = rte_bsf32(pg_sz);
total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
size = rte_mempool_xmem_size(mp->size, total_elt_sz, pg_shift,
- mp->flags);
+ mp->flags, &mp->info);
return size;
}
diff --git a/lib/librte_mempool/rte_mempool.h b/lib/librte_mempool/rte_mempool.h
index 3c59d36..9bcb8b7 100644
--- a/lib/librte_mempool/rte_mempool.h
+++ b/lib/librte_mempool/rte_mempool.h
@@ -220,7 +220,10 @@ struct rte_mempool_memhdr {
/*
* Additional information about the mempool
*/
-struct rte_mempool_info;
+struct rte_mempool_info {
+ /** Number of objects in a cluster */
+ unsigned int cluster_size;
+};
/**
* The RTE mempool structure.
@@ -265,6 +268,7 @@ struct rte_mempool {
struct rte_mempool_objhdr_list elt_list; /**< List of objects in pool */
uint32_t nb_mem_chunks; /**< Number of memory chunks */
struct rte_mempool_memhdr_list mem_list; /**< List of memory chunks */
+ struct rte_mempool_info info; /**< Additional mempool info */
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
/** Per-lcore statistics. */
@@ -298,6 +302,17 @@ struct rte_mempool {
#define MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS 0x0080
/**
+ * This capability flag is advertised by a mempool handler. Used for a case
+ * where mempool driver wants clusters of objects start at a power-of-two
+ * boundary
+ *
+ * Note:
+ * - This flag should not be passed by application.
+ * Flag used for mempool driver only.
+ */
+#define MEMPOOL_F_CAPA_ALLOCATE_IN_CLUSTERS 0x0100
+
+/**
* @internal When debug is enabled, store some statistics.
*
* @param mp
@@ -1605,11 +1620,15 @@ uint32_t rte_mempool_calc_obj_size(uint32_t elt_size,
uint32_t flags,
* LOG2 of the physical pages size. If set to 0, ignore page boundaries.
* @param flags
* The mempool flags.
+ * @param info
+ * A pointer to the mempool's additional info (may be NULL unless
+ * MEMPOOL_F_CAPA_ALLOCATE_IN_CLUSTERS is set in @arg flags)
* @return
* Required memory size aligned at page boundary.
*/
size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz,
- uint32_t pg_shift, unsigned int flags);
+ uint32_t pg_shift, unsigned int flags,
+ const struct rte_mempool_info *info);
/**
* Get the size of memory required to store mempool elements.
diff --git a/test/test/test_mempool.c b/test/test/test_mempool.c
index 37ead50..f4bb9a9 100644
--- a/test/test/test_mempool.c
+++ b/test/test/test_mempool.c
@@ -485,7 +485,7 @@ test_mempool_xmem_misc(void)
elt_num = MAX_KEEP;
total_size = rte_mempool_calc_obj_size(MEMPOOL_ELT_SIZE, 0, NULL);
sz = rte_mempool_xmem_size(elt_num, total_size, MEMPOOL_PG_SHIFT_MAX,
- 0);
+ 0, NULL);
usz = rte_mempool_xmem_usage(NULL, elt_num, total_size, 0, 1,
MEMPOOL_PG_SHIFT_MAX, 0);
--
2.7.4
