On Thu, Nov 29, 2018 at 01:48:35PM +0000, Anatoly Burakov wrote:
> Add multiprocess support for externally allocated memory areas that
> are not added to DPDK heap (and add relevant doc sections).
>
> Signed-off-by: Anatoly Burakov <anatoly.bura...@intel.com>
> ---
Acked-by: Yongseok Koh <ys...@mellanox.com>
Thanks
> .../prog_guide/env_abstraction_layer.rst | 3 +
> lib/librte_eal/common/eal_common_memory.c | 42 +++++++++++++
> lib/librte_eal/common/include/rte_memory.h | 59 +++++++++++++++++++
> lib/librte_eal/rte_eal_version.map | 2 +
> 4 files changed, 106 insertions(+)
>
> diff --git a/doc/guides/prog_guide/env_abstraction_layer.rst
> b/doc/guides/prog_guide/env_abstraction_layer.rst
> index d7799b626..b0491bf2d 100644
> --- a/doc/guides/prog_guide/env_abstraction_layer.rst
> +++ b/doc/guides/prog_guide/env_abstraction_layer.rst
> @@ -276,11 +276,14 @@ The expected workflow is as follows:
> * Register memory within DPDK
> - If IOVA table is not specified, IOVA addresses will be assumed to be
> unavailable
> + - Other processes must attach to the memory area before they can use it
> * Perform DMA mapping with ``rte_vfio_dma_map`` if needed
> * Use the memory area in your application
> * If memory area is no longer needed, it can be unregistered
> - If the area was mapped for DMA, unmapping must be performed before
> unregistering memory
> + - Other processes must detach from the memory area before it can be
> + unregistered
>
> Since these externally allocated memory areas will not be managed by DPDK,
> it is
> therefore up to the user application to decide how to use them and what to do
> diff --git a/lib/librte_eal/common/eal_common_memory.c
> b/lib/librte_eal/common/eal_common_memory.c
> index a2e085ae8..67b445c31 100644
> --- a/lib/librte_eal/common/eal_common_memory.c
> +++ b/lib/librte_eal/common/eal_common_memory.c
> @@ -849,6 +849,48 @@ rte_extmem_unregister(void *va_addr, size_t len)
> return ret;
> }
>
> +static int
> +sync_memory(void *va_addr, size_t len, bool attach)
> +{
> + struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
> + struct rte_memseg_list *msl;
> + int ret = 0;
> +
> + if (va_addr == NULL || len == 0) {
> + rte_errno = EINVAL;
> + return -1;
> + }
> + rte_rwlock_write_lock(&mcfg->memory_hotplug_lock);
> +
> + /* find our segment */
> + msl = malloc_heap_find_external_seg(va_addr, len);
> + if (msl == NULL) {
> + rte_errno = ENOENT;
> + ret = -1;
> + goto unlock;
> + }
> + if (attach)
> + ret = rte_fbarray_attach(&msl->memseg_arr);
> + else
> + ret = rte_fbarray_detach(&msl->memseg_arr);
> +
> +unlock:
> + rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock);
> + return ret;
> +}
> +
> +int __rte_experimental
> +rte_extmem_attach(void *va_addr, size_t len)
> +{
> + return sync_memory(va_addr, len, true);
> +}
> +
> +int __rte_experimental
> +rte_extmem_detach(void *va_addr, size_t len)
> +{
> + return sync_memory(va_addr, len, false);
> +}
> +
> /* init memory subsystem */
> int
> rte_eal_memory_init(void)
> diff --git a/lib/librte_eal/common/include/rte_memory.h
> b/lib/librte_eal/common/include/rte_memory.h
> index 4a43c1a9e..050bb6d8e 100644
> --- a/lib/librte_eal/common/include/rte_memory.h
> +++ b/lib/librte_eal/common/include/rte_memory.h
> @@ -435,6 +435,10 @@ rte_memseg_get_fd_offset_thread_unsafe(const struct
> rte_memseg *ms,
> * @note This API will not perform any DMA mapping. It is expected that user
> * will do that themselves.
> *
> + * @note Before accessing this memory in other processes, it needs to be
> + * attached in each of those processes by calling ``rte_extmem_attach`` in
> + * each other process.
> + *
> * @param va_addr
> * Start of virtual area to register
> * @param len
> @@ -472,6 +476,9 @@ rte_extmem_register(void *va_addr, size_t len, rte_iova_t
> iova_addrs[],
> * @note This API will not perform any DMA unmapping. It is expected that
> user
> * will do that themselves.
> *
> + * @note Before calling this function, all other processes must call
> + * ``rte_extmem_detach`` to detach from the memory area.
> + *
> * @param va_addr
> * Start of virtual area to unregister
> * @param len
> @@ -486,6 +493,58 @@ rte_extmem_register(void *va_addr, size_t len,
> rte_iova_t iova_addrs[],
> int __rte_experimental
> rte_extmem_unregister(void *va_addr, size_t len);
>
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Attach to external memory chunk registered in another process.
> + *
> + * @note Using this API is mutually exclusive with ``rte_malloc`` family of
> + * API's.
> + *
> + * @note This API will not perform any DMA mapping. It is expected that user
> + * will do that themselves.
> + *
> + * @param va_addr
> + * Start of virtual area to register
> + * @param len
> + * Length of virtual area to register
> + *
> + * @return
> + * - 0 on success
> + * - -1 in case of error, with rte_errno set to one of the following:
> + * EINVAL - one of the parameters was invalid
> + * ENOENT - memory chunk was not found
> + */
> +int __rte_experimental
> +rte_extmem_attach(void *va_addr, size_t len);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Detach from external memory chunk registered in another process.
> + *
> + * @note Using this API is mutually exclusive with ``rte_malloc`` family of
> + * API's.
> + *
> + * @note This API will not perform any DMA unmapping. It is expected that
> user
> + * will do that themselves.
> + *
> + * @param va_addr
> + * Start of virtual area to unregister
> + * @param len
> + * Length of virtual area to unregister
> + *
> + * @return
> + * - 0 on success
> + * - -1 in case of error, with rte_errno set to one of the following:
> + * EINVAL - one of the parameters was invalid
> + * ENOENT - memory chunk was not found
> + */
> +int __rte_experimental
> +rte_extmem_detach(void *va_addr, size_t len);
> +
> /**
> * Dump the physical memory layout to a file.
> *
> diff --git a/lib/librte_eal/rte_eal_version.map
> b/lib/librte_eal/rte_eal_version.map
> index 593691a14..eb5f7b9cb 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -296,6 +296,8 @@ EXPERIMENTAL {
> rte_devargs_remove;
> rte_devargs_type_count;
> rte_eal_cleanup;
> + rte_extmem_attach;
> + rte_extmem_detach;
> rte_extmem_register;
> rte_extmem_unregister;
> rte_fbarray_attach;
> --
> 2.17.1
