<snip>
>
> From: Elena Agostini <eagost...@nvidia.com>
>
> The new library gpudev is for dealing with GPU from a DPDK application in a
> vendor-agnostic way.
It would be good to explain how the application using GPU+DPDK would look like.
Which parts of the workload need DPDK's support?
Any requirements on co-existence of GPU with other accelerators?
>
> As a first step, the features are focused on memory management.
> A function allows to allocate memory inside the GPU, while another one
> allows to use main (CPU) memory from the GPU.
Is this memory for packet buffers or something else?
>
> The infrastructure is prepared to welcome drivers in drivers/gpu/ as the
> upcoming NVIDIA one, implementing the gpudev API.
> Other additions planned for next revisions:
> - C implementation file
> - guide documentation
> - unit tests
> - integration in testpmd to enable Rx/Tx to/from GPU memory.
>
> The next step should focus on GPU processing task control.
>
> Signed-off-by: Elena Agostini <eagost...@nvidia.com>
> Signed-off-by: Thomas Monjalon <tho...@monjalon.net>
> ---
> .gitignore | 1 +
> MAINTAINERS | 6 +
> doc/api/doxy-api-index.md | 1 +
> doc/api/doxy-api.conf.in | 1 +
> doc/guides/conf.py | 8 ++
> doc/guides/gpus/features/default.ini | 13 ++
> doc/guides/gpus/index.rst | 11 ++
> doc/guides/gpus/overview.rst | 7 +
> doc/guides/index.rst | 1 +
> doc/guides/prog_guide/gpu.rst | 5 +
> doc/guides/prog_guide/index.rst | 1 +
> drivers/gpu/meson.build | 4 +
> drivers/meson.build | 1 +
> lib/gpudev/gpu_driver.h | 44 +++++++
> lib/gpudev/meson.build | 9 ++
> lib/gpudev/rte_gpudev.h | 183 +++++++++++++++++++++++++++
> lib/gpudev/version.map | 11 ++
> lib/meson.build | 1 +
> 18 files changed, 308 insertions(+)
> create mode 100644 doc/guides/gpus/features/default.ini
> create mode 100644 doc/guides/gpus/index.rst create mode 100644
> doc/guides/gpus/overview.rst create mode 100644
> doc/guides/prog_guide/gpu.rst create mode 100644
> drivers/gpu/meson.build create mode 100644 lib/gpudev/gpu_driver.h
> create mode 100644 lib/gpudev/meson.build create mode 100644
> lib/gpudev/rte_gpudev.h create mode 100644 lib/gpudev/version.map
>
> diff --git a/.gitignore b/.gitignore
> index b19c0717e6..49494e0c6c 100644
> --- a/.gitignore
> +++ b/.gitignore
> @@ -14,6 +14,7 @@ doc/guides/compressdevs/overview_feature_table.txt
> doc/guides/regexdevs/overview_feature_table.txt
> doc/guides/vdpadevs/overview_feature_table.txt
> doc/guides/bbdevs/overview_feature_table.txt
> +doc/guides/gpus/overview_feature_table.txt
>
> # ignore generated ctags/cscope files
> cscope.out.po
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 5877a16971..c4755dfe9a 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -452,6 +452,12 @@ F: app/test-regex/
> F: doc/guides/prog_guide/regexdev.rst
> F: doc/guides/regexdevs/features/default.ini
>
> +GPU API - EXPERIMENTAL
> +M: Elena Agostini <eagost...@nvidia.com>
> +F: lib/gpudev/
> +F: doc/guides/prog_guide/gpu.rst
> +F: doc/guides/gpus/features/default.ini
> +
> Eventdev API
> M: Jerin Jacob <jer...@marvell.com>
> T: git://dpdk.org/next/dpdk-next-eventdev
> diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md index
> 1992107a03..bd10342ca2 100644
> --- a/doc/api/doxy-api-index.md
> +++ b/doc/api/doxy-api-index.md
> @@ -21,6 +21,7 @@ The public API headers are grouped by topics:
> [compressdev] (@ref rte_compressdev.h),
> [compress] (@ref rte_comp.h),
> [regexdev] (@ref rte_regexdev.h),
> + [gpudev] (@ref rte_gpudev.h),
> [eventdev] (@ref rte_eventdev.h),
> [event_eth_rx_adapter] (@ref rte_event_eth_rx_adapter.h),
> [event_eth_tx_adapter] (@ref rte_event_eth_tx_adapter.h),
> diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in index
> 325a0195c6..831b9a6b33 100644
> --- a/doc/api/doxy-api.conf.in
> +++ b/doc/api/doxy-api.conf.in
> @@ -40,6 +40,7 @@ INPUT = @TOPDIR@/doc/api/doxy-api-
> index.md \
> @TOPDIR@/lib/eventdev \
> @TOPDIR@/lib/fib \
> @TOPDIR@/lib/flow_classify \
> + @TOPDIR@/lib/gpudev \
> @TOPDIR@/lib/graph \
> @TOPDIR@/lib/gro \
> @TOPDIR@/lib/gso \ diff --git a/doc/guides/conf.py
> b/doc/guides/conf.py index 67d2dd62c7..7930da9ceb 100644
> --- a/doc/guides/conf.py
> +++ b/doc/guides/conf.py
> @@ -152,6 +152,9 @@ def generate_overview_table(output_filename,
> table_id, section, table_name, titl
> name = ini_filename[:-4]
> name = name.replace('_vf', 'vf')
> pmd_names.append(name)
> + if not pmd_names:
> + # Add an empty column if table is empty (required by RST syntax)
> + pmd_names.append(' ')
>
> # Pad the table header names.
> max_header_len = len(max(pmd_names, key=len)) @@ -388,6 +391,11
> @@ def setup(app):
> 'Features',
> 'Features availability in bbdev drivers',
> 'Feature')
> + table_file = dirname(__file__) + '/gpus/overview_feature_table.txt'
> + generate_overview_table(table_file, 1,
> + 'Features',
> + 'Features availability in GPU drivers',
> + 'Feature')
>
> if LooseVersion(sphinx_version) < LooseVersion('1.3.1'):
> print('Upgrade sphinx to version >= 1.3.1 for '
> diff --git a/doc/guides/gpus/features/default.ini
> b/doc/guides/gpus/features/default.ini
> new file mode 100644
> index 0000000000..c363447b0d
> --- /dev/null
> +++ b/doc/guides/gpus/features/default.ini
> @@ -0,0 +1,13 @@
> +;
> +; Features of a GPU driver.
> +;
> +; This file defines the features that are valid for inclusion in ; the
> +other driver files and also the order that they appear in ; the
> +features table in the documentation. The feature description ; string
> +should not exceed feature_str_len defined in conf.py.
> +;
> +[Features]
> +Get device info =
> +Share CPU memory with GPU =
> +Allocate GPU memory =
> +Free memory =
> diff --git a/doc/guides/gpus/index.rst b/doc/guides/gpus/index.rst new file
> mode 100644 index 0000000000..f9c62aeb36
> --- /dev/null
> +++ b/doc/guides/gpus/index.rst
> @@ -0,0 +1,11 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> + Copyright 2021 NVIDIA Corporation & Affiliates
> +
> +GPU Drivers
> +===========
> +
> +.. toctree::
> + :maxdepth: 2
> + :numbered:
> +
> + overview
> diff --git a/doc/guides/gpus/overview.rst b/doc/guides/gpus/overview.rst
> new file mode 100644 index 0000000000..e7f985e98b
> --- /dev/null
> +++ b/doc/guides/gpus/overview.rst
> @@ -0,0 +1,7 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> + Copyright 2021 NVIDIA Corporation & Affiliates
> +
> +Overview of GPU Drivers
> +=======================
> +
> +.. include:: overview_feature_table.txt
> diff --git a/doc/guides/index.rst b/doc/guides/index.rst index
> 857f0363d3..ee4d79a4eb 100644
> --- a/doc/guides/index.rst
> +++ b/doc/guides/index.rst
> @@ -21,6 +21,7 @@ DPDK documentation
> compressdevs/index
> vdpadevs/index
> regexdevs/index
> + gpus/index
> eventdevs/index
> rawdevs/index
> mempool/index
> diff --git a/doc/guides/prog_guide/gpu.rst b/doc/guides/prog_guide/gpu.rst
> new file mode 100644 index 0000000000..54f9fa8300
> --- /dev/null
> +++ b/doc/guides/prog_guide/gpu.rst
> @@ -0,0 +1,5 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> + Copyright 2021 NVIDIA Corporation & Affiliates
> +
> +GPU Library
> +===========
> diff --git a/doc/guides/prog_guide/index.rst
> b/doc/guides/prog_guide/index.rst index 2dce507f46..dfddf90b51 100644
> --- a/doc/guides/prog_guide/index.rst
> +++ b/doc/guides/prog_guide/index.rst
> @@ -27,6 +27,7 @@ Programmer's Guide
> cryptodev_lib
> compressdev
> regexdev
> + gpu
> rte_security
> rawdev
> link_bonding_poll_mode_drv_lib
> diff --git a/drivers/gpu/meson.build b/drivers/gpu/meson.build new file mode
> 100644 index 0000000000..5189950616
> --- /dev/null
> +++ b/drivers/gpu/meson.build
> @@ -0,0 +1,4 @@
> +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2021 NVIDIA
> +Corporation & Affiliates
> +
> +drivers = []
> diff --git a/drivers/meson.build b/drivers/meson.build index
> bc6f4f567f..f607040d79 100644
> --- a/drivers/meson.build
> +++ b/drivers/meson.build
> @@ -18,6 +18,7 @@ subdirs = [
> 'vdpa', # depends on common, bus and mempool.
> 'event', # depends on common, bus, mempool and net.
> 'baseband', # depends on common and bus.
> + 'gpu', # depends on common and bus.
> ]
>
> if meson.is_cross_build()
> diff --git a/lib/gpudev/gpu_driver.h b/lib/gpudev/gpu_driver.h new file mode
> 100644 index 0000000000..5ff609e49d
> --- /dev/null
> +++ b/lib/gpudev/gpu_driver.h
> @@ -0,0 +1,44 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 NVIDIA Corporation & Affiliates */
> +
> +#ifndef GPU_DRIVER_H
> +#define GPU_DRIVER_H
> +
> +#include <stdint.h>
> +
> +#include <rte_common.h>
> +
> +#include "rte_gpudev.h"
> +
> +struct rte_gpu_dev;
> +
> +typedef int (*gpu_malloc_t)(struct rte_gpu_dev *dev, size_t size, void
> +**ptr); typedef int (*gpu_free_t)(struct rte_gpu_dev *dev, void *ptr);
> +
> +struct rte_gpu_dev {
> + /* Backing device. */
> + struct rte_device *device;
> + /* GPU info structure. */
> + struct rte_gpu_info info;
> + /* Counter of processes using the device. */
> + uint16_t process_cnt;
> + /* If device is currently used or not. */
> + enum rte_gpu_state state;
> + /* FUNCTION: Allocate memory on the GPU. */
> + gpu_malloc_t gpu_malloc;
> + /* FUNCTION: Allocate memory on the CPU visible from the GPU. */
> + gpu_malloc_t gpu_malloc_visible;
> + /* FUNCTION: Free allocated memory on the GPU. */
> + gpu_free_t gpu_free;
> + /* Device interrupt handle. */
> + struct rte_intr_handle *intr_handle;
> + /* Driver-specific private data. */
> + void *dev_private;
> +} __rte_cache_aligned;
> +
> +struct rte_gpu_dev *rte_gpu_dev_allocate(const char *name); struct
> +rte_gpu_dev *rte_gpu_dev_get_by_name(const char *name); int
> +rte_gpu_dev_release(struct rte_gpu_dev *gpudev);
> +
> +#endif /* GPU_DRIVER_H */
> diff --git a/lib/gpudev/meson.build b/lib/gpudev/meson.build new file mode
> 100644 index 0000000000..f05459e18d
> --- /dev/null
> +++ b/lib/gpudev/meson.build
> @@ -0,0 +1,9 @@
> +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2021 NVIDIA
> +Corporation & Affiliates
> +
> +headers = files(
> + 'rte_gpudev.h',
> +)
> +
> +sources = files(
> +)
> diff --git a/lib/gpudev/rte_gpudev.h b/lib/gpudev/rte_gpudev.h new file mode
> 100644 index 0000000000..b12f35c17e
> --- /dev/null
> +++ b/lib/gpudev/rte_gpudev.h
> @@ -0,0 +1,183 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 NVIDIA Corporation & Affiliates */
> +
> +#ifndef RTE_GPUDEV_H
> +#define RTE_GPUDEV_H
> +
> +#include <stdint.h>
> +#include <stdbool.h>
> +
> +#include <rte_common.h>
> +
> +/**
> + * @file
> + * Generic library to interact with a GPU.
> + *
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +/** Maximum number of GPU engines. */
> +#define RTE_GPU_MAX_DEVS UINT16_C(32)
> +/** Maximum length of device name. */
> +#define RTE_GPU_NAME_MAX_LEN 128
> +
> +/** Flags indicate current state of GPU device. */ enum rte_gpu_state {
> + RTE_GPU_STATE_UNUSED, /**< not initialized */
> + RTE_GPU_STATE_INITIALIZED, /**< initialized */
> +};
> +
> +/** Store a list of info for a given GPU. */ struct rte_gpu_info {
> + /** GPU device ID. */
> + uint16_t gpu_id;
> + /** Unique identifier name. */
> + char name[RTE_GPU_NAME_MAX_LEN];
> + /** Total memory available on device. */
> + size_t total_memory;
> + /** Total processors available on device. */
> + int processor_count;
> +};
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Returns the number of GPUs detected and associated to DPDK.
> + *
> + * @return
> + * The number of available GPUs.
> + */
> +__rte_experimental
> +uint16_t rte_gpu_dev_count_avail(void);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Check if the device is valid and initialized in DPDK.
> + *
> + * @param gpu_id
> + * The input GPU ID.
> + *
> + * @return
> + * - True if gpu_id is a valid and initialized GPU.
> + * - False otherwise.
> + */
> +__rte_experimental
> +bool rte_gpu_dev_is_valid(uint16_t gpu_id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Get the GPU ID of the next valid GPU initialized in DPDK.
> + *
> + * @param gpu_id
> + * The initial GPU ID to start the research.
> + *
> + * @return
> + * Next GPU ID corresponding to a valid and initialized GPU device.
> + */
> +__rte_experimental
> +uint16_t rte_gpu_dev_find_next(uint16_t gpu_id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Macro to iterate over all valid GPUs.
> + *
> + * @param gpu_id
> + * The ID of the next possible valid GPU.
> + * @return
> + * Next valid GPU ID, RTE_GPU_MAX_DEVS if there is none.
> + */
> +#define RTE_GPU_FOREACH_DEV(gpu_id) \
> + for (gpu_id = rte_gpu_find_next(0); \
> + gpu_id < RTE_GPU_MAX_DEVS; \
> + gpu_id = rte_gpu_find_next(gpu_id + 1))
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Return GPU specific info.
> + *
> + * @param gpu_id
> + * GPU ID to get info.
> + * @param info
> + * Memory structure to fill with the info.
> + *
> + * @return
> + * 0 on success, -1 otherwise.
> + */
> +__rte_experimental
> +int rte_gpu_dev_info_get(uint16_t gpu_id, struct rte_gpu_info **info);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Allocate a chunk of memory on the GPU.
> + *
> + * @param gpu_id
> + * GPU ID to allocate memory.
> + * @param size
> + * Number of bytes to allocate.
> + * @param ptr
> + * Pointer to store the address of the allocated memory.
> + *
> + * @return
> + * 0 on success, -1 otherwise.
> + */
> +__rte_experimental
> +int rte_gpu_malloc(uint16_t gpu_id, size_t size, void **ptr);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Allocate a chunk of memory on the CPU that is visible from the GPU.
> + *
> + * @param gpu_id
> + * Reference GPU ID.
> + * @param size
> + * Number of bytes to allocate.
> + * @param ptr
> + * Pointer to store the address of the allocated memory.
> + *
> + * @return
> + * 0 on success, -1 otherwise.
> + */
> +__rte_experimental
> +int rte_gpu_malloc_visible(uint16_t gpu_id, size_t size, void **ptr);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Deallocate a chunk of memory allocated with rte_gpu_malloc*.
> + *
> + * @param gpu_id
> + * Reference GPU ID.
> + * @param ptr
> + * Pointer to the memory area to be deallocated.
> + *
> + * @return
> + * 0 on success, -1 otherwise.
> + */
> +__rte_experimental
> +int rte_gpu_free(uint16_t gpu_id, void *ptr);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* RTE_GPUDEV_H */
> diff --git a/lib/gpudev/version.map b/lib/gpudev/version.map new file mode
> 100644 index 0000000000..9e0f218e8b
> --- /dev/null
> +++ b/lib/gpudev/version.map
> @@ -0,0 +1,11 @@
> +EXPERIMENTAL {
> + global:
> +
> + rte_gpu_dev_count_avail;
> + rte_gpu_dev_find_next;
> + rte_gpu_dev_info_get;
> + rte_gpu_dev_is_valid;
> + rte_gpu_free;
> + rte_gpu_malloc;
> + rte_gpu_malloc_visible;
> +};
> diff --git a/lib/meson.build b/lib/meson.build index 4a64756a68..ffefc64c69
> 100644
> --- a/lib/meson.build
> +++ b/lib/meson.build
> @@ -33,6 +33,7 @@ libraries = [
> 'distributor',
> 'efd',
> 'eventdev',
> + 'gpudev',
> 'gro',
> 'gso',
> 'ip_frag',
> --
> 2.31.1
