Hi Wei,
Let's align and review it offline.
Thanks,
Rosen
> -----Original Message-----
> From: Huang, Wei <wei.hu...@intel.com>
> Sent: Monday, January 04, 2021 9:18
> To: Xu, Rosen <rosen...@intel.com>; dev@dpdk.org; Zhang, Qi Z
> <qi.z.zh...@intel.com>
> Cc: sta...@dpdk.org; Zhang, Tianfei <tianfei.zh...@intel.com>
> Subject: RE: [PATCH v4 3/4] raw/ifpga: add opae API for Cyborg
>
> Hi Rosen,
>
> I cannot change "master" to "primary", how to fix it ?
>
> Wei
>
> -----Original Message-----
> From: Xu, Rosen <rosen...@intel.com>
> Sent: Friday, January 1, 2021 22:39
> To: Huang, Wei <wei.hu...@intel.com>; dev@dpdk.org; Zhang, Qi Z
> <qi.z.zh...@intel.com>
> Cc: sta...@dpdk.org; Zhang, Tianfei <tianfei.zh...@intel.com>
> Subject: RE: [PATCH v4 3/4] raw/ifpga: add opae API for Cyborg
>
> Hi Wei,
>
> Could you pls fix coding style issues? Thanks a lot.
>
> Thanks,
> Rosen
>
> > -----Original Message-----
> > From: Huang, Wei <wei.hu...@intel.com>
> > Sent: Wednesday, December 30, 2020 15:46
> > To: dev@dpdk.org; Xu, Rosen <rosen...@intel.com>; Zhang, Qi Z
> > <qi.z.zh...@intel.com>
> > Cc: sta...@dpdk.org; Zhang, Tianfei <tianfei.zh...@intel.com>; Huang,
> Wei
> > <wei.hu...@intel.com>
> > Subject: [PATCH v4 3/4] raw/ifpga: add opae API for Cyborg
> >
> > Cyborg is part of OpenStack, it needs some OPAE type APIs to manage
> > PACs (Programmable Acceleration Card) with Intel FPGA. Below major
> > functions are added to meets Cyborg requirements.
> > 1. opae_init_eal() set up EAL environment.
> > 2. opae_cleanup_eal() clean up EAL environment.
> > 3. opae_enumerate() searches PAC with specific FPGA.
> > 4. opae_get_property() gets properties of FPGA.
> > 5. opae_partial_reconfigure() perform partial configuration on FPGA.
> > 6. opae_get_image_info() gets information of image file.
> > 7. opae_update_flash() updates FPGA flash with specific image file.
> > 8. opae_cancel_flash_update() cancel process of FPGA flash update.
> > 9. opae_probe_device() manually probe specific FPGA with ifpga driver.
> > 10. opae_remove_device() manually remove specific FPGA from ifpga
> driver.
> > 11. opae_bind_driver() binds specific FPGA with specified kernel driver.
> > 12. opae_unbind_driver() unbinds specific FPGA from kernel driver.
> > 13. opae_reboot_device() reboots specific FPGA (do reconfiguration).
> >
> > Signed-off-by: Wei Huang <wei.hu...@intel.com>
> > ---
> > v2: fix typo in commit log and ifpga_opae_api.h
> > ---
> > v3: fix coding style issue in ifpga_opae_api.c
> > ---
> > v4: enclose macro PCI_EXT_CAP_ID in parentheses
> > ---
> > drivers/raw/ifpga/ifpga_opae_api.c | 1801
> > ++++++++++++++++++++++++++++
> > drivers/raw/ifpga/ifpga_opae_api.h | 245 ++++
> > drivers/raw/ifpga/ifpga_rawdev.c | 152 ++-
> > drivers/raw/ifpga/ifpga_rawdev.h | 15 +
> > drivers/raw/ifpga/meson.build | 4 +-
> > 5 files changed, 2210 insertions(+), 7 deletions(-)
> > create mode 100644 drivers/raw/ifpga/ifpga_opae_api.c
> > create mode 100644 drivers/raw/ifpga/ifpga_opae_api.h
> >
> > diff --git a/drivers/raw/ifpga/ifpga_opae_api.c
> > b/drivers/raw/ifpga/ifpga_opae_api.c
> > new file mode 100644
> > index 000000000..9210092ab
> > --- /dev/null
> > +++ b/drivers/raw/ifpga/ifpga_opae_api.c
> > @@ -0,0 +1,1801 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2020 Intel Corporation
> > + */
> > +
> > +#include <stdlib.h>
> > +#include <string.h>
> > +#include <dirent.h>
> > +#include <fcntl.h>
> > +#include <glob.h>
> > +#include <unistd.h>
> > +#include <sys/stat.h>
> > +#include <rte_eal.h>
> > +#include <rte_bus_pci.h>
> > +#include <rte_rawdev_pmd.h>
> > +#include "base/opae_hw_api.h"
> > +#include "base/ifpga_sec_mgr.h"
> > +#include "ifpga_rawdev.h"
> > +#include "ifpga_opae_api.h"
> > +
> > +
> > +int opae_log_level;
> > +FILE *opae_log_file;
> > +
> > +static opae_api_version api_ver = {21, 2, 0};
> > +static int eal_inited;
> > +static uint32_t dev_aer[2] = {0};
> > +
> > +static const char * const log_level_name[] = {"CRITICAL", "ERROR",
> > + "WARNING", "INFORMATION", "DEBUG"};
> > +static const char * const proc_type_name[] = {"NON-DPDK", "PRIMARY",
> > + "SECONDARY"};
> > +static const char * const platform_name[] = {"Vista Creek", "Rush Creek",
> > + "Darby Creek", "Lightning Creek"};
> > +static const char * const release_name[] = {"Pre-Alpha", "Alpha", "Beta",
> > "PV"};
> > +static const char * const interface_type[] = {"8x10G", "4x25G", "2x1x25G",
> > + "4x25G+2x25G", "2x2x25G", "2x1x25Gx2FVL", "1x2x25G"};
> > +static const char * const kdrv[] = {OPAE_KDRV_UNKNOWN,
> > OPAE_KDRV_IGB_UIO,
> > + OPAE_KDRV_VFIO_PCI, OPAE_KDRV_UIO_PCI};
> > +
> > +RTE_INIT(init_api_env)
> > +{
> > + eal_inited = 0;
> > + opae_log_level = OPAE_LOG_ERR;
> > + opae_log_file = NULL;
> > + ifpga_rawdev_logtype = 0;
> > +
> > + opae_log_info("API environment is initialized\n");
> > +}
> > +
> > +RTE_FINI(clean_api_env)
> > +{
> > + if (opae_log_file) {
> > + fclose(opae_log_file);
> > + opae_log_file = NULL;
> > + }
> > + opae_log_info("API environment is cleaned\n");
> > +}
> > +
> > +void opae_get_api_version(opae_api_version *version)
> > +{
> > + if (version)
> > + memcpy(version, &api_ver, sizeof(opae_api_version));
> > + opae_log_info("API version is %u.%u.%u\n",
> > + api_ver.major, api_ver.minor, api_ver.micro);
> > +}
> > +
> > +int opae_set_log_level(int level)
> > +{
> > + if ((level >= OPAE_LOG_API) && (level <= OPAE_LOG_DEBUG))
> > + opae_log_level = level;
> > + opae_log_api("Current log level is %s\n",
> > + log_level_name[opae_log_level]);
> > + return opae_log_level;
> > +}
> > +
> > +int opae_set_log_file(char *path, int clean)
> > +{
> > + FILE *f = NULL;
> > + time_t start;
> > + struct tm *lt = NULL;
> > +
> > + if (path) {
> > + if (clean)
> > + f = fopen(path, "w+");
> > + else
> > + f = fopen(path, "a+");
> > +
> > + if (f) {
> > + if (opae_log_file) {
> > + fclose(opae_log_file);
> > + opae_log_file = NULL;
> > + }
> > + time(&start);
> > + lt = localtime(&start);
> > + if (lt)
> > + fprintf(f, "================%d-%02d-
> > %02d "
> > +
> > "%02d:%02d:%02d================\n",
> > + 1900 + lt->tm_year, 1 + lt->tm_mon,
> > + lt->tm_mday,
> > + lt->tm_hour, lt->tm_min, lt->tm_sec);
> > + fflush(f);
> > + opae_log_file = f;
> > + } else {
> > + opae_log_err("failed to open log file \'%s\'\n", path);
> > + return -1;
> > + }
> > + } else {
> > + if (opae_log_file) {
> > + fclose(opae_log_file);
> > + opae_log_file = NULL;
> > + }
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +int opae_get_image_info(const char *image, opae_img_info *info)
> > +{
> > + int fd = -1;
> > + off_t file_size = 0;
> > + opae_img_hdr hdr;
> > + ssize_t read_size = 0;
> > + int ret = 0;
> > +
> > + if (!image || !info) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + fd = open(image, O_RDONLY);
> > + if (fd < 0) {
> > + opae_log_err("Failed to open \'%s\' for RD [e:%s]\n",
> > + image, strerror(errno));
> > + return -EIO;
> > + }
> > +
> > + file_size = lseek(fd, 0, SEEK_END);
> > + opae_log_dbg("Size of \'%s\' is %lu\n", image, file_size);
> > + if (file_size < (OPAE_IMG_HDR_SIZE + OPAE_IMG_PL_MIN_SIZE)) {
> > + opae_log_err("Size of \'%s\' is less than expected [e:%u]\n",
> > + image, OPAE_IMG_HDR_SIZE +
> > OPAE_IMG_PL_MIN_SIZE);
> > + ret = -EINVAL;
> > + goto close_fd;
> > + }
> > +
> > + /* read image header */
> > + lseek(fd, 0, SEEK_SET);
> > + read_size = read(fd, (void *)&hdr, sizeof(opae_img_hdr));
> > + if (read_size < 0) {
> > + opae_log_err("Failed to read from \'%s\' [e:%s]\n",
> > + image, strerror(errno));
> > + ret = -EIO;
> > + goto close_fd;
> > + }
> > + if ((size_t)read_size != sizeof(opae_img_hdr)) {
> > + opae_log_err("Read length %zd is not expected [e:%zu]\n",
> > + read_size, sizeof(opae_img_hdr));
> > + ret = -EIO;
> > + goto close_fd;
> > + }
> > +
> > + info->total_len = file_size;
> > + /* check signed image header */
> > + if (hdr.magic == OPAE_IMG_BLK0_MAGIC) {
> > + info->type = OPAE_IMG_TYPE(hdr.payload_type);
> > + info->subtype = OPAE_IMG_SUBTYPE(hdr.payload_type);
> > + info->payload_offset = OPAE_IMG_HDR_SIZE;
> > + info->payload_len = hdr.payload_len;
> > + } else {
> > + opae_log_err("Image \'%s\' can not be recognized\n",
> > image);
> > + ret = -EINVAL;
> > + }
> > +close_fd:
> > + close(fd);
> > + return ret;
> > +}
> > +
> > +static int write_file(char *path, char *buf, int size)
> > +{
> > + int fd = -1;
> > + ssize_t n = 0;
> > +
> > + if (!path || !buf || (size <= 0))
> > + return -EINVAL;
> > +
> > + fd = open(path, O_WRONLY);
> > + if (fd < 0) {
> > + opae_log_err("Failed to open \'%s\' for WR [e:%s]\n",
> > + path, strerror(errno));
> > + return -EIO;
> > + }
> > + opae_log_dbg("Write \"%s\" to \'%s\'\n", buf, path);
> > + n = write(fd, buf, size);
> > + if (n < size) {
> > + opae_log_err("Failed to write to \'%s\' [e:%s]\n",
> > + path, strerror(errno));
> > + close(fd);
> > + return -EIO;
> > + }
> > + close(fd);
> > +
> > + return 0;
> > +}
> > +
> > +static int read_file(char *path, char *buf, int size)
> > +{
> > + int fd = -1;
> > + ssize_t n = 0;
> > +
> > + if (!path || !buf || (size <= 0))
> > + return -EINVAL;
> > +
> > + fd = open(path, O_RDONLY);
> > + if (fd < 0) {
> > + opae_log_err("Failed to open \'%s\' for RD [e:%s]\n",
> > + path, strerror(errno));
> > + return -EIO;
> > + }
> > + n = read(fd, buf, size);
> > + if (n < 0) {
> > + opae_log_err("Failed to read from \'%s\' [e:%s]\n",
> > + path, strerror(errno));
> > + close(fd);
> > + return -EIO;
> > + }
> > + close(fd);
> > +
> > + if (n > 0)
> > + buf[n-1] = 0;
> > +
> > + opae_log_dbg("Read \"%s\" from \'%s\'\n", buf, path);
> > + return 0;
> > +}
> > +
> > +int opae_get_proc_type(void)
> > +{
> > + int type = -1;
> > +
> > + if (eal_inited) {
> > + if (rte_eal_process_type() == RTE_PROC_PRIMARY)
> > + type = 0;
> > + else
> > + type = 1;
> > + }
> > + opae_log_info("Current process type is %s\n",
> > proc_type_name[type+1]);
> > +
> > + return type;
> > +}
> > +
> > +static bool check_eal(int inited)
> > +{
> > + if (!eal_inited) {
> > + if (inited) {
> > + opae_log_warn("EAL is not initialized\n");
> > + return 0;
> > + }
> > + } else {
> > + if (!inited) {
> > + opae_log_warn("EAL is already initialized\n");
> > + return 0;
> > + }
> > + }
> > +
> > + return 1;
> > +}
> > +
> > +int opae_init_eal(int argc, char **argv)
> > +{
> > + int ret = 0;
> > +
> > + if (!check_eal(0))
> > + return ret;
> > +
> > + ret = rte_eal_init(argc, argv);
> > + if (ret < 0) {
> > + if (rte_errno == EALREADY) {
> > + eal_inited = 1;
> > + return 0;
> > + }
> > + opae_log_err("Cannot initialize EAL [e:%d]\n", ret);
> > + if (rte_eal_cleanup())
> > + opae_log_warn("EAL could not release all
> > resources\n");
> > + } else {
> > + eal_inited = 1;
> > + opae_log_info("Initialize EAL done\n");
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +int opae_cleanup_eal(void)
> > +{
> > + int ret = 0;
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + ifpga_rawdev_cleanup();
> > +
> > + ret = rte_eal_cleanup();
> > + if (ret)
> > + opae_log_err("Failed to cleanup EAL [e:%d]\n", ret);
> > +
> > + return ret;
> > +}
> > +
> > +static int compare_pci_id(opae_pci_id *id, opae_pci_id *expected_id)
> > +{
> > + if ((expected_id->class_id != BIT_SET_32) &&
> > + (expected_id->class_id != id->class_id))
> > + return -1;
> > + if ((expected_id->vendor_id != BIT_SET_16) &&
> > + (expected_id->vendor_id != id->vendor_id))
> > + return -1;
> > + if ((expected_id->device_id != BIT_SET_16) &&
> > + (expected_id->device_id != id->device_id))
> > + return -1;
> > + if ((expected_id->subsystem_vendor_id != BIT_SET_16) &&
> > + (expected_id->subsystem_vendor_id != id-
> > >subsystem_vendor_id))
> > + return -1;
> > + if ((expected_id->subsystem_device_id != BIT_SET_16) &&
> > + (expected_id->subsystem_device_id != id-
> > >subsystem_device_id))
> > + return -1;
> > +
> > + return 0;
> > +}
> > +
> > +static int parse_sysfs_value(char *node, uint32_t *val)
> > +{
> > + char buf[16];
> > + char *end = NULL;
> > + int ret = 0;
> > +
> > + ret = read_file(node, buf, sizeof(buf));
> > + if (ret < 0)
> > + return ret;
> > +
> > + *val = (uint32_t)strtoul(buf, &end, 0);
> > + return 0;
> > +}
> > +
> > +static int get_pci_id(const char *dev_path, opae_pci_id *id)
> > +{
> > + char path[PATH_MAX] = {0};
> > + uint32_t tmp;
> > +
> > + if (!dev_path || !id)
> > + return -EINVAL;
> > +
> > + snprintf(path, sizeof(path), "%s/vendor", dev_path);
> > + if (parse_sysfs_value(path, &tmp) < 0)
> > + return -ENODEV;
> > + id->vendor_id = (uint16_t)tmp;
> > +
> > + snprintf(path, sizeof(path), "%s/device", dev_path);
> > + if (parse_sysfs_value(path, &tmp) < 0)
> > + return -ENODEV;
> > + id->device_id = (uint16_t)tmp;
> > +
> > + snprintf(path, sizeof(path), "%s/subsystem_vendor", dev_path);
> > + if (parse_sysfs_value(path, &tmp) < 0)
> > + return -ENODEV;
> > + id->subsystem_vendor_id = (uint16_t)tmp;
> > +
> > + snprintf(path, sizeof(path), "%s/subsystem_device", dev_path);
> > + if (parse_sysfs_value(path, &tmp) < 0)
> > + return -ENODEV;
> > + id->subsystem_device_id = (uint16_t)tmp;
> > +
> > + snprintf(path, sizeof(path), "%s/class", dev_path);
> > + if (parse_sysfs_value(path, &tmp) < 0)
> > + return -ENODEV;
> > + id->class_id = (uint32_t)tmp & RTE_CLASS_ANY_ID;
> > +
> > + return 0;
> > +}
> > +
> > +static int extract_path(char *in, int ridx, char *out, uint32_t size)
> > +{
> > + char src[PATH_MAX] = {0};
> > + char *p = NULL;
> > + int ret = 0;
> > +
> > + if (!in || (strlen(in) > PATH_MAX) || (ridx < 0) || !out)
> > + return -EINVAL;
> > +
> > + strncpy(src, in, sizeof(src));
> > + *out = 0;
> > +
> > + while (1) {
> > + p = strrchr(src, '/');
> > + if (p) {
> > + *p++ = 0;
> > + if (*p) {
> > + if (ridx-- <= 0) {
> > + if (size > strlen(p)) {
> > + strncpy(out, p, size);
> > + ret = strlen(p);
> > + }
> > + break;
> > + }
> > + }
> > + } else {
> > + break;
> > + }
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +int opae_enumerate(opae_pci_id *filter, pcidev_id list, int size)
> > +{
> > + DIR *dir = NULL;
> > + struct dirent *dirent = NULL;
> > + char path[PATH_MAX] = {0};
> > + opae_pci_id id;
> > + int n = 0;
> > +
> > + if (!filter || (size < 0) || (!list && (size > 0))) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + dir = opendir(rte_pci_get_sysfs_path());
> > + if (!dir) {
> > + opae_log_err("Failed to open \'%s\'\n",
> > + rte_pci_get_sysfs_path());
> > + return -EINVAL;
> > + }
> > + while ((dirent = readdir(dir))) {
> > + if (!strcmp(dirent->d_name, "."))
> > + continue;
> > + if (!strcmp(dirent->d_name, ".."))
> > + continue;
> > +
> > + snprintf(path, PATH_MAX, "%s/%s",
> > rte_pci_get_sysfs_path(),
> > + dirent->d_name);
> > + if (get_pci_id(path, &id) < 0)
> > + continue;
> > + if (compare_pci_id(&id, filter) < 0)
> > + continue;
> > +
> > + if (n++ < size) {
> > + snprintf(list->bdf, sizeof(list->bdf), "%s",
> > + dirent->d_name);
> > + list++;
> > + }
> > + }
> > + closedir(dir);
> > +
> > + return n;
> > +}
> > +
> > +static int get_driver(pcidev_id id, char *drv_name, uint32_t size)
> > +{
> > + char path[PATH_MAX] = {0};
> > + char link[PATH_MAX] = {0};
> > + int ret = 0;
> > +
> > + if (!id || !drv_name) {
> > + ret = -EINVAL;
> > + goto end;
> > + }
> > + size--; /* reserve one byte for the end of string */
> > +
> > + snprintf(path, PATH_MAX, "%s/%s/driver",
> > + rte_pci_get_sysfs_path(), id->bdf);
> > + ret = readlink(path, link, PATH_MAX);
> > + if (ret >= PATH_MAX) {
> > + opae_log_err("Link path too long [%d]\n", ret);
> > + ret = -ENAMETOOLONG;
> > + goto end;
> > + }
> > + if (ret > 0) {
> > + ret = extract_path(link, 0, drv_name, size);
> > + } else {
> > + *drv_name = 0;
> > + opae_log_info("No link path for \'%s\'\n", path);
> > + ret = 0;
> > + }
> > +
> > +end:
> > + if (ret < 0)
> > + opae_log_err("Failed to get driver of %s\n", id->bdf);
> > +
> > + return ret;
> > +}
> > +
> > +static int get_pci_addr(const char *bdf, opae_pci_addr *addr)
> > +{
> > + unsigned int domain = 0;
> > + unsigned int bus = 0;
> > + unsigned int devid = 0;
> > + unsigned int function = 0;
> > + int ret = 0;
> > +
> > + if (!bdf || !addr)
> > + return -EINVAL;
> > +
> > + ret = sscanf(bdf, "%04x:%02x:%02x.%d",
> > + &domain, &bus, &devid, &function);
> > + if (ret == 4) {
> > + addr->domain = (uint32_t)domain;
> > + addr->bus = (uint8_t)bus;
> > + addr->devid = (uint8_t)devid;
> > + addr->function = (uint8_t)function;
> > + return 0;
> > + }
> > +
> > + return -EINVAL;
> > +}
> > +
> > +static struct rte_rawdev *get_rte_rawdev(pcidev_id id, int log)
> > +{
> > + opae_pci_addr addr;
> > + struct rte_rawdev *rdev = NULL;
> > + char rdev_name[OPAE_NAME_SIZE] = {0};
> > +
> > + if (!id)
> > + return NULL;
> > +
> > + if (get_pci_addr(id->bdf, &addr) < 0)
> > + return NULL;
> > +
> > + snprintf(rdev_name, OPAE_NAME_SIZE, "IFPGA:%02x:%02x.%x",
> > + addr.bus, addr.devid, addr.function);
> > + rdev = rte_rawdev_pmd_get_named_dev(rdev_name);
> > + if (log && !rdev)
> > + opae_log_warn("%s is not probed\n", id->bdf);
> > +
> > + return rdev;
> > +}
> > +
> > +static struct rte_pci_device *get_rte_pcidev(pcidev_id id, int log)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + struct rte_pci_bus *pci_bus = NULL;
> > + struct rte_pci_device *pci_dev = NULL;
> > +
> > + if (!id)
> > + return NULL;
> > +
> > + pci_bus = ifpga_get_pci_bus();
> > + if (pci_bus) {
> > + TAILQ_FOREACH(pci_dev, &pci_bus->device_list, next) {
> > + if (!strcmp(id->bdf, pci_dev->name))
> > + return pci_dev;
> > + }
> > + } else {
> > + rdev = get_rte_rawdev(id, 0);
> > + if (rdev && rdev->device) {
> > + pci_dev = RTE_DEV_TO_PCI(rdev->device);
> > + return pci_dev;
> > + }
> > + }
> > +
> > + if (log)
> > + opae_log_err("No rte_pci_device for %s\n", id->bdf);
> > +
> > + return NULL;
> > +}
> > +
> > +static int lock(pcidev_id id)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + int ret = 0;
> > +
> > + rdev = get_rte_rawdev(id, 0);
> > + if (rdev)
> > + ret = ifpga_rawdev_lock(rdev);
> > +
> > + return ret;
> > +}
> > +
> > +static int unlock(pcidev_id id)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + int ret = 0;
> > +
> > + rdev = get_rte_rawdev(id, 0);
> > + if (rdev)
> > + ret = ifpga_rawdev_unlock(rdev);
> > +
> > + return ret;
> > +}
> > +
> > +int opae_load_rsu_status(pcidev_id id, uint32_t *status, uint32_t
> *progress)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + uint32_t value = 0;
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (rdev)
> > + value = ifpga_rawdev_get_rsu_stat(rdev);
> > + else
> > + return -ENODEV;
> > +
> > + if (status)
> > + *status = (value >> 16) & 0xffff;
> > + if (progress)
> > + *progress = value & 0xffff;
> > +
> > + return 0;
> > +}
> > +
> > +int opae_store_rsu_status(pcidev_id id, uint32_t status, uint32_t
> progress)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + uint32_t value = 0;
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (rdev) {
> > + value = ((status << 16) & 0xffff0000) | (progress & 0xffff);
> > + ifpga_rawdev_set_rsu_stat(rdev, value);
> > + } else {
> > + return -ENODEV;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static int get_pci_property(pcidev_id id, opae_pci_property *prop)
> > +{
> > + char path[PATH_MAX] = {0};
> > + int ret = 0;
> > +
> > + if (!id || !prop)
> > + return -EINVAL;
> > +
> > + snprintf(path, PATH_MAX, "%s/%s", rte_pci_get_sysfs_path(), id-
> > >bdf);
> > +
> > + ret = get_pci_id(path, &prop->id);
> > + if (ret < 0)
> > + return ret;
> > +
> > + ret = get_pci_addr(id->bdf, &prop->addr);
> > + if (ret < 0)
> > + return ret;
> > +
> > + snprintf(prop->pci_addr, OPAE_NAME_SIZE, "%s", id->bdf);
> > + get_driver(id, prop->drv_name, sizeof(prop->drv_name));
> > +
> > + return 0;
> > +}
> > +
> > +static int get_fme_property(pcidev_id id, opae_fme_property *prop)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + ifpga_fme_property fme_prop;
> > + opae_bitstream_id bbs_id;
> > + int ret = 0;
> > +
> > + if (!prop)
> > + return -EINVAL;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (!rdev)
> > + return -ENODEV;
> > +
> > + ret = ifpga_rawdev_get_fme_property(rdev, &fme_prop);
> > + if (!ret) {
> > + prop->boot_page = fme_prop.boot_page;
> > + prop->num_ports = fme_prop.num_ports;
> > + prop->bitstream_id = fme_prop.bitstream_id;
> > + prop->bitstream_metadata = fme_prop.bitstream_metadata;
> > + memcpy(prop->pr_id.b, fme_prop.pr_id.b,
> > sizeof(opae_uuid));
> > +
> > + bbs_id.id = prop->bitstream_id;
> > + if (bbs_id.major < sizeof(platform_name) / sizeof(char *)) {
> > + snprintf(prop->platform_name,
> > + sizeof(prop->platform_name), "%s",
> > + platform_name[bbs_id.major]);
> > + } else {
> > + snprintf(prop->platform_name,
> > + sizeof(prop->platform_name), "unknown");
> > + }
> > +
> > + snprintf(prop->dcp_version, sizeof(prop->dcp_version),
> > + "DCP 1.%u", bbs_id.minor);
> > +
> > + if (bbs_id.patch < sizeof(release_name)/sizeof(char *)) {
> > + snprintf(prop->release_name, sizeof(prop-
> > >release_name),
> > + "%s", release_name[bbs_id.patch]);
> > + } else {
> > + snprintf(prop->release_name, sizeof(prop-
> > >release_name),
> > + "unknown");
> > + }
> > +
> > + if (bbs_id.major == 0) { /* Vista Creek */
> > + if (bbs_id.interface <
> > + sizeof(interface_type) / sizeof(char *)) {
> > + snprintf(prop->interface_type,
> > + sizeof(prop->interface_type), "%s",
> > + interface_type[bbs_id.interface]);
> > + } else {
> > + snprintf(prop->interface_type,
> > + sizeof(prop->interface_type),
> > "unknown");
> > + }
> > + } else {
> > + snprintf(prop->interface_type,
> > + sizeof(prop->interface_type), "unknown");
> > + }
> > +
> > + snprintf(prop->build_version, sizeof(prop->build_version),
> > + "%u.%u.%u", bbs_id.build_major,
> > bbs_id.build_minor,
> > + bbs_id.build_patch);
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +static int get_port_property(pcidev_id id, uint32_t port,
> > + opae_port_property *prop)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + ifpga_port_property port_prop;
> > + int ret = 0;
> > +
> > + if (!prop || (port >= OPAE_MAX_PORT_NUM))
> > + return -EINVAL;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (!rdev)
> > + return -ENODEV;
> > +
> > + ret = ifpga_rawdev_get_port_property(rdev, port, &port_prop);
> > + if (!ret) {
> > + memcpy(prop->afu_id.b, port_prop.afu_id.b,
> > sizeof(opae_uuid));
> > + prop->type = port_prop.type;
> > + prop->index = port;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static int get_bmc_property(pcidev_id id, opae_bmc_property *prop)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + ifpga_bmc_property bmc_prop;
> > + opae_bmc_version ver;
> > + int ret = 0;
> > +
> > + if (!prop)
> > + return -EINVAL;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (!rdev)
> > + return -ENODEV;
> > +
> > + ret = ifpga_rawdev_get_bmc_property(rdev, &bmc_prop);
> > + if (!ret) {
> > + ver.version = bmc_prop.bmc_version;
> > + snprintf(prop->bmc_version, sizeof(prop->bmc_version),
> > "%C.%u.%u.%u",
> > + ver.board, ver.major, ver.minor, ver.micro);
> > +
> > + ver.version = bmc_prop.fw_version;
> > + snprintf(prop->fw_version, sizeof(prop->fw_version),
> > "%C.%u.%u.%u",
> > + ver.board, ver.major, ver.minor, ver.micro);
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +int opae_get_property(pcidev_id id, opae_fpga_property *prop, int type)
> > +{
> > + uint32_t status = 0;
> > + uint32_t i = 0;
> > + int ret = 0;
> > +
> > + if (!prop) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + if (type == 0)
> > + type = OPAE_PROP_ALL;
> > +
> > + memset(prop, 0, sizeof(opae_fpga_property));
> > +
> > + /* PCI properties */
> > + if (type & OPAE_PROP_PCI) {
> > + ret = get_pci_property(id, &prop->pci);
> > + if (ret < 0) {
> > + opae_log_err("Failed to get PCI property\n");
> > + return ret;
> > + }
> > + }
> > +
> > + if (type == OPAE_PROP_PCI)
> > + return 0;
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + if (!get_rte_rawdev(id, 1))
> > + return -ENODEV;
> > +
> > + lock(id);
> > + opae_load_rsu_status(id, &status, NULL);
> > + if (status == IFPGA_RSU_REBOOT) {
> > + opae_log_warn("Reboot is in progress\n");
> > + ret = -EAGAIN;
> > + goto unlock_dev;
> > + }
> > +
> > + /* FME properties */
> > + if (type & (OPAE_PROP_FME | OPAE_PROP_PORT)) {
> > + ret = get_fme_property(id, &prop->fme);
> > + if (ret) {
> > + opae_log_err("Failed to get FME property\n");
> > + goto unlock_dev;
> > + }
> > + }
> > +
> > + /* PORT properties */
> > + if (type & OPAE_PROP_PORT) {
> > + for (i = 0; i < prop->fme.num_ports; i++) {
> > + ret = get_port_property(id, i, &prop->port[i]);
> > + if (ret) {
> > + opae_log_err("Failed to get port
> > property\n");
> > + goto unlock_dev;
> > + }
> > + }
> > + }
> > +
> > + /* BMC properties */
> > + if (type & OPAE_PROP_BMC) {
> > + ret = get_bmc_property(id, &prop->bmc);
> > + if (ret) {
> > + opae_log_err("Failed to get BMC property\n");
> > + goto unlock_dev;
> > + }
> > + }
> > +
> > +unlock_dev:
> > + unlock(id);
> > + return ret;
> > +}
> > +
> > +int opae_get_phy_info(pcidev_id id, opae_phy_info *info)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + ifpga_phy_info phy_info;
> > + int ret = 0;
> > +
> > + if (!info)
> > + return -EINVAL;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (!rdev)
> > + return -ENODEV;
> > +
> > + ret = ifpga_rawdev_get_phy_info(rdev, &phy_info);
> > + if (!ret) {
> > + info->num_retimers = phy_info.num_retimers;
> > + info->link_speed = phy_info.link_speed;
> > + info->link_status = phy_info.link_status;
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +static int update_driver(pcidev_id id, char *drv_name)
> > +{
> > + struct rte_pci_device *pci_dev = NULL;
> > + char name[OPAE_NAME_SIZE] = {0};
> > + int ret = 0;
> > +
> > + if (!id)
> > + return -EINVAL;
> > +
> > + if (drv_name) {
> > + if (strlen(drv_name) >= OPAE_NAME_SIZE) {
> > + opae_log_err("Driver name \'%s\' too long\n",
> > + drv_name);
> > + return -EINVAL;
> > + }
> > + strncpy(name, drv_name, sizeof(name));
> > + } else {
> > + ret = get_driver(id, name, sizeof(name));
> > + if (ret < 0)
> > + return ret;
> > + }
> > +
> > + pci_dev = get_rte_pcidev(id, 0);
> > + if (pci_dev) {
> > + if (strlen(name) == 0) {
> > + pci_dev->kdrv = RTE_PCI_KDRV_NONE;
> > + } else {
> > + if (!strcmp(name, OPAE_KDRV_VFIO_PCI))
> > + pci_dev->kdrv = RTE_PCI_KDRV_VFIO;
> > + else if (!strcmp(name, OPAE_KDRV_IGB_UIO))
> > + pci_dev->kdrv = RTE_PCI_KDRV_IGB_UIO;
> > + else if (!strcmp(name, OPAE_KDRV_UIO_PCI))
> > + pci_dev->kdrv =
> > RTE_PCI_KDRV_UIO_GENERIC;
> > + else
> > + pci_dev->kdrv = RTE_PCI_KDRV_UNKNOWN;
> > + }
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +int opae_unbind_driver(pcidev_id id)
> > +{
> > + char path[PATH_MAX] = {0};
> > + char drv_name[OPAE_NAME_SIZE] = {0};
> > + char null[] = {0};
> > + int ret = 0;
> > +
> > + if (get_rte_rawdev(id, 0)) {
> > + opae_log_err("%s is probed, remove it first\n", id->bdf);
> > + return -EBUSY;
> > + }
> > +
> > + ret = get_driver(id, drv_name, sizeof(drv_name));
> > + if (ret < 0)
> > + return ret;
> > +
> > + if (strlen(drv_name) > 0) {
> > + snprintf(path, PATH_MAX, "/sys/bus/pci/drivers/%s/unbind",
> > + drv_name);
> > + ret = write_file(path, id->bdf, strlen(id->bdf) + 1);
> > + if (ret == 0)
> > + ret = update_driver(id, null);
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +static int check_driver(const char *drv_name)
> > +{
> > + char path[PATH_MAX] = {0};
> > + struct stat buf;
> > +
> > + if (!drv_name)
> > + return -EINVAL;
> > +
> > + if (strlen(drv_name) > 0) {
> > + snprintf(path, PATH_MAX, "/sys/bus/pci/drivers/%s",
> > drv_name);
> > + if ((stat(path, &buf) < 0) || ((buf.st_mode & S_IFDIR) == 0)) {
> > + opae_log_warn("Driver %s is not installed\n",
> > + drv_name);
> > + return -EINVAL;
> > + }
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +int opae_bind_driver(pcidev_id id, char *drv_name)
> > +{
> > + char path[PATH_MAX] = {0};
> > + char name[OPAE_NAME_SIZE] = {0};
> > + char null[] = {0};
> > + int ret = 0;
> > +
> > + ret = check_driver(drv_name);
> > + if (ret < 0)
> > + return ret;
> > +
> > + ret = get_driver(id, name, sizeof(name));
> > + if (ret < 0)
> > + return ret;
> > +
> > + if (!strcmp(drv_name, name)) /* driver not change */
> > + return 0;
> > +
> > + ret = opae_unbind_driver(id);
> > + if (ret < 0)
> > + return ret;
> > +
> > + if (strlen(drv_name) > 0) {
> > + /* bind driver */
> > + snprintf(path, PATH_MAX, "%s/%s/driver_override",
> > + rte_pci_get_sysfs_path(), id->bdf);
> > + ret = write_file(path, drv_name, strlen(drv_name) + 1);
> > + if (ret < 0)
> > + goto update_drv;
> > +
> > + snprintf(path, PATH_MAX, "/sys/bus/pci/drivers/%s/bind",
> > + drv_name);
> > + ret = write_file(path, id->bdf, strlen(id->bdf) + 1);
> > + if (ret < 0)
> > + goto update_drv;
> > +
> > + snprintf(path, PATH_MAX, "%s/%s/driver_override",
> > + rte_pci_get_sysfs_path(), id->bdf);
> > + ret = write_file(path, null, 1);
> > + if (ret < 0)
> > + goto update_drv;
> > + }
> > +
> > +update_drv:
> > + ret = update_driver(id, NULL);
> > + if (ret < 0)
> > + opae_log_err("Failed to update driver information of %s\n",
> > + id->bdf);
> > +
> > + return 0;
> > +}
> > +
> > +int opae_probe_device(pcidev_id id)
> > +{
> > + struct rte_pci_bus *pci_bus = NULL;
> > + struct rte_pci_device *pci_dev = NULL;
> > +
> > + if (!id) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + /* make sure device is added in rte_pci_bus devices list */
> > + pci_bus = ifpga_get_pci_bus();
> > + if (pci_bus && pci_bus->bus.scan)
> > + pci_bus->bus.scan();
> > +
> > + pci_dev = get_rte_pcidev(id, 1);
> > + if (!pci_dev)
> > + return -ENODEV;
> > +
> > + if (pci_dev->kdrv != RTE_PCI_KDRV_VFIO) {
> > + opae_log_err("vfio-pci driver is not bound to %s\n", id->bdf);
> > + return -EINVAL;
> > + }
> > +
> > + if (!pci_bus || !pci_bus->bus.plug)
> > + return -ENODEV;
> > +
> > + return pci_bus->bus.plug(&pci_dev->device);
> > +}
> > +
> > +int opae_remove_device(pcidev_id id)
> > +{
> > + struct rte_pci_device *pci_dev = NULL;
> > + struct rte_pci_driver *pci_drv = NULL;
> > + int ret = 0;
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + pci_dev = get_rte_pcidev(id, 0);
> > + if (pci_dev && pci_dev->driver) {
> > + pci_drv = pci_dev->driver;
> > + ret = pci_drv->remove(pci_dev);
> > + if (ret < 0) {
> > + opae_log_err("Failed to remove %s [e:%d]\n",
> > + id->bdf, ret);
> > + return ret;
> > + }
> > + pci_dev->driver = NULL;
> > + pci_dev->device.driver = NULL;
> > + if (pci_drv->drv_flags & RTE_PCI_DRV_NEED_MAPPING)
> > + rte_pci_unmap_device(pci_dev);
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +static int is_pac(pcidev_id id)
> > +{
> > + char path[PATH_MAX] = {0};
> > + opae_pci_id pci_id;
> > +
> > + if (!id)
> > + return 0;
> > +
> > + snprintf(path, PATH_MAX, "%s/%s", rte_pci_get_sysfs_path(), id-
> > >bdf);
> > + if (get_pci_id(path, &pci_id) < 0)
> > + return 0;
> > +
> > + if ((pci_id.vendor_id == 0x8086) && (pci_id.device_id == 0x0b30))
> > + return 1;
> > +
> > + return 0;
> > +}
> > +
> > +int opae_get_parent(pcidev_id id, pcidev_id parent)
> > +{
> > + char path[PATH_MAX] = {0};
> > + char link[PATH_MAX] = {0};
> > + int ret = 0;
> > +
> > + if (!id || !parent) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + ret = -EINVAL;
> > + goto end;
> > + }
> > +
> > + snprintf(path, PATH_MAX, "%s/%s", rte_pci_get_sysfs_path(), id-
> > >bdf);
> > + ret = readlink(path, link, PATH_MAX);
> > + if (ret >= PATH_MAX) {
> > + opae_log_err("Length of link path exceeds %u\n",
> > PATH_MAX);
> > + ret = -ENAMETOOLONG;
> > + goto end;
> > + }
> > +
> > + if (ret > 0) {
> > + ret = extract_path(link, 1, parent->bdf, sizeof(parent->bdf));
> > + if (!strncmp(parent->bdf, "pci", 3)) {
> > + parent->bdf[0] = 0;
> > + ret = -ENODEV;
> > + }
> > + } else {
> > + parent->bdf[0] = 0;
> > + if (ret == 0)
> > + opae_log_err("Length of link path is 0\n");
> > + else
> > + opae_log_err("No link path for \'%s\'\n", path);
> > + }
> > +end:
> > + if (ret <= 0)
> > + opae_log_err("%s has no parent\n", id->bdf);
> > +
> > + return ret;
> > +}
> > +
> > +int opae_get_child(pcidev_id id, pcidev_id child, int size)
> > +{
> > + glob_t pglob = {.gl_pathc = 0, .gl_pathv = NULL};
> > + char path[PATH_MAX] = {0};
> > + int i, count = 0;
> > + int len = 0;
> > + int ret = 0;
> > +
> > + if (!id || (size < 0)) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + snprintf(path, PATH_MAX, "%s/%s/*:*:*.?",
> > rte_pci_get_sysfs_path(),
> > + id->bdf);
> > + ret = glob(path, 0, NULL, &pglob);
> > + if (ret == 0) {
> > + if (child && (size > 0)) {
> > + for (i = 0; i < (int)pglob.gl_pathc; i++) {
> > + len = extract_path(pglob.gl_pathv[i], 0,
> > + child->bdf, sizeof(child->bdf));
> > + if (len <= 0) {
> > + child->bdf[0] = 0;
> > + continue;
> > + }
> > + if (++count >= size)
> > + break;
> > + child++;
> > + }
> > + } else {
> > + count = (int)pglob.gl_pathc;
> > + }
> > + globfree(&pglob);
> > + } else {
> > + if (pglob.gl_pathv)
> > + globfree(&pglob);
> > + }
> > +
> > + return count;
> > +}
> > +
> > +int opae_get_pf1(pcidev_id id, pcidev_id peer, int size)
> > +{
> > + opae_pci_device parent;
> > + opae_pci_device child[4];
> > + int n = 0;
> > + int ret = 0;
> > +
> > + if (!id) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + if (!is_pac(id)) {
> > + opae_log_info("%s has no peer function\n", id->bdf);
> > + return -EINVAL;
> > + }
> > +
> > + ret = opae_get_parent(id, &parent);
> > + if (ret < 0)
> > + return -ENODEV;
> > + ret = opae_get_parent(&parent, &parent);
> > + if (ret < 0)
> > + return -ENODEV;
> > +
> > + n = opae_get_child(&parent, child,
> > + sizeof(child) / sizeof(opae_pci_device));
> > + /* there should have four downstream ports of PCI switch on board
> > */
> > + if (n == 4) {
> > + n = opae_get_child(&child[3], peer, size);
> > + } else {
> > + peer->bdf[0] = 0;
> > + opae_log_dbg("%s has %d child(s)\n", parent.bdf, n);
> > + n = 0;
> > + }
> > +
> > + return n;
> > +}
> > +
> > +void opae_check_pcidev_list(void)
> > +{
> > + int i = 0;
> > + unsigned int k = 0;
> > + struct rte_pci_bus *pci_bus = NULL;
> > + struct rte_pci_device *pci_dev = NULL;
> > +
> > + if (!check_eal(1))
> > + return;
> > +
> > + pci_bus = ifpga_get_pci_bus();
> > + if (!pci_bus)
> > + return;
> > +
> > + printf(" ID NAME SEG BUS DEV FUNC VID DID KDRV\n");
> > + TAILQ_FOREACH(pci_dev, &pci_bus->device_list, next) {
> > + k = pci_dev->kdrv;
> > + printf("%3d %s %04x %02x %02x %2d %04x %04x %s\n",
> > + i, pci_dev->name, pci_dev->addr.domain,
> > + pci_dev->addr.bus, pci_dev->addr.devid,
> > + pci_dev->addr.function, pci_dev->id.vendor_id,
> > + pci_dev->id.device_id,
> > + k > RTE_PCI_KDRV_UIO_GENERIC ? "" : kdrv[k]);
> > + i++;
> > + }
> > +}
> > +
> > +int opae_update_flash(pcidev_id id, const char *image, uint64_t *status)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + opae_img_info info;
> > + int ret = 0;
> > +
> > + ret = opae_get_image_info(image, &info);
> > + if (ret < 0) {
> > + opae_log_err("Failed to get image information [e:%d]\n",
> > ret);
> > + return -EINVAL;
> > + }
> > +
> > + if ((info.type != OPAE_IMG_TYPE_BBS) &&
> > + (info.type != OPAE_IMG_TYPE_BMC)) {
> > + opae_log_err("Image is not supported [t:%u]\n", info.type);
> > + return -EOPNOTSUPP;
> > + }
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (!rdev)
> > + return -ENODEV;
> > +
> > + return ifpga_rawdev_update_flash(rdev, image, status);
> > +}
> > +
> > +int opae_cancel_flash_update(pcidev_id id, int force)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (!rdev)
> > + return -ENODEV;
> > +
> > + return ifpga_rawdev_stop_flash_update(rdev, force);
> > +}
> > +
> > +#define PCI_EXT_CAP_ID_ERR 0x01 /* Advanced Error
> > Reporting */
> > +#define PCI_CFG_SPACE_SIZE 256
> > +#define PCI_CFG_SPACE_EXP_SIZE 4096
> > +#define PCI_EXT_CAP_ID(hdr) ((int)((hdr) & 0x0000ffff))
> > +#define PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xffc)
> > +
> > +static int find_pci_ecap(int fd, int cap)
> > +{
> > + uint32_t header = 0;
> > + int ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
> > + int pos = PCI_CFG_SPACE_SIZE; /* start of extension capability area
> > */
> > + int ret = 0;
> > +
> > + ret = pread(fd, &header, sizeof(header), pos);
> > + if (ret < 0) {
> > + opae_log_err("Failed to read from PCI configuration space
> > [e:%s]\n",
> > + strerror(errno));
> > + return ret;
> > + }
> > + opae_log_dbg("Read 0x%08x from PCI configuration space 0x%x\n",
> > + header, pos);
> > +
> > + if (header == 0) {
> > + opae_log_err("Capability is empty\n");
> > + return 0;
> > + }
> > +
> > + while (ttl-- > 0) {
> > + if ((PCI_EXT_CAP_ID(header) == cap) && (pos != 0))
> > + return pos;
> > +
> > + pos = PCI_EXT_CAP_NEXT(header);
> > + if (pos < PCI_CFG_SPACE_SIZE) {
> > + opae_log_err("Position of capability is invalid"
> > + "[e:%d]\n", pos);
> > + break;
> > + }
> > + ret = pread(fd, &header, sizeof(header), pos);
> > + if (ret < 0) {
> > + opae_log_err("Failed to read from PCI config space
> > [e:%s]\n",
> > + strerror(errno));
> > + return ret;
> > + }
> > + opae_log_dbg("Read 0x%08x from PCI configuration space
> > 0x%x\n",
> > + header, pos);
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static int set_aer(pcidev_id id, uint32_t v1, uint32_t v2, int record)
> > +{
> > + char path[PATH_MAX] = {0};
> > + uint32_t val = 0;
> > + int fd = -1;
> > + int pos = 0;
> > + int ret = 0;
> > +
> > + if (!id)
> > + return -EINVAL;
> > +
> > + snprintf(path, PATH_MAX, "%s/%s/config",
> > + rte_pci_get_sysfs_path(), id->bdf);
> > + fd = open(path, O_RDWR);
> > + if (fd < 0) {
> > + opae_log_err("Failed to open \'%s\' for RDWR [e:%s]\n",
> > + path, strerror(errno));
> > + return -EIO;
> > + }
> > +
> > + pos = find_pci_ecap(fd, PCI_EXT_CAP_ID_ERR);
> > + if (pos <= 0) {
> > + opae_log_warn("AER capability is not present\n");
> > + ret = -ENXIO;
> > + goto close_fd;
> > + }
> > +
> > + if (record) {
> > + ret = pread(fd, &val, sizeof(val), pos + 0x08);
> > + if (ret < 0) {
> > + opae_log_err("Failed to read from PCI config space
> > [e:%s]\n",
> > + strerror(errno));
> > + goto close_fd;
> > + }
> > + opae_log_dbg("Read 0x%08x from PCI configuration space
> > 0x%x\n",
> > + val, pos + 0x08);
> > + dev_aer[0] = val;
> > +
> > + ret = pread(fd, &val, sizeof(val), pos + 0x14);
> > + if (ret < 0) {
> > + opae_log_err("Failed to read from PCI config space
> > [e:%s]\n",
> > + strerror(errno));
> > + goto close_fd;
> > + }
> > + opae_log_dbg("Read 0x%08x from PCI configuration space
> > 0x%x\n",
> > + val, pos + 0x14);
> > + dev_aer[1] = val;
> > + }
> > +
> > + opae_log_dbg("Write 0x%08x to PCI configuration space 0x%x\n",
> > + v1, pos + 0x08);
> > + ret = pwrite(fd, &v1, sizeof(v1), pos + 0x08);
> > + if (ret < 0) {
> > + opae_log_err("Failed to write to PCI config space 0x%x
> > [e:%s]\n",
> > + pos + 0x08, strerror(errno));
> > + goto close_fd;
> > + }
> > +
> > + opae_log_dbg("Write 0x%08x to PCI configuration space 0x%x\n",
> > + v2, pos + 0x14);
> > + ret = pwrite(fd, &v2, sizeof(v2), pos + 0x14);
> > + if (ret < 0) {
> > + opae_log_err("Failed to write to PCI config space 0x%x
> > [e:%s]\n",
> > + pos + 0x14, strerror(errno));
> > + }
> > +
> > +close_fd:
> > + close(fd);
> > + return ret < 0 ? ret : 0;
> > +}
> > +
> > +static int enable_aer(pcidev_id id)
> > +{
> > + if (!id) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + opae_log_info("Enable AER of %s\n", id->bdf);
> > +
> > + return set_aer(id, dev_aer[0], dev_aer[1], 0);
> > +}
> > +
> > +static int disable_aer(pcidev_id id)
> > +{
> > + if (!id) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + opae_log_info("Disable AER of %s\n", id->bdf);
> > +
> > + return set_aer(id, 0xffffffff, 0xffffffff, 1);
> > +}
> > +
> > +static int reload(pcidev_id id, int type, int page)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > + int ret = 0;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (rdev)
> > + ret = ifpga_rawdev_reload(rdev, type, page);
> > + else
> > + ret = -ENODEV;
> > +
> > + return ret;
> > +}
> > +
> > +static int remove_tree(pcidev_id id)
> > +{
> > + int i, n = 0;
> > + pcidev_id child;
> > + int ret = 0;
> > +
> > + if (!id) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + n = opae_get_child(id, NULL, 0);
> > + if (n > 0) {
> > + child = (pcidev_id)rte_zmalloc(NULL,
> > + sizeof(opae_pci_device) * n, 0);
> > + if (!child) {
> > + opae_log_err("Failed to malloc for children of %s\n",
> > + id->bdf);
> > + ret = -ENOMEM;
> > + goto end;
> > + }
> > +
> > + opae_get_child(id, child, n);
> > + for (i = 0; i < n; i++)
> > + remove_tree(&child[i]);
> > + opae_free(child);
> > + }
> > +
> > +end:
> > + opae_remove_device(id);
> > + return ret;
> > +}
> > +
> > +static int remove_device(pcidev_id id)
> > +{
> > + char path[PATH_MAX] = {0};
> > + char one[] = {'1', 0};
> > + int ret = 0;
> > +
> > + if (!id) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > + opae_log_info("Remove %s from system\n", id->bdf);
> > +
> > + snprintf(path, PATH_MAX, "%s/%s/remove",
> > + rte_pci_get_sysfs_path(), id->bdf);
> > + ret = write_file(path, one, strlen(one));
> > + if (ret < 0) {
> > + opae_log_err("Failed to remove %s from system\n", id->bdf);
> > + return ret;
> > + }
> > +
> > + remove_tree(id);
> > +
> > + return 0;
> > +}
> > +
> > +static int scan_device(pcidev_id parent, pcidev_id id)
> > +{
> > + char path[PATH_MAX] = {0};
> > + char bus[8] = {0};
> > + char one[] = {'1', 0};
> > + char pwr[16] = {0};
> > + char pwr_on[] = {'o', 'n', 0};
> > + int pwr_on_failed = 0;
> > + int ret = 0;
> > +
> > + if (!parent) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > + opae_log_info("Rescan devices under %s\n", parent->bdf);
> > +
> > + if (id) { /* scan specified bus under parent device */
> > + snprintf(path, PATH_MAX, "%s/%s/power/control",
> > + rte_pci_get_sysfs_path(), parent->bdf);
> > + ret = read_file(path, pwr, sizeof(pwr));
> > + if (ret < 0)
> > + return ret;
> > +
> > + if (strcmp(pwr, "on")) {
> > + ret = write_file(path, pwr_on, strlen(pwr_on));
> > + if (ret < 0)
> > + pwr_on_failed = 1;
> > + else
> > + sleep(1);
> > + }
> > +
> > + snprintf(bus, sizeof(bus), "%s", id->bdf);
> > + snprintf(path, PATH_MAX, "%s/%s/pci_bus/%s/rescan",
> > + rte_pci_get_sysfs_path(), parent->bdf, bus);
> > + ret = write_file(path, one, strlen(one));
> > + if (ret < 0)
> > + return ret;
> > +
> > + if (pwr_on_failed) { /* workaround for power on failed */
> > + ret = write_file(path, one, strlen(one));
> > + if (ret < 0)
> > + return ret;
> > + }
> > +
> > + if (strcmp(pwr, "on")) {
> > + snprintf(path, PATH_MAX, "%s/%s/power/control",
> > + rte_pci_get_sysfs_path(), parent->bdf);
> > + ret = write_file(path, pwr, strlen(pwr));
> > + }
> > + } else { /* scan all buses under parent device */
> > + snprintf(path, PATH_MAX, "%s/%s/rescan",
> > + rte_pci_get_sysfs_path(), parent->bdf);
> > + ret = write_file(path, one, strlen(one));
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +int opae_reboot_device(pcidev_id id, int type, int page)
> > +{
> > + opae_pci_device fpga; /* FPGA after reboot */
> > + opae_pci_device parent;
> > + opae_pci_device peer[2]; /* physical function 1 of FPGA */
> > + opae_pci_device peer_parent;
> > + opae_pci_device ups; /* upstream port device */
> > + opae_pci_device root; /* port connected to PAC */
> > + pcidev_id peer_master = NULL;
> > + uint32_t rsu_stat = 0;
> > + char drv_name[OPAE_NAME_SIZE] = {0};
> > + int n = 0;
> > + int i = 0;
> > + int ret = 0;
> > +
> > + if (!id) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + if (!is_pac(id)) {
> > + opae_log_err("%s can not be rebooted\n", id->bdf);
> > + return -EINVAL;
> > + }
> > +
> > + ret = opae_get_parent(id, &parent);
> > + if (ret < 0)
> > + return -ENODEV;
> > + ret = opae_get_parent(&parent, &ups);
> > + if (ret < 0)
> > + return -ENODEV;
> > + ret = opae_get_parent(&ups, &root);
> > + if (ret < 0)
> > + return -ENODEV;
> > +
> > + n = opae_get_pf1(id, peer, sizeof(peer) / sizeof(opae_pci_device));
> > + if (n <= 0) {
> > + opae_log_err("PF1 of %s is not found\n", id->bdf);
> > + } else {
> > + peer_master = &peer[0];
> > + ret = opae_get_parent(peer_master, &peer_parent);
> > + if (ret < 0)
> > + return -ENODEV;
> > + }
> > +
> > + get_driver(id, drv_name, sizeof(drv_name)); /* save original driver
> > */
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + if (!get_rte_rawdev(id, 1))
> > + return -ENODEV;
> > +
> > + lock(id);
> > + opae_load_rsu_status(id, &rsu_stat, NULL);
> > + if (rsu_stat != IFPGA_RSU_IDLE) {
> > + unlock(id);
> > + if (rsu_stat == IFPGA_RSU_REBOOT)
> > + opae_log_warn("Reboot is in progress\n");
> > + else
> > + opae_log_warn("Flash is in progress\n");
> > + return -EAGAIN;
> > + }
> > + opae_store_rsu_status(id, IFPGA_RSU_REBOOT, 0);
> > + unlock(id);
> > +
> > + if (type == IFPGA_BOOT_TYPE_FPGA) {
> > + /* disable AER */
> > + ret = disable_aer(&parent);
> > + if (ret < 0) {
> > + opae_log_err("Failed to disable AER of %s\n",
> > + parent.bdf);
> > + goto reboot_end;
> > + }
> > + ret = disable_aer(&peer_parent);
> > + if (ret < 0) {
> > + opae_log_err("Failed to disable AER of %s\n",
> > + peer_parent.bdf);
> > + goto reboot_end;
> > + }
> > + opae_store_rsu_status(id, IFPGA_RSU_REBOOT, 1);
> > +
> > + /* trigger reconfiguration */
> > + ret = reload(id, type, page);
> > + opae_store_rsu_status(id, IFPGA_RSU_REBOOT, 2);
> > + if (ret == 0) {
> > + ret = remove_device(id);
> > + for (i = 0; i < n; i++)
> > + ret += remove_device(&peer[i]);
> > + if (ret == 0) {
> > + opae_log_info("Wait 10 seconds for FPGA
> > reloading\n");
> > + sleep(10);
> > + ret = scan_device(&parent, id);
> > + if (ret < 0)
> > + opae_log_err("Failed to rescan %s\n",
> > + id->bdf);
> > + if (peer_master) {
> > + ret = scan_device(&peer_parent,
> > + peer_master);
> > + if (ret < 0) {
> > + opae_log_err("Failed to
> > rescan %s\n",
> > + peer_master->bdf);
> > + }
> > + }
> > + }
> > + }
> > +
> > + /* restore AER */
> > + if (enable_aer(&parent) < 0) {
> > + opae_log_err("Failed to enable AER of %s\n",
> > + parent.bdf);
> > + }
> > + if (enable_aer(&peer_parent) < 0) {
> > + opae_log_err("Failed to enable AER of %s\n",
> > + peer_parent.bdf);
> > + }
> > + } else if (type == IFPGA_BOOT_TYPE_BMC) {
> > + /* disable AER */
> > + ret = disable_aer(&root);
> > + if (ret < 0) {
> > + opae_log_err("Failed to disable AER of %s\n",
> > root.bdf);
> > + goto reboot_end;
> > + }
> > + opae_store_rsu_status(id, IFPGA_RSU_REBOOT, 1);
> > +
> > + /* trigger reconfiguration */
> > + ret = reload(id, type, page);
> > + opae_store_rsu_status(id, IFPGA_RSU_REBOOT, 2);
> > + if (ret == 0) {
> > + ret += remove_device(&ups);
> > + if (ret == 0) {
> > + opae_log_info("Wait 10 seconds for BMC
> > reloading\n");
> > + sleep(10);
> > + ret = scan_device(&root, &ups);
> > + if (ret < 0)
> > + opae_log_err("Failed to rescan %s\n",
> > + ups.bdf);
> > + }
> > + }
> > +
> > + /* restore AER */
> > + if (enable_aer(&root) < 0)
> > + opae_log_err("Failed to enable AER of %s\n",
> > root.bdf);
> > + } else {
> > + opae_log_err("Type of reboot is not supported [t:%d]\n",
> > type);
> > + ret = -EINVAL;
> > + goto reboot_end;
> > + }
> > +
> > + /* update id if bdf changed after reboot */
> > + if (opae_get_child(&parent, &fpga, 1) == 1) {
> > + if (strcmp(id->bdf, fpga.bdf))
> > + id = &fpga;
> > + }
> > +
> > + ret = opae_bind_driver(id, drv_name);
> > + if (ret < 0)
> > + opae_log_err("Failed to bind original driver of %s\n", id-
> > >bdf);
> > +
> > + ret = opae_probe_device(id);
> > + if (ret < 0)
> > + opae_log_err("Failed to probe %s [e:%d]\n", id->bdf, ret);
> > +
> > +reboot_end:
> > + opae_store_rsu_status(id, IFPGA_RSU_IDLE, 0);
> > + return ret;
> > +}
> > +
> > +int opae_partial_reconfigure(pcidev_id id, int port, const char *gbs)
> > +{
> > + struct rte_rawdev *rdev = NULL;
> > +
> > + if (!id || !gbs) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + if (!check_eal(1))
> > + return -EPERM;
> > +
> > + rdev = get_rte_rawdev(id, 1);
> > + if (!rdev)
> > + return -ENODEV;
> > +
> > + return ifpga_rawdev_partial_reconfigure(rdev, port, gbs);
> > +}
> > +
> > +int opae_read_pci_cfg(pcidev_id id, uint32_t address, uint32_t *value)
> > +{
> > + char path[PATH_MAX] = {0};
> > + int fd = -1;
> > + int ret = 0;
> > +
> > + if (!id || !value) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + snprintf(path, PATH_MAX, "%s/%s/config",
> > rte_pci_get_sysfs_path(),
> > + id->bdf);
> > + fd = open(path, O_RDONLY);
> > + if (fd < 0) {
> > + opae_log_dbg("Failed to open \'%s\' for RDONLY [e:%s]\n",
> > + path, strerror(errno));
> > + return -EIO;
> > + }
> > +
> > + ret = pread(fd, value, 4, address);
> > + if (ret < 0) {
> > + opae_log_err("Failed to read from PCI device %s [e:%s]\n",
> > + id->bdf, strerror(errno));
> > + close(fd);
> > + return ret;
> > + }
> > +
> > + opae_log_dbg("CONFIG+0x%08x -> 0x%08x\n", address, *value);
> > + close(fd);
> > + return 0;
> > +}
> > +
> > +int opae_write_pci_cfg(pcidev_id id, uint32_t address, uint32_t value)
> > +{
> > + char path[PATH_MAX] = {0};
> > + int fd = -1;
> > + int ret = 0;
> > +
> > + if (!id) {
> > + opae_log_err("Input parameter of %s is invalid\n",
> > __func__);
> > + return -EINVAL;
> > + }
> > +
> > + snprintf(path, PATH_MAX, "%s/%s/config",
> > rte_pci_get_sysfs_path(),
> > + id->bdf);
> > + fd = open(path, O_WRONLY);
> > + if (fd < 0) {
> > + opae_log_dbg("Failed to open \'%s\' for WRONLY [e:%s]\n",
> > + path, strerror(errno));
> > + return -EIO;
> > + }
> > +
> > + ret = pwrite(fd, &value, 4, address);
> > + if (ret < 0) {
> > + opae_log_err("Failed to write to PCI device %s [e:%s]\n",
> > + id->bdf, strerror(errno));
> > + close(fd);
> > + return ret;
> > + }
> > +
> > + opae_log_dbg("CONFIG+0x%08x <- 0x%08x\n", address, value);
> > + close(fd);
> > + return 0;
> > +}
> > diff --git a/drivers/raw/ifpga/ifpga_opae_api.h
> > b/drivers/raw/ifpga/ifpga_opae_api.h
> > new file mode 100644
> > index 000000000..d4ce64280
> > --- /dev/null
> > +++ b/drivers/raw/ifpga/ifpga_opae_api.h
> > @@ -0,0 +1,245 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2020 Intel Corporation
> > + */
> > +
> > +#ifndef _OPAE_API_H
> > +#define _OPAE_API_H
> > +
> > +
> > +#ifdef __cplusplus
> > +extern "C" {
> > +#endif
> > +
> > +#include <stdint.h>
> > +
> > +extern int opae_log_level;
> > +extern FILE *opae_log_file;
> > +
> > +#define OPAE_LOG_API 0 /**< Critical conditions. */
> > +#define OPAE_LOG_ERR 1 /**< Error conditions. */
> > +#define OPAE_LOG_WARN 2 /**< Warning conditions. */
> > +#define OPAE_LOG_INFO 3 /**< Informational. */
> > +#define OPAE_LOG_DEBUG 4 /**< Debug-level messages. */
> > +
> > +#define opae_log(type, fmt, args...) \
> > +do { \
> > + if (opae_log_level >= OPAE_LOG_##type) { \
> > + printf(fmt, ##args); \
> > + if (opae_log_file) { \
> > + fprintf(opae_log_file, fmt, ##args); \
> > + fflush(opae_log_file); \
> > + } \
> > + } \
> > +} while (0)
> > +
> > +#define opae_log_api(fmt, args...) opae_log(API, "OPAE-API: "fmt,
> ##args)
> > +#define opae_log_err(fmt, args...) opae_log(ERR, "OPAE-ERR: "fmt,
> ##args)
> > +#define opae_log_dbg(fmt, args...) opae_log(DEBUG, "OPAE-DBG: "fmt,
> > ##args)
> > +#define opae_log_warn(fmt, args...) opae_log(WARN, "OPAE-WARN:
> "fmt,
> > ##args)
> > +#define opae_log_info(fmt, args...) opae_log(INFO, "OPAE-INFO: "fmt,
> > ##args)
> > +
> > +#define EAL_INIT_FUNCTION "init"
> > +#define EAL_DEFAULT_OPTIONS "--proc-type auto"
> > +
> > +#define OPAE_KDRV_UNKNOWN "unknown"
> > +#define OPAE_KDRV_VFIO_PCI "vfio-pci"
> > +#define OPAE_KDRV_IGB_UIO "igb_uio"
> > +#define OPAE_KDRV_UIO_PCI "uio_pci_generic"
> > +#define OPAE_KDRV_INTEL_FPGA_PCI "intel-fpga-pci"
> > +
> > +typedef struct {
> > + uint32_t major;
> > + uint32_t minor;
> > + uint32_t micro;
> > +} opae_api_version;
> > +
> > +#define OPAE_NAME_SIZE 32
> > +
> > +typedef struct {
> > + char bdf[OPAE_NAME_SIZE]; /* segment:bus:device.function */
> > +} opae_pci_device;
> > +
> > +typedef opae_pci_device *pcidev_id;
> > +
> > +typedef struct {
> > + uint32_t class_id; /**< Class ID or RTE_CLASS_ANY_ID. */
> > + uint16_t vendor_id; /**< Vendor ID or PCI_ANY_ID. */
> > + uint16_t device_id; /**< Device ID or PCI_ANY_ID. */
> > + uint16_t subsystem_vendor_id; /**< Subsystem vendor ID or
> > PCI_ANY_ID. */
> > + uint16_t subsystem_device_id; /**< Subsystem device ID or
> > PCI_ANY_ID. */
> > +} opae_pci_id;
> > +
> > +typedef struct {
> > + uint32_t domain; /**< Device domain */
> > + uint8_t bus; /**< Device bus */
> > + uint8_t devid; /**< Device ID */
> > + uint8_t function; /**< Device function. */
> > +} opae_pci_addr;
> > +
> > +typedef struct {
> > + char pci_addr[OPAE_NAME_SIZE]; /* segment:bus:device.function
> > */
> > + char drv_name[OPAE_NAME_SIZE]; /* vfio-pci, intel-fpga-pci, etc.
> > */
> > + opae_pci_id id;
> > + opae_pci_addr addr;
> > +} opae_pci_property;
> > +
> > +#define BIT_SET_8 0xFF
> > +#define BIT_SET_16 0xFFFF
> > +#define BIT_SET_32 0xFFFFFFFF
> > +
> > +typedef struct {
> > + uint8_t b[16];
> > +} opae_uuid;
> > +
> > +typedef struct {
> > + uint32_t boot_page;
> > + uint32_t num_ports;
> > + uint64_t bitstream_id;
> > + uint64_t bitstream_metadata;
> > + opae_uuid pr_id;
> > + char platform_name[OPAE_NAME_SIZE];
> > + char dcp_version[OPAE_NAME_SIZE];
> > + char release_name[OPAE_NAME_SIZE];
> > + char interface_type[OPAE_NAME_SIZE];
> > + char build_version[OPAE_NAME_SIZE];
> > +} opae_fme_property;
> > +
> > +typedef struct {
> > + opae_uuid afu_id;
> > + uint32_t type; /* AFU memory access control type */
> > + uint32_t index; /* PORT index */
> > +} opae_port_property;
> > +
> > +typedef struct {
> > + char bmc_version[OPAE_NAME_SIZE];
> > + char fw_version[OPAE_NAME_SIZE];
> > +} opae_bmc_property;
> > +
> > +typedef struct {
> > + uint32_t num_retimers;
> > + uint32_t link_speed;
> > + uint32_t link_status; /* each bit corresponding to one link status */
> > +} opae_phy_info;
> > +
> > +typedef struct {
> > + union {
> > + uint64_t id;
> > + struct {
> > + uint8_t build_patch;
> > + uint8_t build_minor;
> > + uint8_t build_major;
> > + uint8_t fvl_bypass:1;
> > + uint8_t mac_lightweight:1;
> > + uint8_t disagregate:1;
> > + uint8_t lightweiht:1;
> > + uint8_t seu:1;
> > + uint8_t ptp:1;
> > + uint8_t reserve:2;
> > + uint16_t interface:4;
> > + uint16_t afu_revision:12;
> > + uint16_t patch:4;
> > + uint16_t minor:4;
> > + uint16_t major:4;
> > + uint16_t reserved:4;
> > + };
> > + };
> > +} opae_bitstream_id;
> > +
> > +typedef struct {
> > + union {
> > + uint32_t version;
> > + struct {
> > + uint8_t micro;
> > + uint8_t minor;
> > + uint8_t major;
> > + uint8_t board;
> > + };
> > + };
> > +} opae_bmc_version;
> > +
> > +#define OPAE_MAX_PORT_NUM 4
> > +
> > +#define OPAE_PROP_PCI 0x01
> > +#define OPAE_PROP_FME 0x02
> > +#define OPAE_PROP_PORT 0x04
> > +#define OPAE_PROP_BMC 0x08
> > +#define OPAE_PROP_ALL \
> > + (OPAE_PROP_PCI | OPAE_PROP_FME | OPAE_PROP_PORT |
> > OPAE_PROP_BMC)
> > +
> > +typedef struct {
> > + opae_pci_property pci;
> > + opae_fme_property fme;
> > + opae_port_property port[OPAE_MAX_PORT_NUM];
> > + opae_bmc_property bmc;
> > +} opae_fpga_property;
> > +
> > +typedef struct {
> > + uint64_t guid_h;
> > + uint64_t guid_l;
> > + uint32_t metadata_len;
> > +} gbs_header;
> > +
> > +#define OPAE_IMG_TYPE_BBS 0
> > +#define OPAE_IMG_TYPE_BMC 1
> > +#define OPAE_IMG_TYPE_GBS 2
> > +#define OPAE_IMG_TYPE(t) ((t) & 0xff)
> > +
> > +#define OPAE_IMG_SUBTYPE_UPDATE 0
> > +#define OPAE_IMG_SUBTYPE_CANCELLATION 1
> > +#define OPAE_IMG_SUBTYPE_ROOT_KEY_HASH_256 2
> > +#define OPAE_IMG_SUBTYPE_ROOT_KEY_HASH_384 3
> > +#define OPAE_IMG_SUBTYPE(t) (((t) >> 8) & 0xff)
> > +
> > +#define OPAE_IMG_BLK0_SIZE 128
> > +#define OPAE_IMG_BLK0_MAGIC 0xb6eafd19
> > +#define OPAE_IMG_BLK1_SIZE 896
> > +#define OPAE_IMG_HDR_SIZE (OPAE_IMG_BLK0_SIZE +
> > OPAE_IMG_BLK1_SIZE)
> > +#define OPAE_IMG_PL_MIN_SIZE 128
> > +
> > +typedef struct {
> > + uint32_t magic;
> > + uint32_t payload_len;
> > + uint32_t payload_type;
> > +} opae_img_hdr;
> > +
> > +typedef struct {
> > + int type;
> > + int subtype;
> > + uint32_t total_len;
> > + uint32_t payload_offset;
> > + uint32_t payload_len;
> > +} opae_img_info;
> > +
> > +void opae_get_api_version(opae_api_version *version);
> > +void opae_check_pcidev_list(void);
> > +int opae_set_log_level(int level);
> > +int opae_set_log_file(char *path, int clean);
> > +int opae_get_proc_type(void);
> > +int opae_get_parent(pcidev_id id, pcidev_id parent);
> > +int opae_get_child(pcidev_id id, pcidev_id child, int size);
> > +int opae_get_pf1(pcidev_id id, pcidev_id peer, int size);
> > +int opae_init_eal(int argc, char **argv);
> > +int opae_cleanup_eal(void);
> > +int opae_enumerate(opae_pci_id *filter, pcidev_id list, int size);
> > +int opae_probe_device(pcidev_id id);
> > +int opae_remove_device(pcidev_id id);
> > +int opae_unbind_driver(pcidev_id id);
> > +int opae_bind_driver(pcidev_id id, char *drv_name);
> > +int opae_get_property(pcidev_id id, opae_fpga_property *prop, int
> type);
> > +int opae_get_phy_info(pcidev_id id, opae_phy_info *info);
> > +int opae_partial_reconfigure(pcidev_id id, int port, const char *gbs);
> > +int opae_get_image_info(const char *image, opae_img_info *info);
> > +int opae_cancel_flash_update(pcidev_id id, int force);
> > +int opae_update_flash(pcidev_id id, const char *image, uint64_t *status);
> > +int opae_reboot_device(pcidev_id id, int type, int page);
> > +int opae_store_rsu_status(pcidev_id id, uint32_t status, uint32_t
> progress);
> > +int opae_load_rsu_status(pcidev_id id, uint32_t *status, uint32_t
> > *progress);
> > +int opae_read_pci_cfg(pcidev_id id, uint32_t address, uint32_t *value);
> > +int opae_write_pci_cfg(pcidev_id id, uint32_t address, uint32_t value);
> > +
> > +#ifdef __cplusplus
> > +}
> > +#endif
> > +
> > +
> > +#endif /* _OPAE_API_H */
> > diff --git a/drivers/raw/ifpga/ifpga_rawdev.c
> > b/drivers/raw/ifpga/ifpga_rawdev.c
> > index 8dd566e44..64ed9903e 100644
> > --- a/drivers/raw/ifpga/ifpga_rawdev.c
> > +++ b/drivers/raw/ifpga/ifpga_rawdev.c
> > @@ -1738,6 +1738,79 @@
> > RTE_PMD_REGISTER_PARAM_STRING(ifpga_rawdev_cfg,
> > "port=<int> "
> > "afu_bts=<path>");
> >
> > +struct rte_pci_bus *ifpga_get_pci_bus(void)
> > +{
> > + return rte_ifpga_rawdev_pmd.bus;
> > +}
> > +
> > +int ifpga_rawdev_lock(struct rte_rawdev *dev)
> > +{
> > + if (!dev) {
> > + IFPGA_RAWDEV_PMD_ERR("rawdev is invalid");
> > + return -EINVAL;
> > + }
> > + return opae_adapter_lock(ifpga_rawdev_get_priv(dev), -1);
> > +}
> > +
> > +int ifpga_rawdev_unlock(struct rte_rawdev *dev)
> > +{
> > + if (!dev) {
> > + IFPGA_RAWDEV_PMD_ERR("rawdev is invalid");
> > + return -EINVAL;
> > + }
> > + return opae_adapter_unlock(ifpga_rawdev_get_priv(dev));
> > +}
> > +
> > +uint32_t ifpga_rawdev_get_rsu_stat(struct rte_rawdev *dev)
> > +{
> > + struct opae_adapter *adapter = NULL;
> > + opae_share_data *sd = NULL;
> > +
> > + if (!dev) {
> > + IFPGA_RAWDEV_PMD_ERR("rawdev is invalid");
> > + return 0;
> > + }
> > +
> > + adapter = ifpga_rawdev_get_priv(dev);
> > + if (!adapter) {
> > + IFPGA_RAWDEV_PMD_ERR("adapter is invalid");
> > + return 0;
> > + }
> > +
> > + sd = (opae_share_data *)adapter->shm.ptr;
> > + if (!sd) {
> > + IFPGA_RAWDEV_PMD_ERR("shared memory is invalid");
> > + return 0;
> > + }
> > +
> > + return sd->rsu_stat;
> > +}
> > +
> > +void ifpga_rawdev_set_rsu_stat(struct rte_rawdev *dev, uint32_t value)
> > +{
> > + struct opae_adapter *adapter = NULL;
> > + opae_share_data *sd = NULL;
> > +
> > + if (!dev) {
> > + IFPGA_RAWDEV_PMD_ERR("rawdev is invalid");
> > + return;
> > + }
> > +
> > + adapter = ifpga_rawdev_get_priv(dev);
> > + if (!adapter) {
> > + IFPGA_RAWDEV_PMD_ERR("adapter is invalid");
> > + return;
> > + }
> > +
> > + sd = (opae_share_data *)adapter->shm.ptr;
> > + if (!sd) {
> > + IFPGA_RAWDEV_PMD_ERR("shared memory is invalid");
> > + return;
> > + }
> > +
> > + sd->rsu_stat = value;
> > +}
> > +
> > int ifpga_rawdev_get_fme_property(struct rte_rawdev *dev,
> > ifpga_fme_property *prop)
> > {
> > @@ -1748,8 +1821,8 @@ int ifpga_rawdev_get_fme_property(struct
> > rte_rawdev *dev,
> > struct uuid pr_id;
> > int ret = 0;
> >
> > - if (!dev) {
> > - IFPGA_RAWDEV_PMD_ERR("rawdev is invalid");
> > + if (!dev || !prop) {
> > + IFPGA_RAWDEV_PMD_ERR("Input parameter is invalid");
> > return -EINVAL;
> > }
> >
> > @@ -1820,8 +1893,8 @@ int ifpga_rawdev_get_port_property(struct
> > rte_rawdev *dev, uint32_t port,
> > struct uuid afu_id;
> > int ret = 0;
> >
> > - if (!dev) {
> > - IFPGA_RAWDEV_PMD_ERR("rawdev is invalid");
> > + if (!dev || !prop) {
> > + IFPGA_RAWDEV_PMD_ERR("Input parameter is invalid");
> > return -EINVAL;
> > }
> >
> > @@ -1867,8 +1940,8 @@ int ifpga_rawdev_get_bmc_property(struct
> > rte_rawdev *dev,
> > struct opae_board_info *info = NULL;
> > int ret = 0;
> >
> > - if (!dev) {
> > - IFPGA_RAWDEV_PMD_ERR("rawdev is invalid");
> > + if (!dev || !prop) {
> > + IFPGA_RAWDEV_PMD_ERR("Input parameter is invalid");
> > return -EINVAL;
> > }
> >
> > @@ -1895,6 +1968,48 @@ int ifpga_rawdev_get_bmc_property(struct
> > rte_rawdev *dev,
> > return 0;
> > }
> >
> > +int ifpga_rawdev_get_phy_info(struct rte_rawdev *dev, ifpga_phy_info
> > *info)
> > +{
> > + struct opae_adapter *adapter = NULL;
> > + struct opae_retimer_info rtm_info;
> > + struct opae_retimer_status rtm_status;
> > + int ret = 0;
> > +
> > + if (!dev || !info) {
> > + IFPGA_RAWDEV_PMD_ERR("Input parameter is invalid");
> > + return -EINVAL;
> > + }
> > +
> > + adapter = ifpga_rawdev_get_priv(dev);
> > + if (!adapter) {
> > + IFPGA_RAWDEV_PMD_ERR("adapter is invalid");
> > + return -ENODEV;
> > + }
> > +
> > + if (!adapter->mgr) {
> > + IFPGA_RAWDEV_PMD_ERR("manager is invalid");
> > + return -ENODEV;
> > + }
> > +
> > + ret = opae_manager_get_retimer_info(adapter->mgr, &rtm_info);
> > + if (ret) {
> > + IFPGA_RAWDEV_PMD_ERR("Failed to get retimer info");
> > + return ret;
> > + }
> > +
> > + ret = opae_manager_get_retimer_status(adapter->mgr,
> > &rtm_status);
> > + if (ret) {
> > + IFPGA_RAWDEV_PMD_ERR("Failed to get retimer status");
> > + return ret;
> > + }
> > +
> > + info->num_retimers = rtm_info.nums_retimer;
> > + info->link_speed = rtm_status.speed;
> > + info->link_status = rtm_status.line_link_bitmap;
> > +
> > + return 0;
> > +}
> > +
> > int ifpga_rawdev_update_flash(struct rte_rawdev *dev, const char
> *image,
> > uint64_t *status)
> > {
> > @@ -1949,3 +2064,28 @@ int ifpga_rawdev_reload(struct rte_rawdev
> *dev,
> > int type, int page)
> >
> > return opae_mgr_reload(adapter->mgr, type, page);
> > }
> > +
> > +int ifpga_rawdev_partial_reconfigure(struct rte_rawdev *dev, int port,
> > + const char *file)
> > +{
> > + if (!dev) {
> > + IFPGA_RAWDEV_PMD_ERR("rawdev is invalid");
> > + return -EINVAL;
> > + }
> > +
> > + return rte_fpga_do_pr(dev, port, file);
> > +}
> > +
> > +void ifpga_rawdev_cleanup(void)
> > +{
> > + struct ifpga_rawdev *dev;
> > + unsigned int i;
> > +
> > + for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
> > + dev = &ifpga_rawdevices[i];
> > + if (dev->rawdev) {
> > + rte_rawdev_pmd_release(dev->rawdev);
> > + dev->rawdev = NULL;
> > + }
> > + }
> > +}
> > diff --git a/drivers/raw/ifpga/ifpga_rawdev.h
> > b/drivers/raw/ifpga/ifpga_rawdev.h
> > index d4be7913d..185e79071 100644
> > --- a/drivers/raw/ifpga/ifpga_rawdev.h
> > +++ b/drivers/raw/ifpga/ifpga_rawdev.h
> > @@ -89,6 +89,12 @@ typedef struct {
> > uint32_t fw_version;
> > } ifpga_bmc_property;
> >
> > +typedef struct {
> > + uint32_t num_retimers;
> > + uint32_t link_speed;
> > + uint32_t link_status;
> > +} ifpga_phy_info;
> > +
> > int
> > ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
> > enum ifpga_irq_type type, int vec_start, int count,
> > @@ -98,15 +104,24 @@ int
> > ifpga_unregister_msix_irq(enum ifpga_irq_type type,
> > int vec_start, rte_intr_callback_fn handler, void *arg);
> >
> > +struct rte_pci_bus *ifpga_get_pci_bus(void);
> > +int ifpga_rawdev_lock(struct rte_rawdev *dev);
> > +int ifpga_rawdev_unlock(struct rte_rawdev *dev);
> > +uint32_t ifpga_rawdev_get_rsu_stat(struct rte_rawdev *dev);
> > +void ifpga_rawdev_set_rsu_stat(struct rte_rawdev *dev, uint32_t value);
> > int ifpga_rawdev_get_fme_property(struct rte_rawdev *dev,
> > ifpga_fme_property *prop);
> > int ifpga_rawdev_get_port_property(struct rte_rawdev *dev, uint32_t
> port,
> > ifpga_port_property *prop);
> > int ifpga_rawdev_get_bmc_property(struct rte_rawdev *dev,
> > ifpga_bmc_property *prop);
> > +int ifpga_rawdev_get_phy_info(struct rte_rawdev *dev, ifpga_phy_info
> > *info);
> > int ifpga_rawdev_update_flash(struct rte_rawdev *dev, const char
> *image,
> > uint64_t *status);
> > int ifpga_rawdev_stop_flash_update(struct rte_rawdev *dev, int force);
> > int ifpga_rawdev_reload(struct rte_rawdev *dev, int type, int page);
> > +int ifpga_rawdev_partial_reconfigure(struct rte_rawdev *dev, int port,
> > + const char *file);
> > +void ifpga_rawdev_cleanup(void);
> >
> > #endif /* _IFPGA_RAWDEV_H_ */
> > diff --git a/drivers/raw/ifpga/meson.build
> b/drivers/raw/ifpga/meson.build
> > index 027ff8056..417480f19 100644
> > --- a/drivers/raw/ifpga/meson.build
> > +++ b/drivers/raw/ifpga/meson.build
> > @@ -13,8 +13,10 @@ objs = [base_objs]
> > deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs',
> > 'bus_vdev', 'bus_ifpga', 'net', 'net_i40e', 'net_ipn3ke']
> >
> > -sources = files('ifpga_rawdev.c')
> > +sources = files('ifpga_rawdev.c', 'ifpga_opae_api.c')
> >
> > includes += include_directories('base')
> > includes += include_directories('../../net/ipn3ke')
> > includes += include_directories('../../net/i40e')
> > +
> > +install_headers('ifpga_opae_api.h')
> > --
> > 2.29.2
