On Fri, 2025-03-28 at 12:06 -0700, Farhan Ali wrote:
> Starting with z15 (or newer) we can execute mmio
> instructions from userspace. On older platforms
> where we don't have these instructions available
> we can fallback to using system calls to access
> the PCI mapped resources.
>
> This patch adds helper functions for mmio reads
> and writes for s390x.
>
> Reviewed-by: Stefan Hajnoczi <stefa...@redhat.com>
> Signed-off-by: Farhan Ali <al...@linux.ibm.com>
> ---
> include/qemu/s390x_pci_mmio.h | 23 ++++++
> util/meson.build | 2 +
> util/s390x_pci_mmio.c | 148 ++++++++++++++++++++++++++++++++++
> 3 files changed, 173 insertions(+)
> create mode 100644 include/qemu/s390x_pci_mmio.h
> create mode 100644 util/s390x_pci_mmio.c
>
> diff --git a/include/qemu/s390x_pci_mmio.h b/include/qemu/s390x_pci_mmio.h
> new file mode 100644
> index 0000000000..aead791475
> --- /dev/null
> +++ b/include/qemu/s390x_pci_mmio.h
> @@ -0,0 +1,23 @@
> +/*
> + * s390x PCI MMIO definitions
> + *
> + * Copyright 2025 IBM Corp.
> + * Author(s): Farhan Ali <al...@linux.ibm.com>
> + *
> + * SPDX-License-Identifier: GPL-2.0-or-later
> + */
> +#ifndef S390X_PCI_MMIO_H
> +#define S390X_PCI_MMIO_H
> +
> +uint8_t s390x_pci_mmio_read_8(const void *ioaddr);
> +uint16_t s390x_pci_mmio_read_16(const void *ioaddr);
> +uint32_t s390x_pci_mmio_read_32(const void *ioaddr);
> +uint64_t s390x_pci_mmio_read_64(const void *ioaddr);
> +
> +void s390x_pci_mmio_write_8(void *ioaddr, uint8_t val);
> +void s390x_pci_mmio_write_16(void *ioaddr, uint16_t val);
> +void s390x_pci_mmio_write_32(void *ioaddr, uint32_t val);
> +void s390x_pci_mmio_write_64(void *ioaddr, uint64_t val);
> +
> +
> +#endif
> diff --git a/util/meson.build b/util/meson.build
> index 780b5977a8..acb21592f9 100644
> --- a/util/meson.build
> +++ b/util/meson.build
> @@ -131,4 +131,6 @@ elif cpu in ['ppc', 'ppc64']
> util_ss.add(files('cpuinfo-ppc.c'))
> elif cpu in ['riscv32', 'riscv64']
> util_ss.add(files('cpuinfo-riscv.c'))
> +elif cpu == 's390x'
> + util_ss.add(files('s390x_pci_mmio.c'))
> endif
> diff --git a/util/s390x_pci_mmio.c b/util/s390x_pci_mmio.c
> new file mode 100644
> index 0000000000..820458a026
> --- /dev/null
> +++ b/util/s390x_pci_mmio.c
> @@ -0,0 +1,148 @@
> +/*
> + * s390x PCI MMIO definitions
> + *
> + * Copyright 2025 IBM Corp.
> + * Author(s): Farhan Ali <al...@linux.ibm.com>
> + *
> + * SPDX-License-Identifier: GPL-2.0-or-later
> + */
> +
> +#include "qemu/osdep.h"
> +#include <unistd.h>
> +#include <sys/syscall.h>
> +#include "qemu/s390x_pci_mmio.h"
> +#include "elf.h"
> +
> +union register_pair {
> + unsigned __int128 pair;
> + struct {
> + uint64_t even;
> + uint64_t odd;
> + };
> +};
> +
> +static bool is_mio_supported;
> +
> +static __attribute__((constructor)) void check_is_mio_supported(void)
> +{
> + is_mio_supported = !!(qemu_getauxval(AT_HWCAP) & HWCAP_S390_PCI_MIO);
> +}
> +
> +static uint64_t s390x_pcilgi(const void *ioaddr, size_t len)
> +{
> + union register_pair ioaddr_len = { .even = (uint64_t)ioaddr,
> + .odd = len };
> + uint64_t val;
> + int cc;
> +
> + asm volatile(
> + /* pcilgi */
> + ".insn rre,0xb9d60000,%[val],%[ioaddr_len]\n"
> + "ipm %[cc]\n"
> + "srl %[cc],28\n"
> + : [cc] "=d"(cc), [val] "=d"(val),
> + [ioaddr_len] "+&d"(ioaddr_len.pair) :: "cc");
> +
> + if (cc) {
> + val = -1ULL;
> + }
> +
> + return val;
> +}
> +
> +static void s390x_pcistgi(void *ioaddr, uint64_t val, size_t len)
> +{
> + union register_pair ioaddr_len = {.even = (uint64_t)ioaddr, .odd = len};
> +
> + asm volatile (
> + /* pcistgi */
> + ".insn rre,0xb9d40000,%[val],%[ioaddr_len]\n"
> + : [ioaddr_len] "+&d" (ioaddr_len.pair)
> + : [val] "d" (val)
> + : "cc", "memory");
> +}
I can confirm that the PCI MIO inline assembly looks good to me.
Pretty much exactly matches what I did for rdma-core a while back.
> +
> +uint8_t s390x_pci_mmio_read_8(const void *ioaddr)
> +{
> + uint8_t val = 0;
> +
> + if (is_mio_supported) {
> + val = s390x_pcilgi(ioaddr, sizeof(val));
> + } else {
> + syscall(__NR_s390_pci_mmio_read, ioaddr, &val, sizeof(val));
> + }
> + return val;
> +}
> +
>
--- snip ---
> +
> +void s390x_pci_mmio_write_64(void *ioaddr, uint64_t val)
> +{
> + if (is_mio_supported) {
> + s390x_pcistgi(ioaddr, val, sizeof(val));
> + } else {
> + syscall(__NR_s390_pci_mmio_write, ioaddr, &val, sizeof(val));
> + }
> +}
> +
Thanks for the great work!
Reviewed-by: Niklas Schnelle <schne...@linux.ibm.com>
