Ping. Any replies or commeents on this?
On Tue, Sep 15, 2020 at 12:54:49PM -0500, Jordan Hargrave wrote:
> This adds a placeholder vmmpci device that will be used for VMD passthrough
> PCI.
>
> Normally the device will fail to attach unless the PCI domain:bus.dev.func has
> been registered with vmmpci_add. When the device is registered, it will
> detach
> any existing PCI device and reload as vmmpci. It also attaches an interrupt
> handler
> and keeps a running counter of triggered interrupts. VMD will use the counter
> to
> issue commands through to the guest.
>
> diff --git a/sys/arch/amd64/conf/GENERIC b/sys/arch/amd64/conf/GENERIC
> index 2c49f91a1..a69c72c26 100644
> --- a/sys/arch/amd64/conf/GENERIC
> +++ b/sys/arch/amd64/conf/GENERIC
> @@ -108,6 +109,7 @@ ksmn* at pci? # AMD K17 temperature
> sensor
> amas* at pci? disable # AMD memory configuration
> pchtemp* at pci? # Intel C610 termperature sensor
> ccp* at pci? # AMD Cryptographic Co-processor
> +vmmpci* at pci? # VMM Placeholder
>
> # National Semiconductor LM7[89] and compatible hardware monitors
> lm0 at isa? port 0x290
> diff --git a/sys/arch/amd64/conf/files.amd64 b/sys/arch/amd64/conf/files.amd64
> index 7a5d40bf4..74c7fe5a9 100644
> --- a/sys/arch/amd64/conf/files.amd64
> +++ b/sys/arch/amd64/conf/files.amd64
> @@ -132,6 +132,10 @@ device pchb: pcibus, agpbus
> attach pchb at pci
> file arch/amd64/pci/pchb.c pchb
>
> +device vmmpci
> +attach vmmpci at pci
> +file arch/amd64/pci/vmmpci.c vmmpci
> +
> # AMAS AMD memory address switch
> device amas
> attach amas at pci
> diff --git a/sys/arch/amd64/include/vmmpci.h b/sys/arch/amd64/include/vmmpci.h
> new file mode 100644
> index 000000000..e012194df
> --- /dev/null
> +++ b/sys/arch/amd64/include/vmmpci.h
> @@ -0,0 +1,24 @@
> +/* $OpenBSD: vmmvar.h,v 1.70 2020/04/08 07:39:48 pd Exp $ */
> +/*
> + * Copyright (c) 2020 Jordan Hargrave <[email protected]>
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +#ifndef _VMMPCI_H_
> +#define _VMMPCI_H_
> +
> +int vmmpci_find(int, pcitag_t);
> +int vmmpci_add(int, pcitag_t, int);
> +int vmmpci_pending(int, pcitag_t, uint32_t *);
> +
> +#endif
> diff --git a/sys/arch/amd64/pci/vmmpci.c b/sys/arch/amd64/pci/vmmpci.c
> new file mode 100644
> index 000000000..a99efb502
> --- /dev/null
> +++ b/sys/arch/amd64/pci/vmmpci.c
> @@ -0,0 +1,186 @@
> +/* $OpenBSD: pcib.c,v 1.6 2013/05/30 16:15:01 deraadt Exp $ */
> +/* $NetBSD: pcib.c,v 1.6 1997/06/06 23:29:16 thorpej Exp $ */
> +
> +/*-
> + * Copyright (c) 1996 Jordan Hargrave<[email protected]>
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +
> +#include <sys/param.h>
> +#include <sys/systm.h>
> +#include <sys/device.h>
> +
> +#include <machine/bus.h>
> +
> +#include <dev/pci/pcivar.h>
> +#include <dev/pci/pcireg.h>
> +
> +#include <dev/pci/pcidevs.h>
> +
> +#include <machine/vmmvar.h>
> +#include <machine/vmmpci.h>
> +
> +struct vmmpci_softc {
> + struct device sc_dev;
> + void *sc_ih;
> +
> + int sc_domain;
> + pci_chipset_tag_t sc_pc;
> + pcitag_t sc_tag;
> +
> + uint32_t pending; // pending interrupt
> count
> +};
> +
> +#define VP_VALID 0x80000000
> +
> +/* Keep track of registered devices */
> +struct vmmpci_dev {
> + int vp_flags;
> + int vp_domain;
> + pcitag_t vp_tag;
> +};
> +
> +int vmmpci_match(struct device *, void *, void *);
> +void vmmpci_attach(struct device *, struct device *, void *);
> +void vmmpci_callback(struct device *);
> +int vmmpci_print(void *, const char *);
> +int vmmpci_intr(void *arg);
> +
> +struct cfattach vmmpci_ca = {
> + sizeof(struct vmmpci_softc), vmmpci_match, vmmpci_attach
> +};
> +
> +struct cfdriver vmmpci_cd = {
> + NULL, "vmmpci", DV_DULL
> +};
> +
> +#define MAXVMMPCI 4
> +
> +struct vmmpci_dev vmmpcis[MAXVMMPCI];
> +
> +/* Interrupt handler. Increase pending count for ioctl. TODO:better method?
> */
> +int
> +vmmpci_intr(void *arg)
> +{
> + struct vmmpci_softc *sc = arg;
> +
> + sc->pending++;
> + return 1;
> +}
> +
> +/* Get number of pending interrupts for a device */
> +int
> +vmmpci_pending(int domain, pcitag_t tag, uint32_t *pending)
> +{
> + struct vmmpci_softc *sc;
> +
> + /* Are we mapped? */
> + if (!vmmpci_find(domain, tag))
> + return (0);
> +
> + /* If we are mapped, the device should be a VMMPCI */
> + sc = (struct vmmpci_softc *)pci_find_bytag(domain, tag);
> + if (sc == NULL)
> + return (0);
> +
> + /* Return current pending count */
> + *pending = sc->pending;
> + return (1);
> +}
> +
> +/* Check if this PCI device has been registered */
> +int
> +vmmpci_find(int domain, pcitag_t tag)
> +{
> + int i;
> +
> + for (i = 0; i < MAXVMMPCI; i++) {
> + if ((vmmpcis[i].vp_flags & VP_VALID) &&
> + vmmpcis[i].vp_domain == domain &&
> + vmmpcis[i].vp_tag == tag)
> + return (1);
> + }
> + return (0);
> +}
> +
> +/* Add a PCI device to valid passthrough list and reprobe */
> +int
> +vmmpci_add(int domain, pcitag_t tag, int flags)
> +{
> + struct pci_softc *psc;
> + struct device *pd;
> + int i;
> +
> + /* Check if we are already mapped */
> + if (vmmpci_find(domain, tag))
> + return (1);
> +
> + for (i = 0; i < MAXVMMPCI; i++) {
> + if ((vmmpcis[i].vp_flags & VP_VALID) == 0) {
> + /* Find parent device */
> + pd = (struct device *)pci_find_bytag(domain, tag);
> + if (pd == NULL)
> + return (0);
> +
> + vmmpcis[i].vp_domain = domain;
> + vmmpcis[i].vp_tag = tag;
> + vmmpcis[i].vp_flags = VP_VALID | flags;
> +
> + /* detach the old device, reattach */
> + psc = (struct pci_softc *)pd->dv_parent;
> + config_detach(pd, 0);
> +
> + pci_probe_device(psc, tag, NULL, NULL);
> + return (1);
> + }
> + }
> + return (0);
> +}
> +
> +int
> +vmmpci_match(struct device *parent, void *match, void *aux)
> +{
> + struct pci_attach_args *pa = aux;
> + int rc;
> +
> + /* Check if device is registered, claim it */
> + rc = vmmpci_find(pa->pa_domain, pa->pa_tag);
> + if (rc)
> + return (100);
> + return (0);
> +}
> +
> +void
> +vmmpci_attach(struct device *parent, struct device *self, void *aux)
> +{
> + struct vmmpci_softc *sc = (struct vmmpci_softc *)self;
> + struct pci_attach_args *pa = aux;
> + pci_chipset_tag_t pc = pa->pa_pc;
> + pci_intr_handle_t ih;
> +
> + sc->sc_pc = pc;
> + sc->sc_tag = pa->pa_tag;
> +
> + /* Map our interrupt (TODO: what about devices with no interrupt?) */
> + if (pci_intr_map_msi(pa, &ih) || pci_intr_map(pa, &ih)) {
> + printf(": couldn't map interrupt\n");
> + return;
> + }
> + sc->sc_ih = pci_intr_establish(pc, ih, IPL_BIO, vmmpci_intr,
> + sc, sc->sc_dev.dv_xname);
> + if (sc->sc_ih == NULL) {
> + printf(": couldn't establish interrupt");
> + return;
> + }
> +}
> diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
> index bc20739e5..01af8e3b7 100644
> --- a/sys/dev/pci/pci.c
> +++ b/sys/dev/pci/pci.c
> @@ -1162,6 +1162,34 @@ pci_disable_legacy_vga(struct device *dev)
> }
> }
>
> +struct device *
> +pci_find_bytag(int domain, pcitag_t tag)
> +{
> + struct pci_softc *pci;
> + struct pci_dev *pd;
> + int bus, i;
> +
> + /* Lookup pci softc by domain/bus */
> + for (i = 0; i < pci_cd.cd_ndevs; i++) {
> + pci = pci_cd.cd_devs[i];
> + if (pci == NULL || pci->sc_domain != domain)
> + continue;
> + pci_decompose_tag(pci->sc_pc, tag, &bus, NULL, NULL);
> + if (pci->sc_bus == bus)
> + break;
> + }
> +
> + if (!pci)
> + return NULL;
> +
> + /* Lookup by tag (devfn) */
> + LIST_FOREACH(pd, &pci->sc_devs, pd_next)
> + if (tag == pd->pd_tag)
> + return (void *)pd->pd_dev;
> +
> + return NULL;
> +}
> +
> #ifdef USER_PCICONF
> /*
> * This is the user interface to PCI configuration space.
> diff --git a/sys/dev/pci/pcivar.h b/sys/dev/pci/pcivar.h
> index e85f6fe81..6cd0b4660 100644
> --- a/sys/dev/pci/pcivar.h
> +++ b/sys/dev/pci/pcivar.h
> @@ -284,5 +284,7 @@ const struct pci_quirkdata *
> pci_lookup_quirkdata(pci_vendor_id_t, pci_product_id_t);
> void pciagp_set_pchb(struct pci_attach_args *);
>
> +struct device *pci_find_bytag(int, pcitag_t);
> +
> #endif /* _KERNEL */
> #endif /* _DEV_PCI_PCIVAR_H_ */
