Module Name: src Committed By: imil Date: Wed Jan 15 13:16:23 UTC 2025
Modified Files: src/sys/arch/x86/pv: files.pv Added Files: src/share/man/man4: virtio_mmio.4 src/sys/arch/amd64/conf: MICROVM src/sys/dev/virtio/arch/x86: virtio_mmio_cmdline.c Log Message: Add support for command line MMIO devices, at least qemu and Firecracker pass MMIO virtual devices mapping through the kernel command line. This driver is based on Colin Percival's FreeBSD virtio_mmio_cmdline.c https://github.com/freebsd/freebsd-src/blob/main/sys/dev/virtio/mmio/virtio_mmio_cmdline.c The following kernel options are needed options MPBIOS options MPTABLE_LINUX_BUG_COMPAT As are these drivers pv* at pvbus? virtio* at pv? Exemple qemu usage on a Linux host to boot a NetBSD guest: qemu-system-x86_64 \ -M microvm,x-option-roms=off,rtc=on,acpi=off,pic=off,accel=kvm \ -m 256 -cpu host -kernel ${KERNEL} \ -append "root=ld0a console=com rw -v" \ -device virtio-blk-device,drive=hd0 \ -drive file=${IMG},format=raw,id=hd0 \ -device virtio-net-device,netdev=net0 \ -netdev user,id=net0,ipv6=off,hostfwd=::2200-:22 \ -global virtio-mmio.force-legacy=false -display none -serial stdio A lightweight kernel configuration named MICROVM if available for this use case. To generate a diff of this commit: cvs rdiff -u -r0 -r1.1 src/share/man/man4/virtio_mmio.4 cvs rdiff -u -r0 -r1.1 src/sys/arch/amd64/conf/MICROVM cvs rdiff -u -r1.1 -r1.2 src/sys/arch/x86/pv/files.pv cvs rdiff -u -r0 -r1.1 src/sys/dev/virtio/arch/x86/virtio_mmio_cmdline.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/x86/pv/files.pv diff -u src/sys/arch/x86/pv/files.pv:1.1 src/sys/arch/x86/pv/files.pv:1.2 --- src/sys/arch/x86/pv/files.pv:1.1 Thu Jan 2 10:34:33 2025 +++ src/sys/arch/x86/pv/files.pv Wed Jan 15 13:16:22 2025 @@ -2,4 +2,7 @@ define pvbus {} device pv {} attach pv at pvbus -file arch/x86/pv/pvbus.c pvbus needs-flag +file arch/x86/pv/pvbus.c pvbus needs-flag + +attach virtio at pv with mmio_cmdline: virtio_mmio +file dev/virtio/arch/x86/virtio_mmio_cmdline.c mmio_cmdline Added files: Index: src/share/man/man4/virtio_mmio.4 diff -u /dev/null src/share/man/man4/virtio_mmio.4:1.1 --- /dev/null Wed Jan 15 13:16:23 2025 +++ src/share/man/man4/virtio_mmio.4 Wed Jan 15 13:16:22 2025 @@ -0,0 +1,75 @@ +.\" $NetBSD: virtio_mmio.4,v 1.1 2025/01/15 13:16:22 imil Exp $ +.\" +.\" Copyright (c) 2024 The NetBSD Foundation, Inc. +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS +.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS +.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +.\" POSSIBILITY OF SUCH DAMAGE. +.\" +.Dd January 2024 +.Dt VIRTIO_MMIO 4 +.Os NetBSD +.Sh NAME +.Nm virtio_mmio +.Nd VirtIO over memory mapped device. +.Sh SYNOPSIS +.Cd "pv* at pvbus?" +.Cd "virtio* at pv?" +.Pp +.Cd "acpi0 at mainbus0" +.Cd "virtio* at acpi?" +.Sh DESCRIPTION +.Nm +can be used in virtual environments without +.Xr pci 4 +support +.Pq a common situation in embedded devices models +might use simple memory mapped device +.Pq Nm +instead of the +.Xr pci 4 +device. +.Pp +The memory mapped +.Xr virtio 4 +device behaviour is based on the +.Xr pci 4 +device specification. Therefore most operations including device initialization, +queues configuration and buffer transfers are nearly identical. +.Pp +Unlike +.Xr pci 4 , +.Nm +provides no generic device discovery mechanism. For each device, the guest OS will +need to know the location of the registers and interrupt(s) used. +.Pp +Device location can be read from either +.Xr acpi 4 +or via kernel command line parameters, implemented as a +.Xr pv 4 +virtual device. +.Sh SEE ALSO +.Xr virtio 4 +.Pp +.Rs +.%T Virtual I/O Device (VIRTIO) Version 1.2 +.%U https://docs.oasis-open.org/virtio/virtio/v1.2/virtio-v1.2.html +.Re Index: src/sys/arch/amd64/conf/MICROVM diff -u /dev/null src/sys/arch/amd64/conf/MICROVM:1.1 --- /dev/null Wed Jan 15 13:16:23 2025 +++ src/sys/arch/amd64/conf/MICROVM Wed Jan 15 13:16:22 2025 @@ -0,0 +1,143 @@ +# $NetBSD: MICROVM,v 1.1 2025/01/15 13:16:22 imil Exp $ +# +# MICROVM kernel configuration, for use with Qemu microvm machine type +# or Firecracker. +# Stripped-down configuration with no PCI, use VirtIO over MMIO virtual +# bus instead. ACPI is disabled as Firecracker doesn't support it, +# use legacy MP tables instead. +# +# Exemple qemu usage on a Linux host to boot a NetBSD guest: +# +# qemu-system-x86_64 \ +# -M microvm,x-option-roms=off,rtc=on,acpi=off,pic=off,accel=kvm \ +# -m 256 -cpu host -kernel ${KERNEL} \ +# -append "root=ld0a console=com rw -z" \ +# -display none -device virtio-blk-device,drive=hd0 \ +# -drive file=${IMG},format=raw,id=hd0 \ +# -device virtio-net-device,netdev=net0 \ +# -netdev user,id=net0,ipv6=off,hostfwd=::2200-:22 \ +# -global virtio-mmio.force-legacy=false -serial stdio + +machine amd64 x86 xen +include "conf/std" # MI standard options +include "arch/xen/conf/std.xenversion" + +options CPU_IN_CKSUM +options EXEC_ELF64 # exec ELF binaries +options EXEC_SCRIPT # exec #! scripts +options MTRR +options MULTIPROCESSOR + +options CHILD_MAX=1024 # 160 is too few +options OPEN_MAX=1024 # 128 is too few + +mainbus0 at root +cpu* at mainbus? +ioapic* at mainbus? apid ? + +options INCLUDE_CONFIG_FILE # embed config file in kernel binary +maxusers 8 # estimated number of users + +options INSECURE # disable kernel security levels - X needs this + +options RTC_OFFSET=0 # hardware clock is this many mins. west of GMT + +options PIPE_SOCKETPAIR # smaller, but slower pipe(2) + +# Xen PV support for PVH and HVM guests, needed for PVH boot +options XENPVHVM +options XEN +hypervisor* at mainbus? # Xen hypervisor +xenbus* at hypervisor? # Xen virtual bus +xencons* at hypervisor? # Xen virtual console + +# Include NetBSD 10 compatibility +options COMPAT_100 +# +# Because gcc omits the frame pointer for any -O level, the line below +# is needed to make backtraces in DDB work. +# +makeoptions COPTS="-O2 -fno-omit-frame-pointer" + +# File systems +#include "conf/filesystems.config" +file-system FFS +file-system EXT2FS +file-system KERNFS +file-system MFS +file-system TMPFS +file-system PTYFS +file-system MSDOSFS +file-system PROCFS + +options DKWEDGE_AUTODISCOVER +options DKWEDGE_METHOD_GPT +# File system options +# ffs +options FFS_NO_SNAPSHOT # No FFS snapshot support +options WAPBL # File system journaling support + +# Networking options +#options GATEWAY # packet forwarding +options INET # IP + ICMP + TCP + UDP +options INET6 # IPV6 + +# Kernel root file system and dump configuration. +config netbsd root on ? type ? + +# +# Device configuration +# + +# ACPI will be used if present. If not it will fall back to MPBIOS +options MPBIOS # configure CPUs and APICs using MPBIOS +# Provide bug-for-bug compatibility with Linux in MP Table searching +# and parsing. Firecracker relies on these bugs. +options MPTABLE_LINUX_BUG_COMPAT + +#pci* at mainbus? bus ? +#acpi0 at mainbus0 + +# ISA bus support +isa0 at mainbus? + +# ISA serial interfaces +com0 at isa? port 0x3f8 irq 4 # Standard PC serial ports + +# Virtual bus for non-PCI devices +pv* at pvbus? + +## Virtio devices +# Use MMIO by default +virtio* at pv? +#virtio* at acpi? +#virtio* at pci? dev ? function ? # Virtio PCI device +#viomb* at virtio? # Virtio memory balloon device + +ld* at virtio? # Virtio disk device +vioif* at virtio? # Virtio network device +viornd* at virtio? # Virtio entropy device +viocon* at virtio? + +vio9p* at virtio? # Virtio 9P device +#vioscsi* at virtio? +#scsibus* at vioscsi? + +pseudo-device md # memory disk device (ramdisk) +#options MEMORY_DISK_HOOKS # enable md specific hooks +#options MEMORY_DISK_DYNAMIC # enable dynamic resizing +# +pseudo-device vnd # disk-like interface to files +#options VND_COMPRESSION # compressed vnd(4) + +## network pseudo-devices +pseudo-device bpfilter # Berkeley packet filter +pseudo-device loop # network loopback + +## miscellaneous pseudo-devices +pseudo-device pty # pseudo-terminals +# userland interface to drivers, including autoconf and properties retrieval +pseudo-device drvctl + +file-system PUFFS +pseudo-device putter Index: src/sys/dev/virtio/arch/x86/virtio_mmio_cmdline.c diff -u /dev/null src/sys/dev/virtio/arch/x86/virtio_mmio_cmdline.c:1.1 --- /dev/null Wed Jan 15 13:16:23 2025 +++ src/sys/dev/virtio/arch/x86/virtio_mmio_cmdline.c Wed Jan 15 13:16:23 2025 @@ -0,0 +1,343 @@ +/* $NetBSD: virtio_mmio_cmdline.c,v 1.1 2025/01/15 13:16:23 imil Exp $ */ + +/*- + * Copyright (c) 2025 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Emile 'iMil' Heitor. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*- + * Copyright (c) 2022 Colin Percival + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/bus.h> +#include <sys/device.h> +#include <sys/kernel.h> +#include <sys/module.h> +#include <sys/systm.h> + +#define VIRTIO_PRIVATE +#include <dev/virtio/virtio_mmiovar.h> +#include <arch/x86/pv/pvvar.h> +#include <xen/hypervisor.h> + +#include <machine/i82093var.h> +#include "ioapic.h" + +#define VMMIOSTR "virtio_mmio.device=" + +struct mmio_args { + uint64_t sz; + uint64_t baseaddr; + uint64_t irq; + uint64_t id; +}; + +struct virtio_mmio_cmdline_softc { + struct virtio_mmio_softc sc_msc; + struct mmio_args margs; +}; + +static int virtio_mmio_cmdline_match(device_t, cfdata_t, void *); +static void virtio_mmio_cmdline_attach(device_t, device_t, void *); +static int virtio_mmio_cmdline_do_attach(device_t, + struct pv_attach_args *, struct mmio_args *); +static int virtio_mmio_cmdline_detach(device_t, int); +static int virtio_mmio_cmdline_rescan(device_t, const char *, const int *); +static int virtio_mmio_cmdline_alloc_interrupts(struct virtio_mmio_softc *); +static void virtio_mmio_cmdline_free_interrupts(struct virtio_mmio_softc *); + +CFATTACH_DECL3_NEW(mmio_cmdline, + sizeof(struct virtio_mmio_cmdline_softc), + virtio_mmio_cmdline_match, virtio_mmio_cmdline_attach, + virtio_mmio_cmdline_detach, NULL, + virtio_mmio_cmdline_rescan, NULL, 0); + +static int +virtio_mmio_cmdline_match(device_t parent, cfdata_t match, void *aux) +{ + if (strstr(xen_start_info.cmd_line, VMMIOSTR) == NULL) + return 0; + + return 1; +} + +static void +parsearg(struct mmio_args *margs, const char *arg) +{ + char *p; + + /* <size> */ + margs->sz = strtoull(arg, (char **)&p, 0); + if ((margs->sz == 0) || (margs->sz == UINT64_MAX)) + goto bad; + switch (*p) { + case 'E': case 'e': + /* Check for overflow */ + if (margs->sz > (UINT64_MAX >> 60)) + goto bad; + margs->sz <<= 10; + /* FALLTHROUGH */ + case 'P': case 'p': + if (margs->sz > (UINT64_MAX >> 50)) + goto bad; + margs->sz <<= 10; + /* FALLTHROUGH */ + case 'T': case 't': + if (margs->sz > (UINT64_MAX >> 40)) + goto bad; + margs->sz <<= 10; + /* FALLTHROUGH */ + case 'G': case 'g': + if (margs->sz > (UINT64_MAX >> 30)) + goto bad; + margs->sz <<= 10; + /* FALLTHROUGH */ + case 'M': case 'm': + if (margs->sz > (UINT64_MAX >> 20)) + goto bad; + margs->sz <<= 10; + /* FALLTHROUGH */ + case 'K': case 'k': + if (margs->sz > (UINT64_MAX >> 10)) + goto bad; + margs->sz <<= 10; + p++; + break; + } + + /* @<baseaddr> */ + if (*p++ != '@') + goto bad; + margs->baseaddr = strtoull(p, (char **)&p, 0); + if ((margs->baseaddr == 0) || (margs->baseaddr == UINT64_MAX)) + goto bad; + + /* :<irq> */ + if (*p++ != ':') + goto bad; + margs->irq = strtoull(p, (char **)&p, 0); + if ((margs->irq == 0) || (margs->irq == UINT64_MAX)) + goto bad; + + /* Optionally, :<id> */ + if (*p) { + if (*p++ != ':') + goto bad; + margs->id = strtoull(p, (char **)&p, 0); + if ((margs->id == 0) || (margs->id == UINT64_MAX)) + goto bad; + } else { + margs->id = 0; + } + + /* Should have reached the end of the string. */ + if (*p) + goto bad; + + return; + +bad: + aprint_error("Error parsing virtio_mmio parameter: %s\n", arg); +} + +static void +virtio_mmio_cmdline_attach(device_t parent, device_t self, void *aux) +{ + struct virtio_mmio_cmdline_softc *sc = device_private(self); + struct pv_attach_args *pvaa = aux; + struct mmio_args *margs = &sc->margs; + int keylen = strlen(VMMIOSTR); + char *next; + static char cmdline[LINE_MAX], *parg = NULL; + + aprint_normal("\n"); + aprint_naive("\n"); + + if (parg == NULL) { /* first pass */ + strlcpy(cmdline, xen_start_info.cmd_line, sizeof(cmdline)); + aprint_verbose_dev(self, "kernel parameters: %s\n", + cmdline); + parg = strstr(cmdline, VMMIOSTR); + } + + if (parg != NULL) { + parg += keylen; + if (!*parg) + return; + + next = parg; + while (*next && *next != ' ') /* find end of argument */ + next++; + if (*next) { /* space */ + *next++ = '\0'; /* end the argument string */ + next = strstr(next, VMMIOSTR); + } + + aprint_normal_dev(self, "viommio: %s\n", parg); + parsearg(margs, parg); + + if (virtio_mmio_cmdline_do_attach(self, pvaa, margs)) + return; + + if (next) { + parg = next; + config_found(parent, pvaa, NULL, CFARGS_NONE); + } + } +} + +static int +virtio_mmio_cmdline_do_attach(device_t self, + struct pv_attach_args *pvaa, + struct mmio_args *margs) +{ + struct virtio_mmio_cmdline_softc *sc = device_private(self); + struct virtio_mmio_softc *const msc = &sc->sc_msc; + struct virtio_softc *const vsc = &msc->sc_sc; + int error; + + msc->sc_iot = pvaa->pvaa_memt; + vsc->sc_dmat = pvaa->pvaa_dmat; + msc->sc_iosize = margs->sz; + vsc->sc_dev = self; + + error = bus_space_map(msc->sc_iot, margs->baseaddr, margs->sz, 0, + &msc->sc_ioh); + if (error) { + aprint_error_dev(self, "couldn't map %#" PRIx64 ": %d", + margs->baseaddr, error); + return error; + } + + msc->sc_alloc_interrupts = virtio_mmio_cmdline_alloc_interrupts; + msc->sc_free_interrupts = virtio_mmio_cmdline_free_interrupts; + + virtio_mmio_common_attach(msc); + virtio_mmio_cmdline_rescan(self, "virtio", NULL); + + return 0; +} + +static int +virtio_mmio_cmdline_detach(device_t self, int flags) +{ + struct virtio_mmio_cmdline_softc * const sc = device_private(self); + struct virtio_mmio_softc * const msc = &sc->sc_msc; + + return virtio_mmio_common_detach(msc, flags); +} + +static int +virtio_mmio_cmdline_rescan(device_t self, const char *ifattr, const int *locs) +{ + struct virtio_mmio_cmdline_softc *const sc = device_private(self); + struct virtio_mmio_softc *const msc = &sc->sc_msc; + struct virtio_softc *const vsc = &msc->sc_sc; + struct virtio_attach_args va; + + if (vsc->sc_child) + return 0; + + memset(&va, 0, sizeof(va)); + va.sc_childdevid = vsc->sc_childdevid; + + config_found(self, &va, NULL, CFARGS_NONE); + + if (virtio_attach_failed(vsc)) + return 0; + + return 0; +} + + +static int +virtio_mmio_cmdline_alloc_interrupts(struct virtio_mmio_softc *msc) +{ + struct virtio_mmio_cmdline_softc *const sc = + (struct virtio_mmio_cmdline_softc *)msc; + struct virtio_softc *const vsc = &msc->sc_sc; + struct ioapic_softc *ioapic; + struct pic *pic; + int irq = sc->margs.irq; + int pin = irq; + bool mpsafe; + + ioapic = ioapic_find_bybase(irq); + + if (ioapic != NULL) { + KASSERT(ioapic->sc_pic.pic_type == PIC_IOAPIC); + pic = &ioapic->sc_pic; + pin = irq - pic->pic_vecbase; + irq = -1; + } else + pic = &i8259_pic; + + mpsafe = (0 != (vsc->sc_flags & VIRTIO_F_INTR_MPSAFE)); + + msc->sc_ih = intr_establish_xname(irq, pic, pin, IST_LEVEL, vsc->sc_ipl, + virtio_mmio_intr, msc, mpsafe, device_xname(vsc->sc_dev)); + if (msc->sc_ih == NULL) { + aprint_error_dev(vsc->sc_dev, + "failed to establish interrupt\n"); + return -1; + } + aprint_normal_dev(vsc->sc_dev, "interrupting on %d\n", irq); + + return 0; +} + +static void +virtio_mmio_cmdline_free_interrupts(struct virtio_mmio_softc *msc) +{ + if (msc->sc_ih != NULL) { + intr_disestablish(msc->sc_ih); + msc->sc_ih = NULL; + } +} +