The branch main has been updated by markj:
URL:
https://cgit.FreeBSD.org/src/commit/?id=e20b74da223d675321618fe6d67858084d152c9e
commit e20b74da223d675321618fe6d67858084d152c9e
Author: Mark Johnston <ma...@freebsd.org>
AuthorDate: 2023-10-04 16:27:07 +0000
Commit: Mark Johnston <ma...@freebsd.org>
CommitDate: 2023-10-04 16:53:17 +0000
bhyve: Move vcpu initialization into a MD source file
- Make handling of x86 config options, like x86.x2apic, conditional to
amd64.
- Move fbsdrun_set_capabilities() and spinup_vcpu() to a new file,
bhyverun_machdep.c. The moved code is all highly x86 specific.
I'm not sure how best to handle the namespace. I'm using "bhyve_" for
MD functions called from MI code. We also have "fbsdrun_" for some MI
routines that are typically called from MD code. The file name is
prefixed by "bhyverun_".
Reviewed by: corvink
MFC after: 1 week
Sponsored by: Innovate UK
Differential Revision: https://reviews.freebsd.org/D40987
---
usr.sbin/bhyve/Makefile | 1 +
usr.sbin/bhyve/amd64/bhyverun_machdep.c | 125 ++++++++++++++++++++++++++++++++
usr.sbin/bhyve/bhyverun.c | 111 ++++------------------------
usr.sbin/bhyve/bhyverun.h | 12 ++-
4 files changed, 152 insertions(+), 97 deletions(-)
diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile
index 1a8191f9fd3f..de8e87d2ad49 100644
--- a/usr.sbin/bhyve/Makefile
+++ b/usr.sbin/bhyve/Makefile
@@ -21,6 +21,7 @@ SRCS= \
basl.c \
bhyvegc.c \
bhyverun.c \
+ bhyverun_machdep.c \
block_if.c \
bootrom.c \
config.c \
diff --git a/usr.sbin/bhyve/amd64/bhyverun_machdep.c
b/usr.sbin/bhyve/amd64/bhyverun_machdep.c
new file mode 100644
index 000000000000..c6926abe61bc
--- /dev/null
+++ b/usr.sbin/bhyve/amd64/bhyverun_machdep.c
@@ -0,0 +1,125 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <err.h>
+#include <stdbool.h>
+#include <stdlib.h>
+
+#include <vmmapi.h>
+
+#include "bhyverun.h"
+#include "config.h"
+#include "pci_lpc.h"
+
+void
+bhyve_init_config(void)
+{
+ init_config();
+
+ /* Set default values prior to option parsing. */
+ set_config_bool("acpi_tables", true);
+ set_config_bool("acpi_tables_in_memory", true);
+ set_config_value("memory.size", "256M");
+ set_config_bool("x86.strictmsr", true);
+ set_config_value("lpc.fwcfg", "bhyve");
+}
+
+void
+bhyve_init_vcpu(struct vcpu *vcpu)
+{
+ int err, tmp;
+
+ if (get_config_bool_default("x86.vmexit_on_hlt", false)) {
+ err = vm_get_capability(vcpu, VM_CAP_HALT_EXIT, &tmp);
+ if (err < 0) {
+ fprintf(stderr, "VM exit on HLT not supported\n");
+ exit(4);
+ }
+ vm_set_capability(vcpu, VM_CAP_HALT_EXIT, 1);
+ }
+
+ if (get_config_bool_default("x86.vmexit_on_pause", false)) {
+ /*
+ * pause exit support required for this mode
+ */
+ err = vm_get_capability(vcpu, VM_CAP_PAUSE_EXIT, &tmp);
+ if (err < 0) {
+ fprintf(stderr,
+ "SMP mux requested, no pause support\n");
+ exit(4);
+ }
+ vm_set_capability(vcpu, VM_CAP_PAUSE_EXIT, 1);
+ }
+
+ if (get_config_bool_default("x86.x2apic", false))
+ err = vm_set_x2apic_state(vcpu, X2APIC_ENABLED);
+ else
+ err = vm_set_x2apic_state(vcpu, X2APIC_DISABLED);
+
+ if (err) {
+ fprintf(stderr, "Unable to set x2apic state (%d)\n", err);
+ exit(4);
+ }
+
+ vm_set_capability(vcpu, VM_CAP_ENABLE_INVPCID, 1);
+
+ err = vm_set_capability(vcpu, VM_CAP_IPI_EXIT, 1);
+ assert(err == 0);
+}
+
+void
+bhyve_start_vcpu(struct vcpu *vcpu, bool bsp)
+{
+ int error;
+
+ if (bsp) {
+ if (lpc_bootrom()) {
+ error = vm_set_capability(vcpu,
+ VM_CAP_UNRESTRICTED_GUEST, 1);
+ if (error != 0) {
+ err(4, "ROM boot failed: unrestricted guest "
+ "capability not available");
+ }
+ error = vcpu_reset(vcpu);
+ assert(error == 0);
+ }
+ } else {
+ bhyve_init_vcpu(vcpu);
+
+ /*
+ * Enable the 'unrestricted guest' mode for APs.
+ *
+ * APs startup in power-on 16-bit mode.
+ */
+ error = vm_set_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
+ assert(error == 0);
+ }
+
+ fbsdrun_addcpu(vcpu_id(vcpu));
+}
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index d054ec1301f7..f42db8147d54 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -476,17 +476,20 @@ fbsdrun_start_thread(void *param)
return (NULL);
}
-static void
-fbsdrun_addcpu(struct vcpu_info *vi)
+void
+fbsdrun_addcpu(int vcpuid)
{
+ struct vcpu_info *vi;
pthread_t thr;
int error;
+ vi = &vcpu_info[vcpuid];
+
error = vm_activate_cpu(vi->vcpu);
if (error != 0)
err(EX_OSERR, "could not activate CPU %d", vi->vcpuid);
- CPU_SET_ATOMIC(vi->vcpuid, &cpumask);
+ CPU_SET_ATOMIC(vcpuid, &cpumask);
vm_suspend_cpu(vi->vcpu);
@@ -590,49 +593,6 @@ num_vcpus_allowed(struct vmctx *ctx, struct vcpu *vcpu)
return (1);
}
-static void
-fbsdrun_set_capabilities(struct vcpu *vcpu)
-{
- int err, tmp;
-
- if (get_config_bool_default("x86.vmexit_on_hlt", false)) {
- err = vm_get_capability(vcpu, VM_CAP_HALT_EXIT, &tmp);
- if (err < 0) {
- fprintf(stderr, "VM exit on HLT not supported\n");
- exit(4);
- }
- vm_set_capability(vcpu, VM_CAP_HALT_EXIT, 1);
- }
-
- if (get_config_bool_default("x86.vmexit_on_pause", false)) {
- /*
- * pause exit support required for this mode
- */
- err = vm_get_capability(vcpu, VM_CAP_PAUSE_EXIT, &tmp);
- if (err < 0) {
- fprintf(stderr,
- "SMP mux requested, no pause support\n");
- exit(4);
- }
- vm_set_capability(vcpu, VM_CAP_PAUSE_EXIT, 1);
- }
-
- if (get_config_bool_default("x86.x2apic", false))
- err = vm_set_x2apic_state(vcpu, X2APIC_ENABLED);
- else
- err = vm_set_x2apic_state(vcpu, X2APIC_DISABLED);
-
- if (err) {
- fprintf(stderr, "Unable to set x2apic state (%d)\n", err);
- exit(4);
- }
-
- vm_set_capability(vcpu, VM_CAP_ENABLE_INVPCID, 1);
-
- err = vm_set_capability(vcpu, VM_CAP_IPI_EXIT, 1);
- assert(err == 0);
-}
-
static struct vmctx *
do_open(const char *vmname)
{
@@ -697,26 +657,6 @@ do_open(const char *vmname)
return (ctx);
}
-static void
-spinup_vcpu(struct vcpu_info *vi, bool bsp)
-{
- int error;
-
- if (!bsp) {
- fbsdrun_set_capabilities(vi->vcpu);
-
- /*
- * Enable the 'unrestricted guest' mode for APs.
- *
- * APs startup in power-on 16-bit mode.
- */
- error = vm_set_capability(vi->vcpu, VM_CAP_UNRESTRICTED_GUEST,
1);
- assert(error == 0);
- }
-
- fbsdrun_addcpu(vi);
-}
-
static bool
parse_config_option(const char *option)
{
@@ -787,17 +727,6 @@ parse_gdb_options(const char *opt)
}
#endif
-static void
-set_defaults(void)
-{
-
- set_config_bool("acpi_tables", true);
- set_config_bool("acpi_tables_in_memory", true);
- set_config_value("memory.size", "256M");
- set_config_bool("x86.strictmsr", true);
- set_config_value("lpc.fwcfg", "bhyve");
-}
-
int
main(int argc, char *argv[])
{
@@ -814,8 +743,8 @@ main(int argc, char *argv[])
restore_file = NULL;
#endif
- init_config();
- set_defaults();
+ bhyve_init_config();
+
progname = basename(argv[0]);
#ifdef BHYVE_SNAPSHOT
@@ -825,9 +754,11 @@ main(int argc, char *argv[])
#endif
while ((c = getopt(argc, argv, optstr)) != -1) {
switch (c) {
+#ifdef __amd64__
case 'a':
set_config_bool("x86.x2apic", false);
break;
+#endif
case 'A':
/*
* NOP. For backward compatibility. Most systems don't
@@ -903,6 +834,7 @@ main(int argc, char *argv[])
if (!parse_config_option(optarg))
errx(EX_USAGE, "invalid configuration option
'%s'", optarg);
break;
+#ifdef __amd64__
case 'H':
set_config_bool("x86.vmexit_on_hlt", true);
break;
@@ -921,7 +853,6 @@ main(int argc, char *argv[])
case 'e':
set_config_bool("x86.strictio", true);
break;
-#ifdef __amd64__
case 'u':
set_config_bool("rtc.use_localtime", false);
break;
@@ -929,16 +860,18 @@ main(int argc, char *argv[])
case 'U':
set_config_value("uuid", optarg);
break;
+#ifdef __amd64__
case 'w':
set_config_bool("x86.strictmsr", false);
break;
+#endif
case 'W':
set_config_bool("virtio_msix", false);
break;
+#ifdef __amd64__
case 'x':
set_config_bool("x86.x2apic", true);
break;
-#ifdef __amd64__
case 'Y':
set_config_bool("x86.mptable", false);
break;
@@ -1012,7 +945,7 @@ main(int argc, char *argv[])
exit(4);
}
- fbsdrun_set_capabilities(bsp);
+ bhyve_init_vcpu(bsp);
/* Allocate per-VCPU resources. */
vcpu_info = calloc(guest_ncpus, sizeof(*vcpu_info));
@@ -1103,23 +1036,11 @@ main(int argc, char *argv[])
init_gdb(ctx);
#endif
-#ifdef __amd64__
- if (lpc_bootrom()) {
- if (vm_set_capability(bsp, VM_CAP_UNRESTRICTED_GUEST, 1)) {
- fprintf(stderr, "ROM boot failed: unrestricted guest "
- "capability not available\n");
- exit(4);
- }
- error = vcpu_reset(bsp);
- assert(error == 0);
- }
-#endif
-
/*
* Add all vCPUs.
*/
for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++)
- spinup_vcpu(&vcpu_info[vcpuid], vcpuid == BSP);
+ bhyve_start_vcpu(vcpu_info[vcpuid].vcpu, vcpuid == BSP);
#ifdef BHYVE_SNAPSHOT
if (restore_file != NULL) {
diff --git a/usr.sbin/bhyve/bhyverun.h b/usr.sbin/bhyve/bhyverun.h
index b1cfb99a964e..39e0916f08ef 100644
--- a/usr.sbin/bhyve/bhyverun.h
+++ b/usr.sbin/bhyve/bhyverun.h
@@ -26,8 +26,10 @@
* SUCH DAMAGE.
*/
-#ifndef _FBSDRUN_H_
-#define _FBSDRUN_H_
+#ifndef _BHYVERUN_H_
+#define _BHYVERUN_H_
+
+#include <stdbool.h>
#define VMEXIT_CONTINUE (0)
#define VMEXIT_ABORT (-1)
@@ -46,6 +48,7 @@ uintptr_t paddr_host2guest(struct vmctx *ctx, void *addr);
struct vcpu;
struct vcpu *fbsdrun_vcpu(int vcpuid);
+void fbsdrun_addcpu(int vcpuid);
void fbsdrun_deletecpu(int vcpuid);
int fbsdrun_suspendcpu(int vcpuid);
@@ -53,4 +56,9 @@ int fbsdrun_virtio_msix(void);
typedef int (*vmexit_handler_t)(struct vmctx *, struct vcpu *, struct vm_run
*);
+/* Interfaces implemented by machine-dependent code. */
+void bhyve_init_config(void);
+void bhyve_init_vcpu(struct vcpu *vcpu);
+void bhyve_start_vcpu(struct vcpu *vcpu, bool bsp);
+
#endif
