Implement the scaffolding for an MHSLD with a simple multi-headed
command set. This device inherits the cxl-type3 device and will
provide the hooks needed to serialize multi-system devices.
This device requires linux, as it uses shared memory (shmem) to
implement the shared state. This also limits the emulation of
multiple guests to a single host.
To instantiate:
-device
cxl-mhsld,bus=rp0,volatile-memdev=mem0,id=cxl-mem0,sn=66666,mhd-head=0,mhd-shmid=15
The MH-SLD shared memory region must be initialized via 'init_mhsld'
tool located in the cxl/mhsld directory.
usage: init_mhsld <heads> <shmid>
heads : number of heads on the device
shmid : shmid produced by ipcmk
Example:
$shmid1=ipcmk -M 131072
./init_mhseld 4 $shmid
What we should discuss:
* What state needs to be in MHSLDSharedState
- for DCD
- for other CXL extentions? (RAS?)
* What hooks we need to add to cxl-type3 for MHDs
- Example: cvc->mhd_access_valid was added for Niagara
- Additional calls will need to be added for things like DCD
* Shared state design and serialization
- right now we use the same shared memory solution, this makes
serialization a little easier since we can use a mutex in shmem
- This limits emulation of MHSLD devices to QEMU guests located
on the same host, but this seems fine for functional testing.
* Shared state initialization
- right now i use the external tool, but if we want to limit
the ability to test things like restarting hosts, we could
instead add an explicit `initialize_state=true` option to the
device and ditch the extra program.
Signed-off-by: Gregory Price <gregory.pr...@memverge.com>
---
hw/cxl/Kconfig | 1 +
hw/cxl/meson.build | 1 +
hw/cxl/mhsld/.gitignore | 1 +
hw/cxl/mhsld/Kconfig | 4 +
hw/cxl/mhsld/init_mhsld.c | 76 ++++++++++++++++
hw/cxl/mhsld/meson.build | 3 +
hw/cxl/mhsld/mhsld.c | 177 ++++++++++++++++++++++++++++++++++++++
hw/cxl/mhsld/mhsld.h | 64 ++++++++++++++
8 files changed, 327 insertions(+)
create mode 100644 hw/cxl/mhsld/.gitignore
create mode 100644 hw/cxl/mhsld/Kconfig
create mode 100644 hw/cxl/mhsld/init_mhsld.c
create mode 100644 hw/cxl/mhsld/meson.build
create mode 100644 hw/cxl/mhsld/mhsld.c
create mode 100644 hw/cxl/mhsld/mhsld.h
diff --git a/hw/cxl/Kconfig b/hw/cxl/Kconfig
index e603839a62..919e59b598 100644
--- a/hw/cxl/Kconfig
+++ b/hw/cxl/Kconfig
@@ -1,3 +1,4 @@
+source mhsld/Kconfig
source vendor/Kconfig
config CXL
diff --git a/hw/cxl/meson.build b/hw/cxl/meson.build
index e8c8c1355a..394750dd19 100644
--- a/hw/cxl/meson.build
+++ b/hw/cxl/meson.build
@@ -16,4 +16,5 @@ system_ss.add(when: 'CONFIG_I2C_MCTP_CXL', if_true:
files('i2c_mctp_cxl.c'))
system_ss.add(when: 'CONFIG_ALL', if_true: files('cxl-host-stubs.c'))
+subdir('mhsld')
subdir('vendor')
diff --git a/hw/cxl/mhsld/.gitignore b/hw/cxl/mhsld/.gitignore
new file mode 100644
index 0000000000..f9f6a37cc0
--- /dev/null
+++ b/hw/cxl/mhsld/.gitignore
@@ -0,0 +1 @@
+init_mhsld
diff --git a/hw/cxl/mhsld/Kconfig b/hw/cxl/mhsld/Kconfig
new file mode 100644
index 0000000000..dc2be15140
--- /dev/null
+++ b/hw/cxl/mhsld/Kconfig
@@ -0,0 +1,4 @@
+config CXL_MHSLD
+ bool
+ depends on CXL_MEM_DEVICE
+ default y
diff --git a/hw/cxl/mhsld/init_mhsld.c b/hw/cxl/mhsld/init_mhsld.c
new file mode 100644
index 0000000000..d9e0cd54e0
--- /dev/null
+++ b/hw/cxl/mhsld/init_mhsld.c
@@ -0,0 +1,76 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * Copyright (c) 2024 MemVerge Inc.
+ *
+ */
+
+#include <signal.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+struct mhsld_state {
+ uint8_t nr_heads;
+ uint8_t nr_lds;
+ uint8_t ldmap[65536];
+};
+
+int main(int argc, char *argv[])
+{
+ int shmid = 0;
+ uint32_t sections = 0;
+ uint32_t section_size = 0;
+ uint32_t heads = 0;
+ struct mhsld_state *mhsld_state = NULL;
+ size_t state_size;
+ uint8_t i;
+
+ if (argc != 3) {
+ printf("usage: init_mhsld <heads> <shmid>\n"
+ "\theads : number of heads on the device\n"
+ "\tshmid : /tmp/mytoken.tmp\n\n"
+ "It is recommended your shared memory region is at least
128kb\n");
+ return -1;
+ }
+
+ /* must have at least 1 head */
+ heads = (uint32_t)atoi(argv[1]);
+ if (heads == 0 || heads > 32) {
+ printf("bad heads argument (1-32)\n");
+ return -1;
+ }
+
+ shmid = (uint32_t)atoi(argv[2]);
+ if (shmid == 0) {
+ printf("bad shmid argument\n");
+ return -1;
+ }
+
+ mhsld_state = shmat(shmid, NULL, 0);
+ if (mhsld_state == (void *)-1) {
+ printf("Unable to attach to shared memory\n");
+ return -1;
+ }
+
+ /* Initialize the mhsld_state */
+ state_size = sizeof(struct mhsld_state) + (sizeof(uint32_t) * sections);
+ memset(mhsld_state, 0, state_size);
+ mhsld_state->nr_heads = heads;
+ mhsld_state->nr_lds = heads;
+
+ memset(&mhsld_state->ldmap, '\xff', sizeof(mhsld_state->ldmap));
+ for (i = 0; i < heads; i++) {
+ mhsld_state->ldmap[i] = i;
+ }
+
+ printf("mhsld initialized\n");
+ shmdt(mhsld_state);
+ return 0;
+}
diff --git a/hw/cxl/mhsld/meson.build b/hw/cxl/mhsld/meson.build
new file mode 100644
index 0000000000..c595558f8a
--- /dev/null
+++ b/hw/cxl/mhsld/meson.build
@@ -0,0 +1,3 @@
+if host_os == 'linux'
+ system_ss.add(when: 'CONFIG_CXL_MHSLD', if_true: files('mhsld.c',))
+endif
diff --git a/hw/cxl/mhsld/mhsld.c b/hw/cxl/mhsld/mhsld.c
new file mode 100644
index 0000000000..bef50e1483
--- /dev/null
+++ b/hw/cxl/mhsld/mhsld.c
@@ -0,0 +1,177 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * Copyright (c) 2024 MemVerge Inc.
+ *
+ */
+
+#include <sys/shm.h>
+#include "qemu/osdep.h"
+#include "hw/irq.h"
+#include "migration/vmstate.h"
+#include "qapi/error.h"
+#include "hw/cxl/cxl.h"
+#include "hw/cxl/cxl_mailbox.h"
+#include "hw/cxl/cxl_device.h"
+#include "hw/pci/pcie.h"
+#include "hw/pci/pcie_port.h"
+#include "hw/qdev-properties.h"
+#include "mhsld.h"
+
+#define TYPE_CXL_MHSLD "cxl-mhsld"
+OBJECT_DECLARE_TYPE(CXLMHSLDState, CXLMHSLDClass, CXL_MHSLD)
+
+/*
+ * CXL r3.0 section 7.6.7.5.1 - Get Multi-Headed Info (Opcode 5500h)
+ *
+ * This command retrieves the number of heads, number of supported LDs,
+ * and Head-to-LD mapping of a Multi-Headed device.
+ */
+static CXLRetCode cmd_mhd_get_info(const struct cxl_cmd *cmd,
+ uint8_t *payload_in,
+ size_t len_in,
+ uint8_t *payload_out,
+ size_t *len_out,
+ CXLCCI * cci)
+{
+ CXLMHSLDState *s = CXL_MHSLD(cci->d);
+ MHDGetInfoInput *input = (void *)payload_in;
+ MHDGetInfoOutput *output = (void *)payload_out;
+
+ uint8_t start_ld = input->start_ld;
+ uint8_t ldmap_len = input->ldmap_len;
+ uint8_t i;
+
+ if (start_ld >= s->mhd_state->nr_lds) {
+ return CXL_MBOX_INVALID_INPUT;
+ }
+
+ output->nr_lds = s->mhd_state->nr_lds;
+ output->nr_heads = s->mhd_state->nr_heads;
+ output->resv1 = 0;
+ output->start_ld = start_ld;
+ output->resv2 = 0;
+
+ for (i = 0; i < ldmap_len && (start_ld + i) < output->nr_lds; i++) {
+ output->ldmap[i] = s->mhd_state->ldmap[start_ld + i];
+ }
+ output->ldmap_len = i;
+
+ *len_out = sizeof(*output) + output->ldmap_len;
+ return CXL_MBOX_SUCCESS;
+}
+
+static const struct cxl_cmd cxl_cmd_set_mhsld[256][256] = {
+ [MHSLD_MHD][GET_MHD_INFO] = {"GET_MULTI_HEADED_INFO",
+ cmd_mhd_get_info, 2, 0},
+};
+
+static Property cxl_mhsld_props[] = {
+ DEFINE_PROP_UINT32("mhd-head", CXLMHSLDState, mhd_head, ~(0)),
+ DEFINE_PROP_UINT32("mhd-shmid", CXLMHSLDState, mhd_shmid, 0),
+ DEFINE_PROP_END_OF_LIST(),
+};
+
+static void cxl_mhsld_realize(PCIDevice *pci_dev, Error **errp)
+{
+ CXLMHSLDState *s = CXL_MHSLD(pci_dev);
+
+ ct3_realize(pci_dev, errp);
+
+ if (!s->mhd_shmid || s->mhd_head == ~(0)) {
+ error_setg(errp, "is_mhd requires mhd_shmid and mhd_head settings");
+ return;
+ }
+
+ if (s->mhd_head >= 32) {
+ error_setg(errp, "MHD Head ID must be between 0-31");
+ return;
+ }
+
+ s->mhd_state = shmat(s->mhd_shmid, NULL, 0);
+ if (s->mhd_state == (void *)-1) {
+ s->mhd_state = NULL;
+ error_setg(errp, "Unable to attach MHD State. Check ipcs is valid");
+ return;
+ }
+
+ /* For now, limit the number of LDs to the number of heads (SLD) */
+ if (s->mhd_head >= s->mhd_state->nr_heads) {
+ error_setg(errp, "Invalid head ID for multiheaded device.");
+ return;
+ }
+
+ if (s->mhd_state->nr_lds <= s->mhd_head) {
+ error_setg(errp, "MHD Shared state does not have sufficient lds.");
+ return;
+ }
+
+ s->mhd_state->ldmap[s->mhd_head] = s->mhd_head;
+ return;
+}
+
+static void cxl_mhsld_exit(PCIDevice *pci_dev)
+{
+ CXLMHSLDState *s = CXL_MHSLD(pci_dev);
+
+ ct3_exit(pci_dev);
+
+ if (s->mhd_state) {
+ shmdt(s->mhd_state);
+ }
+}
+
+static void cxl_mhsld_reset(DeviceState *d)
+{
+ CXLMHSLDState *s = CXL_MHSLD(d);
+
+ ct3d_reset(d);
+ cxl_add_cci_commands(&s->ct3d.cci, cxl_cmd_set_mhsld, 512);
+}
+
+/* TODO: Implement shared device hooks
+ *
+ * Example: DCD-add events need to validate that the requested extent
+ * does not already have a mapping (or, if it does, it is
+ * a shared extent with the right tagging).
+ *
+ * Since this operates on the shared state, we will need to serialize
+ * these callbacks across QEMU instances via a mutex in shared state.
+ */
+
+static void cxl_mhsld_class_init(ObjectClass *klass, void *data)
+{
+ DeviceClass *dc = DEVICE_CLASS(klass);
+ PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
+
+ pc->realize = cxl_mhsld_realize;
+ pc->exit = cxl_mhsld_exit;
+ dc->reset = cxl_mhsld_reset;
+ device_class_set_props(dc, cxl_mhsld_props);
+
+ /* TODO: type3 class callbacks for multi-host access validation
+ * CXLType3Class *cvc = CXL_TYPE3_CLASS(klass);
+ * cvc->mhd_access_valid = mhsld_access_valid;
+ * cvc->dcd_extent_action = mhsld_dcd_extent_action;
+ */
+}
+
+static const TypeInfo cxl_mhsld_info = {
+ .name = TYPE_CXL_MHSLD,
+ .parent = TYPE_CXL_TYPE3,
+ .class_size = sizeof(struct CXLMHSLDClass),
+ .class_init = cxl_mhsld_class_init,
+ .instance_size = sizeof(CXLMHSLDState),
+ .interfaces = (InterfaceInfo[]) {
+ { INTERFACE_CXL_DEVICE },
+ { INTERFACE_PCIE_DEVICE },
+ {}
+ },
+};
+
+static void cxl_mhsld_register_types(void)
+{
+ type_register_static(&cxl_mhsld_info);
+}
+
+type_init(cxl_mhsld_register_types)
diff --git a/hw/cxl/mhsld/mhsld.h b/hw/cxl/mhsld/mhsld.h
new file mode 100644
index 0000000000..0cde86ea8b
--- /dev/null
+++ b/hw/cxl/mhsld/mhsld.h
@@ -0,0 +1,64 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * Copyright (c) 2024 MemVerge Inc.
+ *
+ */
+
+#ifndef CXL_MHSLD_H
+#define CXL_MHSLD_H
+#include <stdint.h>
+#include "hw/cxl/cxl.h"
+#include "hw/cxl/cxl_mailbox.h"
+#include "hw/cxl/cxl_device.h"
+#include "qemu/units.h"
+
+#define MHSLD_MIN_MEMBLK (128 * MiB)
+
+/*
+ * The shared state cannot have 2 variable sized regions
+ * so we have to max out the ldmap.
+ */
+typedef struct MHSLDSharedState {
+ uint8_t nr_heads;
+ uint8_t nr_lds;
+ uint8_t ldmap[65536];
+ /* TODO: shared state Mutex */
+ /* TODO: Extent information */
+} MHSLDSharedState;
+
+struct CXLMHSLDState {
+ CXLType3Dev ct3d;
+ uint32_t mhd_head;
+ uint32_t mhd_shmid;
+ MHSLDSharedState *mhd_state;
+};
+
+struct CXLMHSLDClass {
+ CXLType3Class parent_class;
+};
+
+enum {
+ MHSLD_MHD = 0x55,
+ #define GET_MHD_INFO 0x0
+};
+
+/*
+ * MHD Get Info Command
+ * Returns information the LD's associated with this head
+ */
+typedef struct MHDGetInfoInput {
+ uint8_t start_ld;
+ uint8_t ldmap_len;
+} QEMU_PACKED MHDGetInfoInput;
+
+typedef struct MHDGetInfoOutput {
+ uint8_t nr_lds;
+ uint8_t nr_heads;
+ uint16_t resv1;
+ uint8_t start_ld;
+ uint8_t ldmap_len;
+ uint16_t resv2;
+ uint8_t ldmap[];
+} QEMU_PACKED MHDGetInfoOutput;
+#endif
--
2.39.1
