On 2025-03-07 04:01, Julien Grall wrote:
Hi,
On 06/03/2025 22:03, Jason Andryuk wrote:
Add capabilities property to dom0less to allow building a
disaggregated system.
Introduce bootfdt.h to contain these constants.
When using the hardware or xenstore capabilities, adjust the grant and
event channel limits similar to dom0.
> > Also for the hardware domain, set directmap and iommu. This brings
its
configuration in line with a dom0.
Looking the device tree bindings, a user would be allowed to disable
"passthrough" or even "directmap". This means, we would never be able to
disable "directmap" for the hardware domain in the future with the
existing property (this is to avoid break backwards compatibility).
Instead, I think we should check what the user provided and confirm this
is matching our expectation for an hardware domain.
>
That said, I am not entirely sure why we should force directmap for the
HW domain. We are starting from a clean slate, so I think it would be
better to have by default no directmap and imposing the presence of an
IOMMU in the system.
Ok, it seems like directmap is not necessary. It was helpful early on
to get things booting, but I think it's no longer necessary after
factoring out construct_hwdom().
What exactly do you mean by imposing with respect to the iommu? Require
one, or mirror the dom0 code and set it for the hwdom?
if ( iommu_enabled )
dom0_cfg.flags |= XEN_DOMCTL_CDF_iommu;
Lastly, can you provide an example of what the hardware domain DT node
would looke like?
I've attached a dump of /sys/firmware/fdt from hwdom. (This is direct
mapped).
--- a/xen/arch/arm/dom0less-build.c
+++ b/xen/arch/arm/dom0less-build.c
@@ -12,6 +12,7 @@
#include <xen/sizes.h>
#include <xen/vmap.h>
+#include <public/bootfdt.h>
#include <public/io/xs_wire.h>
#include <asm/arm64/sve.h>
@@ -994,6 +995,34 @@ void __init create_domUs(void)
if ( (max_init_domid + 1) >= DOMID_FIRST_RESERVED )
panic("No more domain IDs available\n");
+ if ( dt_property_read_u32(node, "capabilities", &val) )
+ {
+ if ( val & ~DOMAIN_CAPS_MASK )
+ panic("Invalid capabilities (%"PRIx32")\n", val);
+
+ if ( val & DOMAIN_CAPS_CONTROL )
+ flags |= CDF_privileged;
+
+ if ( val & DOMAIN_CAPS_HARDWARE )
+ {
+ if ( hardware_domain )
+ panic("Only 1 hardware domain can be specified!
(%pd)\n",
+ hardware_domain);
+
+ d_cfg.max_grant_frames = gnttab_dom0_frames();
+ d_cfg.max_evtchn_port = -1;
What about d_cfg.arch.nr_spis? Are we expecting the user to pass "nr_spis"?
Further down, when nr_spis isn't specified in the DT, it defaults to:
d_cfg.arch.nr_spis = gic_number_lines() - 32;
Dom0 does:
/*
* Xen vGIC supports a maximum of 992 interrupt lines.
* 32 are substracted to cover local IRQs.
*/
dom0_cfg.arch.nr_spis = min(gic_number_lines(), (unsigned int) 992)
- 32;
if ( gic_number_lines() > 992 )
printk(XENLOG_WARNING "Maximum number of vGIC IRQs exceeded.\n");
So I think it's fine as-is? I could add the min() and warning if you
think it's necessary.
Regards,
Jason
/dts-v1/;
/ {
interrupt-parent = <0x8005>;
dma-coherent;
model = "linux,dummy-virt";
#size-cells = <0x02>;
#address-cells = <0x02>;
compatible = "linux,dummy-virt";
platform-bus@c000000 {
interrupt-parent = <0x8005>;
ranges = <0x00 0x00 0xc000000 0x2000000>;
#address-cells = <0x01>;
#size-cells = <0x01>;
compatible = "qemu,platform", "simple-bus";
};
fw-cfg@9020000 {
dma-coherent;
reg = <0x00 0x9020000 0x00 0x18>;
compatible = "qemu,fw-cfg-mmio";
};
virtio_mmio@a000000 {
dma-coherent;
interrupts = <0x00 0x10 0x01>;
reg = <0x00 0xa000000 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a000200 {
dma-coherent;
interrupts = <0x00 0x11 0x01>;
reg = <0x00 0xa000200 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a000400 {
dma-coherent;
interrupts = <0x00 0x12 0x01>;
reg = <0x00 0xa000400 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a000600 {
dma-coherent;
interrupts = <0x00 0x13 0x01>;
reg = <0x00 0xa000600 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a000800 {
dma-coherent;
interrupts = <0x00 0x14 0x01>;
reg = <0x00 0xa000800 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a000a00 {
dma-coherent;
interrupts = <0x00 0x15 0x01>;
reg = <0x00 0xa000a00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a000c00 {
dma-coherent;
interrupts = <0x00 0x16 0x01>;
reg = <0x00 0xa000c00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a000e00 {
dma-coherent;
interrupts = <0x00 0x17 0x01>;
reg = <0x00 0xa000e00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a001000 {
dma-coherent;
interrupts = <0x00 0x18 0x01>;
reg = <0x00 0xa001000 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a001200 {
dma-coherent;
interrupts = <0x00 0x19 0x01>;
reg = <0x00 0xa001200 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a001400 {
dma-coherent;
interrupts = <0x00 0x1a 0x01>;
reg = <0x00 0xa001400 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a001600 {
dma-coherent;
interrupts = <0x00 0x1b 0x01>;
reg = <0x00 0xa001600 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a001800 {
dma-coherent;
interrupts = <0x00 0x1c 0x01>;
reg = <0x00 0xa001800 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a001a00 {
dma-coherent;
interrupts = <0x00 0x1d 0x01>;
reg = <0x00 0xa001a00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a001c00 {
dma-coherent;
interrupts = <0x00 0x1e 0x01>;
reg = <0x00 0xa001c00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a001e00 {
dma-coherent;
interrupts = <0x00 0x1f 0x01>;
reg = <0x00 0xa001e00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a002000 {
dma-coherent;
interrupts = <0x00 0x20 0x01>;
reg = <0x00 0xa002000 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a002200 {
dma-coherent;
interrupts = <0x00 0x21 0x01>;
reg = <0x00 0xa002200 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a002400 {
dma-coherent;
interrupts = <0x00 0x22 0x01>;
reg = <0x00 0xa002400 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a002600 {
dma-coherent;
interrupts = <0x00 0x23 0x01>;
reg = <0x00 0xa002600 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a002800 {
dma-coherent;
interrupts = <0x00 0x24 0x01>;
reg = <0x00 0xa002800 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a002a00 {
dma-coherent;
interrupts = <0x00 0x25 0x01>;
reg = <0x00 0xa002a00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a002c00 {
dma-coherent;
interrupts = <0x00 0x26 0x01>;
reg = <0x00 0xa002c00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a002e00 {
dma-coherent;
interrupts = <0x00 0x27 0x01>;
reg = <0x00 0xa002e00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a003000 {
dma-coherent;
interrupts = <0x00 0x28 0x01>;
reg = <0x00 0xa003000 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a003200 {
dma-coherent;
interrupts = <0x00 0x29 0x01>;
reg = <0x00 0xa003200 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a003400 {
dma-coherent;
interrupts = <0x00 0x2a 0x01>;
reg = <0x00 0xa003400 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a003600 {
dma-coherent;
interrupts = <0x00 0x2b 0x01>;
reg = <0x00 0xa003600 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a003800 {
dma-coherent;
interrupts = <0x00 0x2c 0x01>;
reg = <0x00 0xa003800 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a003a00 {
dma-coherent;
interrupts = <0x00 0x2d 0x01>;
reg = <0x00 0xa003a00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a003c00 {
dma-coherent;
interrupts = <0x00 0x2e 0x01>;
reg = <0x00 0xa003c00 0x00 0x200>;
compatible = "virtio,mmio";
};
virtio_mmio@a003e00 {
dma-coherent;
interrupts = <0x00 0x2f 0x01>;
reg = <0x00 0xa003e00 0x00 0x200>;
compatible = "virtio,mmio";
};
gpio-keys {
compatible = "gpio-keys";
poweroff {
gpios = <0x8007 0x03 0x00>;
linux,code = <0x74>;
label = "GPIO Key Poweroff";
};
};
pl061@9030000 {
phandle = <0x8007>;
clock-names = "apb_pclk";
clocks = <0x8000>;
interrupts = <0x00 0x07 0x04>;
gpio-controller;
#gpio-cells = <0x02>;
compatible = "arm,pl061", "arm,primecell";
reg = <0x00 0x9030000 0x00 0x1000>;
status = "disabled";
};
pcie@10000000 {
interrupt-map-mask = <0x1800 0x00 0x00 0x07>;
interrupt-map = <0x00 0x00 0x00 0x01 0x8005 0x00 0x00 0x00 0x03
0x04 0x00 0x00 0x00 0x02 0x8005 0x00 0x00 0x00 0x04 0x04 0x00 0x00 0x00 0x03
0x8005 0x00 0x00 0x00 0x05 0x04 0x00 0x00 0x00 0x04 0x8005 0x00 0x00 0x00 0x06
0x04 0x800 0x00 0x00 0x01 0x8005 0x00 0x00 0x00 0x04 0x04 0x800 0x00 0x00 0x02
0x8005 0x00 0x00 0x00 0x05 0x04 0x800 0x00 0x00 0x03 0x8005 0x00 0x00 0x00 0x06
0x04 0x800 0x00 0x00 0x04 0x8005 0x00 0x00 0x00 0x03 0x04 0x1000 0x00 0x00 0x01
0x8005 0x00 0x00 0x00 0x05 0x04 0x1000 0x00 0x00 0x02 0x8005 0x00 0x00 0x00
0x06 0x04 0x1000 0x00 0x00 0x03 0x8005 0x00 0x00 0x00 0x03 0x04 0x1000 0x00
0x00 0x04 0x8005 0x00 0x00 0x00 0x04 0x04 0x1800 0x00 0x00 0x01 0x8005 0x00
0x00 0x00 0x06 0x04 0x1800 0x00 0x00 0x02 0x8005 0x00 0x00 0x00 0x03 0x04
0x1800 0x00 0x00 0x03 0x8005 0x00 0x00 0x00 0x04 0x04 0x1800 0x00 0x00 0x04
0x8005 0x00 0x00 0x00 0x05 0x04>;
#interrupt-cells = <0x01>;
ranges = <0x1000000 0x00 0x00 0x00 0x3eff0000 0x00 0x10000
0x2000000 0x00 0x10000000 0x00 0x10000000 0x00 0x2eff0000 0x3000000 0x80 0x00
0x80 0x00 0x80 0x00>;
reg = <0x40 0x10000000 0x00 0x10000000>;
msi-map = <0x00 0x8006 0x00 0x10000>;
dma-coherent;
bus-range = <0x00 0xff>;
linux,pci-domain = <0x00>;
#size-cells = <0x02>;
#address-cells = <0x03>;
device_type = "pci";
compatible = "pci-host-ecam-generic";
};
pl031@9010000 {
clock-names = "apb_pclk";
clocks = <0x8000>;
interrupts = <0x00 0x02 0x04>;
reg = <0x00 0x9010000 0x00 0x1000>;
compatible = "arm,pl031", "arm,primecell";
};
intc@8000000 {
phandle = <0x8005>;
#address-cells = <0x02>;
#size-cells = <0x02>;
#interrupt-cells = <0x03>;
interrupt-controller;
compatible = "arm,gic-v3";
#redistributor-regions = <0x01>;
reg = <0x00 0x8000000 0x00 0x10000 0x00 0x80a0000 0x00
0xf60000>;
};
flash@0 {
bank-width = <0x04>;
reg = <0x00 0x00 0x00 0x4000000 0x00 0x4000000 0x00 0x4000000>;
compatible = "cfi-flash";
};
timer {
compatible = "arm,armv8-timer";
interrupts = <0x01 0x0d 0xf08 0x01 0x0e 0xf08 0x01 0x0b 0xf08>;
interrupt-parent = <0x8005>;
};
apb-pclk {
phandle = <0x8000>;
clock-output-names = "clk24mhz";
clock-frequency = <0x16e3600>;
#clock-cells = <0x00>;
compatible = "fixed-clock";
};
aliases {
serial0 = "/pl011@9000000";
};
chosen {
u-boot,version = "2024.10";
#size-cells = <0x02>;
#address-cells = <0x02>;
kaslr-seed = <0xcfbd6cf4 0xaf233870>;
bootargs = "console=hvc0 console=ttyAMA0 earlycon=xen
earlyprintk=xen clk_ignore_unused";
linux,initrd-start = <0x00 0xc8200000>;
linux,initrd-end = <0x00 0xcbd2faef>;
};
hypervisor {
compatible = "xen,xen-4.21", "xen,xen";
reg = <0x00 0x4d200000 0x00 0x40000 0x00 0x40000000 0x00
0xd000000 0x00 0x4d400000 0x00 0x72a00000 0x00 0xe0000000 0x00 0xfe00000 0x00
0xf8000000 0x00 0x32600000 0x01 0x2f400000 0x00 0x10a00000>;
interrupts = <0x01 0x00 0xf08>;
interrupt-parent = <0x8005>;
};
psci {
compatible = "arm,psci-1.0", "arm,psci-0.2", "arm,psci";
method = "hvc";
cpu_off = <0x01>;
cpu_on = <0x02>;
};
cpus {
#address-cells = <0x01>;
#size-cells = <0x00>;
cpu@0 {
compatible = "arm,cortex-a57";
device_type = "cpu";
reg = <0x00>;
enable-method = "psci";
};
};
memory@c0000000 {
device_type = "memory";
reg = <0x00 0xc0000000 0x00 0x20000000 0x00 0xf0000000 0x00
0x8000000 0x01 0x2a800000 0x00 0x3800000 0x01 0x2f000000 0x00 0x400000>;
};
};
