> On May 22, 2025, at 2:52, Jonathan Cameron <jonathan.came...@huawei.com>
> wrote:
>
> On Wed, 21 May 2025 16:38:10 +0900
> Itaru Kitayama <itaru.kitay...@linux.dev> wrote:
>
>>> On May 19, 2025, at 21:54, Jonathan Cameron <jonathan.came...@huawei.com>
>>> wrote:
>>>
>>> On Thu, 15 May 2025 18:04:18 +0900
>>> Itaru Kitayama <itaru.kitay...@linux.dev> wrote:
>>>
>>>>> On May 13, 2025, at 20:14, Jonathan Cameron <jonathan.came...@huawei.com>
>>>>> wrote:
>>>>>
>>>>> Add a single complex case for aarch64 virt machine.
>>>>> Given existing much more comprehensive tests for x86 cover the
>>>>> common functionality, a single test should be enough to verify
>>>>> that the aarch64 part continue to work.
>>>>>
>>>>> Signed-off-by: Jonathan Cameron <jonathan.came...@huawei.com>
>>>>> ---
>>>>> tests/qtest/cxl-test.c | 59 ++++++++++++++++++++++++++++++++---------
>>>>> tests/qtest/meson.build | 1 +
>>>>> 2 files changed, 47 insertions(+), 13 deletions(-)
>>>>>
>>>>> diff --git a/tests/qtest/cxl-test.c b/tests/qtest/cxl-test.c
>>>>> index a600331843..c7189d6222 100644
>>>>> --- a/tests/qtest/cxl-test.c
>>>>> +++ b/tests/qtest/cxl-test.c
>>>>> @@ -19,6 +19,12 @@
>>>>> "-device pxb-cxl,id=cxl.1,bus=pcie.0,bus_nr=53 " \
>>>>> "-M
>>>>> cxl-fmw.0.targets.0=cxl.0,cxl-fmw.0.targets.1=cxl.1,cxl-fmw.0.size=4G "
>>>>>
>>>>> +#define QEMU_VIRT_2PXB_CMD \
>>>>> + "-machine virt,cxl=on -cpu max " \
>>>>> + "-device pxb-cxl,id=cxl.0,bus=pcie.0,bus_nr=52 " \
>>>>> + "-device pxb-cxl,id=cxl.1,bus=pcie.0,bus_nr=53 " \
>>>>> + "-M
>>>>> cxl-fmw.0.targets.0=cxl.0,cxl-fmw.0.targets.1=cxl.1,cxl-fmw.0.size=4G "
>>>>> +
>>>>> #define QEMU_RP \
>>>>> "-device cxl-rp,id=rp0,bus=cxl.0,chassis=0,slot=0 "
>>>>>
>>>>> @@ -197,25 +203,52 @@ static void cxl_2pxb_4rp_4t3d(void)
>>>>> qtest_end();
>>>>> rmdir(tmpfs);
>>>>> }
>>>>> +
>>>>> +static void cxl_virt_2pxb_4rp_4t3d(void)
>>>>> +{
>>>>> + g_autoptr(GString) cmdline = g_string_new(NULL);
>>>>> + char template[] = "/tmp/cxl-test-XXXXXX";
>>>>> + const char *tmpfs;
>>>>> +
>>>>> + tmpfs = mkdtemp(template);
>>>>> +
>>>>> + g_string_printf(cmdline, QEMU_VIRT_2PXB_CMD QEMU_4RP QEMU_4T3D,
>>>>> + tmpfs, tmpfs, tmpfs, tmpfs, tmpfs, tmpfs,
>>>>> + tmpfs, tmpfs);
>>>>> +
>>>>> + qtest_start(cmdline->str);
>>>>> + qtest_end();
>>>>> + rmdir(tmpfs);
>>>>> +}
>>>>> #endif /* CONFIG_POSIX */
>>>>>
>>>>> int main(int argc, char **argv)
>>>>> {
>>>>> - g_test_init(&argc, &argv, NULL);
>>>>> + const char *arch = qtest_get_arch();
>>>>>
>>>>> - qtest_add_func("/pci/cxl/basic_hostbridge", cxl_basic_hb);
>>>>> - qtest_add_func("/pci/cxl/basic_pxb", cxl_basic_pxb);
>>>>> - qtest_add_func("/pci/cxl/pxb_with_window", cxl_pxb_with_window);
>>>>> - qtest_add_func("/pci/cxl/pxb_x2_with_window", cxl_2pxb_with_window);
>>>>> - qtest_add_func("/pci/cxl/rp", cxl_root_port);
>>>>> - qtest_add_func("/pci/cxl/rp_x2", cxl_2root_port);
>>>>> + g_test_init(&argc, &argv, NULL);
>>>>> + if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
>>>>> + qtest_add_func("/pci/cxl/basic_hostbridge", cxl_basic_hb);
>>>>> + qtest_add_func("/pci/cxl/basic_pxb", cxl_basic_pxb);
>>>>> + qtest_add_func("/pci/cxl/pxb_with_window", cxl_pxb_with_window);
>>>>> + qtest_add_func("/pci/cxl/pxb_x2_with_window",
>>>>> cxl_2pxb_with_window);
>>>>> + qtest_add_func("/pci/cxl/rp", cxl_root_port);
>>>>> + qtest_add_func("/pci/cxl/rp_x2", cxl_2root_port);
>>>>> #ifdef CONFIG_POSIX
>>>>> - qtest_add_func("/pci/cxl/type3_device", cxl_t3d_deprecated);
>>>>> - qtest_add_func("/pci/cxl/type3_device_pmem", cxl_t3d_persistent);
>>>>> - qtest_add_func("/pci/cxl/type3_device_vmem", cxl_t3d_volatile);
>>>>> - qtest_add_func("/pci/cxl/type3_device_vmem_lsa",
>>>>> cxl_t3d_volatile_lsa);
>>>>> - qtest_add_func("/pci/cxl/rp_x2_type3_x2", cxl_1pxb_2rp_2t3d);
>>>>> - qtest_add_func("/pci/cxl/pxb_x2_root_port_x4_type3_x4",
>>>>> cxl_2pxb_4rp_4t3d);
>>>>> + qtest_add_func("/pci/cxl/type3_device", cxl_t3d_deprecated);
>>>>> + qtest_add_func("/pci/cxl/type3_device_pmem", cxl_t3d_persistent);
>>>>> + qtest_add_func("/pci/cxl/type3_device_vmem", cxl_t3d_volatile);
>>>>> + qtest_add_func("/pci/cxl/type3_device_vmem_lsa",
>>>>> cxl_t3d_volatile_lsa);
>>>>> + qtest_add_func("/pci/cxl/rp_x2_type3_x2", cxl_1pxb_2rp_2t3d);
>>>>> + qtest_add_func("/pci/cxl/pxb_x2_root_port_x4_type3_x4",
>>>>> + cxl_2pxb_4rp_4t3d);
>>>>> #endif
>>>>> + } else if (strcmp(arch, "aarch64") == 0) {
>>>>> +#ifdef CONFIG_POSIX
>>>>> + qtest_add_func("/pci/cxl/virt/pxb_x2_root_port_x4_type3_x4",
>>>>> + cxl_virt_2pxb_4rp_4t3d);
>>>>> +#endif
>>>>> + }
>>>>> +
>>>>> return g_test_run();
>>>>> }
>>>>> diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build
>>>>> index 7daf619845..361000267a 100644
>>>>> --- a/tests/qtest/meson.build
>>>>> +++ b/tests/qtest/meson.build
>>>>> @@ -258,6 +258,7 @@ qtests_aarch64 = \
>>>>> (config_all_accel.has_key('CONFIG_TCG') and
>>>>> \
>>>>> config_all_devices.has_key('CONFIG_TPM_TIS_I2C') ? ['tpm-tis-i2c-test']
>>>>> : []) + \
>>>>> (config_all_devices.has_key('CONFIG_ASPEED_SOC') ? qtests_aspeed64 : [])
>>>>> + \
>>>>> + qtests_cxl +
>>>>> \
>>>>> ['arm-cpu-features',
>>>>> 'numa-test',
>>>>> 'boot-serial-test',
>>>>> --
>>>>> 2.43.0
>>>>>
>>>>
>>>> ~/projects/qemu/build$ meson test qtest-aarch64/cxl-test
>>>> ninja: Entering directory `/home/realm/projects/qemu/build'
>>>> [1/8] Generating qemu-version.h with a custom command (wrapped by meson to
>>>> capture output)
>>>> 1/1 qemu:qtest+qtest-aarch64 / qtest-aarch64/cxl-test OK
>>>> 0.17s 1 subtests passed
>>>>
>>>> Ok: 1
>>>> Expected Fail: 0
>>>> Fail: 0
>>>> Unexpected Pass: 0
>>>> Skipped: 0
>>>> Timeout: 0
>>>>
>>>> Tested-by: Itaru Kitayama <itaru.kitay...@fujitsu.com
>>>> <mailto:itaru.kitay...@fujitsu.com>>
>>>>
>>>> Jonathan, would you push your branch this series applied? I manually
>>>> applied your series no issues though.
>>>
>>> I'm reluctant to push a 'normal' staging CXL tree whilst we have the TCG
>>> issue outstanding (which is in mainline).
>>> I can probably push one with a name that makes it clear we know it will
>>> crash under some circumstances though. I'll aim to get that done later this
>>> week.
>>>
>>> After talking to Richard Henderson I'm going to spin some images etc to
>>> make it easier for him to replicate that TCG issue.
>>
>> While QEMU (the kernel is built off of cxl branch) boots fine and lspci
>> shows CXL devices as shown:
>>
>> root@localhost:~# lspci -mm
>> 00:00.0 "Host bridge" "Red Hat, Inc." "QEMU PCIe Host bridge" -p00 "Red Hat,
>> Inc." "Device 1100"
>> 00:01.0 "SCSI storage controller" "Red Hat, Inc." "Virtio block device" -p00
>> "Red Hat, Inc." "Device 0002"
>> 00:02.0 "Ethernet controller" "Intel Corporation" "82540EM Gigabit Ethernet
>> Controller" -r03 -p00 "Red Hat, Inc." "QEMU Virtual Machine"
>> 00:03.0 "Host bridge" "Red Hat, Inc." "QEMU PCIe Expander bridge" -p00 "Red
>> Hat, Inc." "Device 1100"
>> 00:04.0 "Host bridge" "Red Hat, Inc." "QEMU PCIe Expander bridge" -p00 "Red
>> Hat, Inc." "Device 1100"
>> 35:00.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel
>> Corporation" "Device 0000"
>> 35:01.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel
>> Corporation" "Device 0000"
>> 36:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a128" -p00 "" ""
>> 36:00.1 "Serial bus controller [0c0b]" "Huawei Technologies Co., Ltd."
>> "Device a123" -p00 "Red Hat, Inc." "Device 1100"
>> 37:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
>> 37:01.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
>> 37:02.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
>> 37:03.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
>> 38:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc."
>> "Device 1100"
>> 3a:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc."
>> "Device 1100"
>> bf:00.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel
>> Corporation" "Device 0000"
>> bf:01.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel
>> Corporation" "Device 0000"
>> c0:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a128" -p00 "" ""
>> c0:00.1 "Serial bus controller [0c0b]" "Huawei Technologies Co., Ltd."
>> "Device a123" -p00 "Red Hat, Inc." "Device 1100"
>> c1:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
>> c1:01.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
>> c1:02.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
>> c1:03.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
>> c2:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc."
>> "Device 1100"
>> c4:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc."
>> "Device 1100”
>>
>> but the cxl-list command takes 5-10 minutes to return the information. I’ll
>> test with your qtest minimalist setup to see if the system is loaded. Am I
>> seeing the TCG issue you mentioned?
>
> I don't think so. That should only matter if the memory is hotplugged in
> as normal system RAM.
>
>>
>> root@localhost:~# cxl list -M
>> [
>> {
>> "memdev":"mem10",
>> "ram_size":2147483648,
>> "ram_qos_class":42,
>> "serial":11,
>> "numa_node":0,
>> "host":"cxl_rcd.10",
>> "firmware_version":"mock fw v1 "
>
> So this seems to be with both some emulated devices from qemu command line
> and CXL test at the same time.
>
> I replicated a similar setup and not setting a significant delay
> (half a second maybe?) So not sure.
>
Good news. And I confirm that the cxl-list command returns a list quickly with
older cxl/next kernel(s).
I am still in the process of bisecting trying to narrow down the bad commit.
Itaru.
> I did notice that there is a bug in qemu though which has surfaced
> for some reason in the kernel log (it dates all the way back to
> origin CXL support series). In
> include/hw/cxl/cxl_pci.h
> #define REG_LOC_DEVSEC_LENGTH is 0x24 and it should be 0x1C
>
> That results for me in a kernel log entry about a bar not being
> big enough for a huge offset (which is random data coming from
> somewhere in text I think).
>
> Seems unlikely to trigger your issue, but you never know!
>
> Jonathan
>
>
>> },
>> {
>> "memdev":"mem5",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":6,
>> "numa_node":1,
>> "host":"cxl_mem.5",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem1",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":2,
>> "numa_node":1,
>> "host":"cxl_mem.1",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem6",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":8,
>> "numa_node":1,
>> "host":"cxl_mem.7",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem3",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":4,
>> "numa_node":1,
>> "host":"cxl_mem.3",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem4",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":5,
>> "numa_node":0,
>> "host":"cxl_mem.4",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem0",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":1,
>> "numa_node":0,
>> "host":"cxl_mem.0",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem2",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":3,
>> "numa_node":0,
>> "host":"cxl_mem.2",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem7",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":7,
>> "numa_node":0,
>> "host":"cxl_mem.6",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem8",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":9,
>> "numa_node":0,
>> "host":"cxl_mem.8",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem9",
>> "pmem_size":1073741824,
>> "pmem_qos_class":42,
>> "ram_size":1073741824,
>> "ram_qos_class":42,
>> "serial":10,
>> "numa_node":1,
>> "host":"cxl_mem.9",
>> "firmware_version":"mock fw v1 "
>> },
>> {
>> "memdev":"mem12",
>> "ram_size":268435456,
>> "serial":0,
>> "host":"0000:c4:00.0",
>> "firmware_version":"BWFW VERSION 00"
>> },
>> {
>> "memdev":"mem11",
>> "ram_size":268435456,
>> "serial":0,
>> "host":"0000:c2:00.0",
>> "firmware_version":"BWFW VERSION 00"
>> },
>> {
>> "memdev":"mem14",
>> "pmem_size":268435456,
>> "serial":0,
>> "host":"0000:3a:00.0",
>> "firmware_version":"BWFW VERSION 00"
>> },
>> {
>> "memdev":"mem13",
>> "pmem_size":268435456,
>> "serial":0,
>> "host":"0000:38:00.0",
>> "firmware_version":"BWFW VERSION 00"
>> }
>> ]
>>
>> Getting this shouldn’t take minutes, even with the emulator I think.
>>
>> Itaru.
>>
>>>
>>> Thanks for reviews.
>>>
>>> Jonathan
