So you now know what command in the dpdk section that dpdk doesn’t like.
Try adding “log-level debug” in the dpdk section of startup.conf to see if you
can find more helpful messages in “vppctl show log” from dpdk why it fails to
probe the NIC.
Steven
From: <vpp-dev@lists.fd.io> on behalf of Gencli Liu <18600640...@163.com>
Date: Tuesday, January 7, 2020 at 7:42 PM
To: "vpp-dev@lists.fd.io" <vpp-dev@lists.fd.io>
Subject: Re: [vpp-dev] "vppctl show int" no NIC (just local0) #vpp #vnet
Hi steven：
Thank you for your reply!
I followed your advice(3), and made some attempts.
I create three startup config files of vpp:
The first one is named "startup.conf.smp", the second one is named
"startup.conf"(my config file).
And The third one is named "startup.conf.ok", it just delete "uio-driver
vfio-pci" on the basis of "startup.conf".
--------- startup.conf.smp ----------------
unix { interactive }
---------- startup.conf -------------------
unix {
nodaemon
log /var/log/vpp/vpp.log
full-coredump
cli-listen /run/vpp/cli.sock
gid vpp
}
api-trace { on }
api-segment { gid vpp }
socksvr { default }
cpu {
main-core 30
corelist-workers 26,28
workers 2
}
dpdk {
dev default {
num-rx-queues 1
num-tx-queues 2
}
dev 0000:3b:00.0
dev 0000:3b:00.1
#dev 0000:3b:00.2
#dev 0000:3b:00.3
uio-driver vfio-pci
}
---------- startup.conf.ok -------------------
unix {
nodaemon
log /var/log/vpp/vpp.log
full-coredump
cli-listen /run/vpp/cli.sock
gid vpp
}
api-trace { on }
api-segment { gid vpp }
socksvr { default }
cpu {
main-core 30
corelist-workers 26,28
workers 2
}
dpdk {
dev default {
num-rx-queues 1
num-tx-queues 2
}
dev 0000:3b:00.0
dev 0000:3b:00.1
#dev 0000:3b:00.2
#dev 0000:3b:00.3
#uio-driver vfio-pci (just modify here on the basis of startup.conf--my config)
# @@@@@steven, do you know why this option makes the difference？@@@@@
}
-----------------------------------------------------------------------------
I'm not familiar with testpmd, but I will take some time to find out how it
works.
(1)When I use “startup.conf.smp", and follow the operation sequence below after
Centos startup, it seems ok:
[root@localhost ~]# modprobe vfio-pci
[root@localhost ~]# lsmod | grep vfio
vfio_pci 41412 2
vfio_iommu_type1 22440 0
vfio 32657 8 vfio_iommu_type1,vfio_pci
irqbypass 13503 4 kvm,vfio_pci
[root@localhost ~]#/usr/bin/numactl --cpubind=0 --membind=0 /usr/bin/vpp -c
/etc/vpp/startup.conf.smp
...........
vpp# show pci
Address Sock VID:PID Link Speed Driver Product Name
Vital Product Data
...........
0000:3b:00.0 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86
0000:3b:00.1 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86
0000:3b:00.2 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86
0000:3b:00.3 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86
vpp# show interface
Name Idx State MTU (L3/IP4/IP6/MPLS)
Counter Count
TenGigabitEthernet3b/0/0 1 down 9000/0/0/0
TenGigabitEthernet3b/0/1 2 down 9000/0/0/0
TenGigabitEthernet3b/0/2 3 down 9000/0/0/0
TenGigabitEthernet3b/0/3 4 down 9000/0/0/0
local0 0 down 0/0/0/0
vpp# show log
2020/01/08 10:35:38:001 warn dpdk Unsupported PCI device
0x14e4:0x165f found at PCI address 0000:18:00.0
2020/01/08 10:35:38:017 warn dpdk Unsupported PCI device
0x14e4:0x165f found at PCI address 0000:18:00.1
2020/01/08 10:35:38:032 warn dpdk Unsupported PCI device
0x14e4:0x165f found at PCI address 0000:19:00.0
2020/01/08 10:35:38:076 warn dpdk Unsupported PCI device
0x14e4:0x165f found at PCI address 0000:19:00.1
2020/01/08 10:35:39:447 warn dpdk EAL init args: -c 2 -n 4
--in-memory --file-prefix vpp --master-lcore 1
2020/01/08 10:35:40:682 notice dpdk EAL: Detected 32 lcore(s)
2020/01/08 10:35:40:682 notice dpdk EAL: Detected 2 NUMA nodes
2020/01/08 10:35:40:682 notice dpdk EAL: Some devices want iova as va
but pa will be used because.. EAL: vfio-noiommu mode configured
2020/01/08 10:35:40:682 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 10:35:40:682 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 10:35:40:682 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 10:35:40:682 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 10:35:40:682 notice dpdk EAL: Probing VFIO support...
2020/01/08 10:35:40:682 notice dpdk EAL: VFIO support initialized
2020/01/08 10:35:40:682 notice dpdk EAL: WARNING! Base virtual
address hint (0xa80001000 != 0x7f4f40000000) not respected!
2020/01/08 10:35:40:682 notice dpdk EAL: This may cause issues
with mapping memory into secondary processes
@Yichen, when I used "dmesg | grep Virtualization", nothing was returned:
[root@localhost ~]# dmesg | grep Virtualization
[root@localhost ~]#
I don't do performance test or other related tests now, so I don't know if
there are any other problems.
(2)When I use “startup.conf", and follow the operation sequence below after
Centos startup, it seems bad:
[root@localhost ~]# modprobe vfio-pci
[root@localhost ~]# lsmod | grep vfio
vfio_pci 41412 2
vfio_iommu_type1 22440 0
vfio 32657 8 vfio_iommu_type1,vfio_pci
irqbypass 13503 4 kvm,vfio_pci
[root@localhost ~]#/usr/bin/numactl --cpubind=0 --membind=0 /usr/bin/vpp -c
/etc/vpp/startup.conf
...........
vpp# show pci
Address Sock VID:PID Link Speed Driver Product Name
Vital Product Data
...........
0000:3b:00.0 0 8086:1572 8.0 GT/s x8 XL710 40GbE
Controller RV: 0x 86
0000:3b:00.1 0 8086:1572 8.0 GT/s x8 XL710 40GbE
Controller RV: 0x 86
0000:3b:00.2 0 8086:1572 8.0 GT/s x8 i40e XL710 40GbE
Controller RV: 0x 86
0000:3b:00.3 0 8086:1572 8.0 GT/s x8 i40e XL710 40GbE
Controller RV: 0x 86
[root@localhost ~]# vppctl show int
Name Idx State MTU (L3/IP4/IP6/MPLS)
Counter Count
local0 0 down 0/0/0/0
vppctl show log
2020/01/08 11:01:40:400 warn dpdk EAL init args: -c 54000000 -n 4
--in-memory --file-prefix vpp -w 0000:3b:00.0 -w 0000:3b:00.1 --master-lcore 30
2020/01/08 11:01:40:570 notice dpdk DPDK drivers found no ports...
2020/01/08 11:01:40:575 notice dpdk EAL: Detected 32 lcore(s)
2020/01/08 11:01:40:575 notice dpdk EAL: Detected 2 NUMA nodes
2020/01/08 11:01:40:575 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 11:01:40:575 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 11:01:40:575 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 11:01:40:575 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 11:01:40:575 notice dpdk EAL: Probing VFIO support...
2020/01/08 11:01:40:575 notice dpdk EAL: VFIO support initialized
2020/01/08 11:01:40:575 notice dpdk EAL: WARNING! Base virtual
address hint (0xa80001000 != 0x7f0d40000000) not respected!
2020/01/08 11:01:40:575 notice dpdk EAL: This may cause issues
with mapping memory into secondary processes
[root@localhost ~]#dmesg
............
[ 498.625440] VFIO - User Level meta-driver version: 0.3
[ 522.934678] i40e 0000:3b:00.0: removed PHC from p1p1
[ 523.142229] vfio-pci: probe of 0000:3b:00.0 failed with error -22
[ 523.154248] i40e 0000:3b:00.1: removed PHC from p1p2
[ 523.359757] vfio-pci: probe of 0000:3b:00.1 failed with error -22
[root@localhost ~]# dmesg | grep Virtualization
[root@localhost ~]#
@Yichen, here is same as above...
(3)I modify the "startup.conf" repeatedly, and finally found the dpdk option
"uio-driver vfio-pci" was a key point. if I delete it, the relust seems ok.
[root@localhost ~]# modprobe vfio-pci
[root@localhost ~]# lsmod | grep vfio
vfio_pci 41412 2
vfio_iommu_type1 22440 0
vfio 32657 8 vfio_iommu_type1,vfio_pci
irqbypass 13503 4 kvm,vfio_pci
[root@localhost ~]#/usr/bin/numactl --cpubind=0 --membind=0 /usr/bin/vpp -c
/etc/vpp/startup.conf.ok
...........
vpp# show pci
Address Sock VID:PID Link Speed Driver Product Name
Vital Product Data
...........
0000:3b:00.0 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86
0000:3b:00.1 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86
0000:3b:00.2 0 8086:1572 8.0 GT/s x8 i40e XL710 40GbE
Controller RV: 0x 86
0000:3b:00.3 0 8086:1572 8.0 GT/s x8 i40e XL710 40GbE
Controller RV: 0x 86
[root@localhost ~]# vppctl show int
Name Idx State MTU (L3/IP4/IP6/MPLS)
Counter Count
TenGigabitEthernet3b/0/0 1 down 9000/0/0/0
TenGigabitEthernet3b/0/1 2 down 9000/0/0/0
local0 0 down 0/0/0/0
[root@localhost ~]# vppctl show log
2020/01/08 11:22:23:789 warn dpdk EAL init args: -c 54000000 -n 4
--in-memory --file-prefix vpp -w 0000:3b:00.0 -w 0000:3b:00.1 --master-lcore 30
2020/01/08 11:22:24:512 notice dpdk EAL: Detected 32 lcore(s)
2020/01/08 11:22:24:512 notice dpdk EAL: Detected 2 NUMA nodes
2020/01/08 11:22:24:512 notice dpdk EAL: Some devices want iova as va
but pa will be used because.. EAL: vfio-noiommu mode configured
2020/01/08 11:22:24:512 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 11:22:24:512 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 11:22:24:512 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 11:22:24:512 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 11:22:24:512 notice dpdk EAL: Probing VFIO support...
2020/01/08 11:22:24:512 notice dpdk EAL: VFIO support initialized
2020/01/08 11:22:24:512 notice dpdk EAL: WARNING! Base virtual
address hint (0xa80001000 != 0x7fe540000000) not respected!
2020/01/08 11:22:24:512 notice dpdk EAL: This may cause issues
with mapping memory into secondary processes
[root@localhost dpdk]# ./dpdk-devbind --status
Network devices using DPDK-compatible driver
============================================
0000:3b:00.0 'Ethernet Controller X710 for 10GbE SFP+ 1572' drv=vfio-pci
unused=i40e
0000:3b:00.1 'Ethernet Controller X710 for 10GbE SFP+ 1572' drv=vfio-pci
unused=i40e
Network devices using kernel driver
===================================
0000:18:00.0 'NetXtreme BCM5720 2-port Gigabit Ethernet PCIe 165f' if=em1
drv=tg3 unused=vfio-pci *Active*
0000:18:00.1 'NetXtreme BCM5720 2-port Gigabit Ethernet PCIe 165f' if=em2
drv=tg3 unused=vfio-pci
0000:19:00.0 'NetXtreme BCM5720 2-port Gigabit Ethernet PCIe 165f' if=em3
drv=tg3 unused=vfio-pci
0000:19:00.1 'NetXtreme BCM5720 2-port Gigabit Ethernet PCIe 165f' if=em4
drv=tg3 unused=vfio-pci *Active*
0000:3b:00.2 'Ethernet Controller X710 for 10GbE SFP+ 1572' if=p1p3 drv=i40e
unused=vfio-pci
0000:3b:00.3 'Ethernet Controller X710 for 10GbE SFP+ 1572' if=p1p4 drv=i40e
unused=vfio-pci
No 'Crypto' devices detected
============================
root@localhost dpdk]# dmesg
..............
[ 167.814084] VFIO - User Level meta-driver version: 0.3
[ 270.848376] i40e 0000:3b:00.0: removed PHC from p1p1
[ 271.061138] iommu: Adding device 0000:3b:00.0 to group 0
[ 271.061159] vfio-pci 0000:3b:00.0: Adding kernel taint for vfio-noiommu
group on device
[ 271.073068] i40e 0000:3b:00.1: removed PHC from p1p2
[ 271.277631] iommu: Adding device 0000:3b:00.1 to group 1
[ 271.277652] vfio-pci 0000:3b:00.1: Adding kernel taint for vfio-noiommu
group on device
[ 271.458185] vfio-pci 0000:3b:00.0: Masking broken INTx support
[ 271.458313] vfio_ecap_init: 0000:3b:00.0 hiding ecap 0x19@0x1d0
[ 271.458326] vfio-pci 0000:3b:00.0: vfio-noiommu device opened by user
(vpp:2421)
[ 271.622873] vfio-pci 0000:3b:00.0: irq 74 for MSI/MSI-X
[ 271.725553] vfio-pci 0000:3b:00.1: Masking broken INTx support
[ 271.725684] vfio-pci 0000:3b:00.1: vfio-noiommu device opened by user
(vpp:2421)
[ 271.889681] vfio-pci 0000:3b:00.1: irq 75 for MSI/MSI-X
[root@localhost dpdk]# dmesg | grep Virtualization
[root@localhost dpdk]#
@Yichen, here is same as above...
Regards，
gencli
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#15090): https://lists.fd.io/g/vpp-dev/message/15090
Mute This Topic: https://lists.fd.io/mt/69347948/21656
Mute #vpp: https://lists.fd.io/mk?hashtag=vpp&subid=1480452
Mute #vnet: https://lists.fd.io/mk?hashtag=vnet&subid=1480452
Group Owner: vpp-dev+ow...@lists.fd.io
Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
