Hello,
I'm seeing a consistent crash when trying to run an iperf test using qede
interface and DPDK. Running one iperf test on its own works fine but as soon as
I attempt to run two in parallel through the same interface the crash occurs.
OVS configuration is:
ovs-vsctl set Open_vSwitch . other_config:dpdk-socket-mem="4096,0"
ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-iommu-support=true
ovs-vsctl --timeout 10 set Open_vSwitch . other_config:max-idle=30000
ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x1e
ovs-vsctl add-port br0 dpdk-p0 -- set Interface dpdk-p0 type=dpdk \
options:dpdk-devargs=0000:45:00.0 \
options:n_rxq=1 other_config:pmd-rxq-affinity="0:1" \
options:flow-ctrl-autoneg=true \
ofport=1
ovs-vsctl -- set Interface br1 mtu_request=1500
ovs-vsctl add-port br1 dpdk-p1 -- set Interface dpdk-p1 type=dpdk \
options:dpdk-devargs=0000:45:00.1 \
options:n_rxq=3 other_config:pmd-rxq-affinity="0:2,1:3,2:4"
options:n_rxq_desc=2048 options:n_txq_desc=2048 \
options:flow-ctrl-autoneg=true \
mtu_request=1504 ofport=1
ip a add 192.168.190.77/25 dev br0
ip a add 192.168.190.200/25 dev br1
rm -f /dev/hugepages/*
ip link set br0 up
ip link set br1 up
ip route add default via 192.168.190.1 dev br0
ovs-vsctl set port dpdk-p1 qos=@qos -- \
--id=@qos create qos type=trtcm-policer other-config:cir=250000000
other-config:cbs=30080 other_config:eir=12500000 other-config:ebs=30080
queues:1=@queue1 queues:2=@queue2 queues:3=@queue3 -- \
--id=@queue1 create queue other-config:cir=220000000
other-config:eir=10000000 other-config:cbs=30080 other-config:ebs=30080 -- \
--id=@queue2 create queue other-config:cir=20000000
other-config:eir=220000000 other-config:cbs=30080 other-config:ebs=30080 -- \
--id=@queue3 create queue other-config:cir=10000000
other-config:eir=10000000 other-config:cbs=30080 other-config:ebs=30080
ovs-ofctl del-flows br1
ovs-ofctl add-flow br1
'table=0,priority=1,ip,ip_dst=0.0.0.201/0.0.0.255,actions=set_queue:0,resubmit(,1)'
ovs-ofctl add-flow br1
'table=0,priority=1,ip,ip_dst=0.0.0.202/0.0.0.255,actions=set_queue:1,resubmit(,1)'
ovs-ofctl add-flow br1
'table=0,priority=1,ip,ip_dst=0.0.0.203/0.0.0.255,actions=set_queue:2,resubmit(,1)'
ovs-ofctl add-flow br1 'table=0,priority=0,ip,actions=resubmit(,1)'
ovs-ofctl add-flow br1 'table=0,priority=0,arp,actions=all'
ovs-ofctl add-flow br1 'table=0,mpls,actions=pop_mpls:0x0800,local'
ovs-ofctl add-flow br1
'table=1,ip,actions=move:ip_src[0..7]->reg0[0..7],push_mpls:0x8847,move:reg0[0..7]->mpls_label[0..7],set_field:2->mpls_tc,set_field:255->mpls_ttl,1'
The crash produces the following back-trace:
Process 2800 (ovs-vswitchd) of user 0 dumped core.
Stack trace of thread
2800:
#0 0x00007f2f94f3470f
raise (libc.so.6)
#1 0x00007f2f94f1eb25
abort (libc.so.6)
#2 0x00007f2f94f77897
__libc_message (libc.so.6)
#3 0x00007f2f94f7dfdc
malloc_printerr (libc.so.6)
#4 0x00007f2f94f7fe44
_int_free (libc.so.6)
#5 0x000000000050881a
dp_packet_delete (ovs-vswitchd)
#6 0x000000000050883e
dp_packet_delete_batch (ovs-vswitchd)
#7 0x000000000050a114
dp_packet_delete_batch (ovs-vswitchd)
#8 0x000000000047d243
netdev_send (ovs-vswitchd)
#9 0x0000000000458910
dp_netdev_pmd_flush_output_on_port (ovs-vswitchd)
#10 0x0000000000458a1a
dp_netdev_pmd_flush_output_packets (ovs-vswitchd)
#11 0x000000000045d705
dp_netdev_process_rxq_port (ovs-vswitchd)
#12 0x000000000045deb5
dpif_netdev_run (ovs-vswitchd)
#13 0x0000000000429e10
type_run (ovs-vswitchd)
#14 0x0000000000418b10
ofproto_type_run (ovs-vswitchd)
#15 0x000000000040c101
bridge_run__ (ovs-vswitchd)
#16 0x0000000000410f3d
bridge_run (ovs-vswitchd)
#17 0x0000000000409802
main (ovs-vswitchd)
#18 0x00007f2f94f206a3
__libc_start_main (libc.so.6)
#19 0x000000000040a2de
_start (ovs-vswitchd)
Stack trace of thread
2959:
#0 0x00007f2f94fedf21
__poll (libc.so.6)
#1 0x00000000004dd2e9
time_poll (ovs-vswitchd)
#2 0x00000000004cdb35
poll_block (ovs-vswitchd)
#3 0x000000000052e9b3
clean_thread_main (ovs-vswitchd)
#4 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
3069:
#0 0x000000000045d981
pmd_thread_main (ovs-vswitchd)
#1 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#2 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#3 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
3071:
#0 0x000000000045d5d7
rte_rdtsc (ovs-vswitchd)
#1 0x000000000045da10
pmd_thread_main (ovs-vswitchd)
#2 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#3 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#4 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
3072:
#0 0x00007ffd370139c9
n/a (linux-vdso.so.1)
#1 0x00007ffd37013bcc
__vdso_clock_gettime (linux-vdso.so.1)
#2 0x00007f2f95006ea2
__clock_gettime (libc.so.6)
#3 0x00000000004dceba
xclock_gettime (ovs-vswitchd)
#4 0x00000000004dcefa
time_timespec__ (ovs-vswitchd)
#5 0x00000000004dd421
time_usec__ (ovs-vswitchd)
#6 0x000000000045d95f
pmd_thread_ctx_time_update (ovs-vswitchd)
#7 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#8 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#9 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
3070:
#0 0x00000000004dced3
time_timespec__ (ovs-vswitchd)
#1 0x00000000004dd421
time_usec__ (ovs-vswitchd)
#2 0x000000000045d95f
pmd_thread_ctx_time_update (ovs-vswitchd)
#3 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#4 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#5 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
2803:
#0 0x00007f2f94fc5238
__nanosleep (libc.so.6)
#1 0x00007f2f94fc513e
sleep (libc.so.6)
#2 0x00000000004e431e
xsleep (ovs-vswitchd)
#3 0x0000000000506ef7
dpdk_watchdog (ovs-vswitchd)
#4 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
2801:
#0 0x00007f2f94ff91b7
epoll_wait (libc.so.6)
#1 0x00007f2f97560a44
eal_intr_thread_main (librte_eal.so.20.0)
#2 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#3 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
2804:
#0 0x00007f2f94fedf21
__poll (libc.so.6)
#1 0x00000000004dd2e9
time_poll (ovs-vswitchd)
#2 0x00000000004cdb35
poll_block (ovs-vswitchd)
#3 0x00000000004bd334
ovsrcu_postpone_thread (ovs-vswitchd)
#4 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
2967:
#0 0x00007f2f94fedf21
__poll (libc.so.6)
#1 0x00000000004dd2e9
time_poll (ovs-vswitchd)
#2 0x00000000004cdb35
poll_block (ovs-vswitchd)
#3 0x000000000043589b
udpif_upcall_handler (ovs-vswitchd)
#4 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
2960:
#0 0x00007f2f94fedf21
__poll (libc.so.6)
#1 0x00000000004dd2e9
time_poll (ovs-vswitchd)
#2 0x00000000004cdb35
poll_block (ovs-vswitchd)
#3 0x0000000000469dce
ipf_clean_thread_main (ovs-vswitchd)
#4 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
2970:
#0 0x00007f2f94fedf21
__poll (libc.so.6)
#1 0x00000000004dd2e9
time_poll (ovs-vswitchd)
#2 0x00000000004cdb35
poll_block (ovs-vswitchd)
#3 0x000000000043589b
udpif_upcall_handler (ovs-vswitchd)
#4 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83
__clone (libc.so.6)
Stack trace of thread
2976:
#0 0x00007f2f94fedf21
__poll (libc.so.6)
#1 0x00000000004dd2e9
time_poll (ovs-vswitchd)
#2 0x00000000004cdb35
poll_block (ovs-vswitchd)
#3 0x000000000043589b
udpif_upcall_handler (ovs-vswitchd)
#4 0x00000000004beba0
ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de
start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83
__clone (libc.so.6)
Any idea as to what I might be doing wrong? Or is this a bug?
Thanks,
Bill Kozak
_______________________________________________
discuss mailing list
disc...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-discuss
