Hi,
Here you are.
This bug is triggered by configuring bond_mode=banlace-tcp and lacp=active
simultaneously.
The version of OVS is 2.3.1.
crash 7.0.2-6.el7
Copyright (C) 2002-2013 Red Hat, Inc.
Copyright (C) 2004, 2005, 2006, 2010 IBM Corporation
Copyright (C) 1999-2006 Hewlett-Packard Co
Copyright (C) 2005, 2006, 2011, 2012 Fujitsu Limited
Copyright (C) 2006, 2007 VA Linux Systems Japan K.K.
Copyright (C) 2005, 2011 NEC Corporation
Copyright (C) 1999, 2002, 2007 Silicon Graphics, Inc.
Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc.
This program is free software, covered by the GNU General Public License,
and you are welcome to change it and/or distribute copies of it under
certain conditions. Enter "help copying" to see the conditions.
This program has absolutely no warranty. Enter "help warranty" for details.
GNU gdb (GDB) 7.6
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-unknown-linux-gnu"...
WARNING: kernel version inconsistency between vmlinux and dumpfile
KERNEL: vmlinux
DUMPFILE: vmcore [PARTIAL DUMP]
CPUS: 4
DATE: Tue May 12 10:31:28 2015
UPTIME: 00:02:02
LOAD AVERAGE: 0.43, 0.39, 0.16
TASKS: 178
NODENAME: centos125
RELEASE: 3.10.0-123.el7.x86_64
VERSION: #1 SMP Mon May 11 21:19:35 CST 2015
MACHINE: x86_64 (3192 Mhz)
MEMORY: 15.9 GB
PANIC: "Kernel panic - not syncing: stack-protector: Kernel stack is
corrupted in: ffffffffa04862ca"
PID: 894
COMMAND: "handler14"
TASK: ffff8804059d2220 [THREAD_INFO: ffff8804064c4000]
CPU: 3
STATE: TASK_RUNNING (PANIC)
crash> bt
PID: 894 TASK: ffff8804059d2220 CPU: 3 COMMAND: "handler14"
#0 [ffff8804064c5658] machine_kexec at ffffffff8104105b
#1 [ffff8804064c56b8] crash_kexec at ffffffff810ceec2
#2 [ffff8804064c5788] panic at ffffffff815dab1e
#3 [ffff8804064c5808] __stack_chk_fail at ffffffff8105dc3b
#4 [ffff8804064c5818] execute_recirc at ffffffffa04862ca [openvswitch]
#5 [ffff8804064c58c8] do_execute_actions at ffffffffa04871ba [openvswitch]
#6 [ffff8804064c5968] ovs_execute_actions at ffffffffa04873f7 [openvswitch]
#7 [ffff8804064c59a0] ovs_packet_cmd_execute at ffffffffa04897b6 [openvswitch]
#8 [ffff8804064c59f8] genl_family_rcv_msg at ffffffff814ff268
#9 [ffff8804064c5ac0] genl_rcv_msg at ffffffff814ff471
#10 [ffff8804064c5ae8] netlink_rcv_skb at ffffffff814fd529
#11 [ffff8804064c5b10] genl_rcv at ffffffff814fda58
#12 [ffff8804064c5b28] netlink_unicast at ffffffff814fcb4d
#13 [ffff8804064c5b70] netlink_sendmsg at ffffffff814fcf37
#14 [ffff8804064c5c08] sock_sendmsg at ffffffff814b6f30
#15 [ffff8804064c5d70] ___sys_sendmsg at ffffffff814b7369
#16 [ffff8804064c5f00] __sys_sendmsg at ffffffff814b8251
#17 [ffff8804064c5f70] sys_sendmsg at ffffffff814b82a2
#18 [ffff8804064c5f80] system_call_fastpath at ffffffff815f1619
RIP: 00007fca637c97bd RSP: 00007fca5bfe5748 RFLAGS: 00000206
RAX: 000000000000002e RBX: ffffffff815f1619 RCX: 00007fca4c0480e0
RDX: 0000000000000000 RSI: 00007fca5bfc9d30 RDI: 0000000000000012
RBP: 0000000000000002 R8: 0000000000000000 R9: 000000000000037e
R10: 0000000000ab32b0 R11: 0000000000000293 R12: ffffffff814b82a2
R13: ffff8804064c5f78 R14: 000000000000027d R15: 0000000000aad9b0
ORIG_RAX: 000000000000002e CS: 0033 SS: 002b
crash> dis execute_recirc
0xffffffffa0486240 <execute_recirc>: nopl 0x0(%rax,%rax,1)
0xffffffffa0486245 <execute_recirc+5>: push %rbp
0xffffffffa0486246 <execute_recirc+6>: mov %rsp,%rbp
0xffffffffa0486249 <execute_recirc+9>: push %r12
0xffffffffa048624b <execute_recirc+11>: lea -0xa0(%rbp),%rcx
0xffffffffa0486252 <execute_recirc+18>: push %rbx
0xffffffffa0486253 <execute_recirc+19>: mov %rdi,%rbx
0xffffffffa0486256 <execute_recirc+22>: mov %rbx,%rdx
0xffffffffa0486259 <execute_recirc+25>: sub $0x90,%rsp
0xffffffffa0486260 <execute_recirc+32>: mov 0x4(%rsi),%edi
0xffffffffa0486263 <execute_recirc+35>: mov 0x30(%rbx),%rsi
0xffffffffa0486267 <execute_recirc+39>: mov %gs:0x28,%rax
0xffffffffa0486270 <execute_recirc+48>: mov %rax,-0x18(%rbp)
0xffffffffa0486274 <execute_recirc+52>: xor %eax,%eax
0xffffffffa0486276 <execute_recirc+54>: callq 0xffffffffa048bc00
<ovs_flow_key_extract_recirc>
0xffffffffa048627b <execute_recirc+59>: test %eax,%eax
0xffffffffa048627d <execute_recirc+61>: mov %eax,%r12d
0xffffffffa0486280 <execute_recirc+64>: jne 0xffffffffa04862b8
0xffffffffa0486282 <execute_recirc+66>: lea -0xa0(%rbp),%rsi
0xffffffffa0486289 <execute_recirc+73>: mov $0x1,%edx
0xffffffffa048628e <execute_recirc+78>: mov %rbx,%rdi
0xffffffffa0486291 <execute_recirc+81>: callq 0xffffffffa048a5b0
<ovs_dp_process_packet_with_key>
0xffffffffa0486296 <execute_recirc+86>: xor %eax,%eax
0xffffffffa0486298 <execute_recirc+88>: mov -0x18(%rbp),%rdx
0xffffffffa048629c <execute_recirc+92>: xor %gs:0x28,%rdx
0xffffffffa04862a5 <execute_recirc+101>: jne 0xffffffffa04862c5
0xffffffffa04862a7 <execute_recirc+103>: add $0x90,%rsp
0xffffffffa04862ae <execute_recirc+110>: pop %rbx
0xffffffffa04862af <execute_recirc+111>: pop %r12
0xffffffffa04862b1 <execute_recirc+113>: pop %rbp
0xffffffffa04862b2 <execute_recirc+114>: retq
0xffffffffa04862b3 <execute_recirc+115>: nopl 0x0(%rax,%rax,1)
0xffffffffa04862b8 <execute_recirc+120>: mov %rbx,%rdi
0xffffffffa04862bb <execute_recirc+123>: callq 0xffffffff814c06f0
<kfree_skb>
0xffffffffa04862c0 <execute_recirc+128>: mov %r12d,%eax
0xffffffffa04862c3 <execute_recirc+131>: jmp 0xffffffffa0486298
0xffffffffa04862c5 <execute_recirc+133>: callq 0xffffffff8105dc20
<__stack_chk_fail>
0xffffffffa04862ca <execute_recirc+138>: nopw 0x0(%rax,%rax,1)
ZHANG Zhiming
Yunshan Networks
From: Andy Zhou
Date: 2015-05-23 04:41
To: Alex Wang
CC: zhangzhiming; discuss
Subject: Re: [ovs-discuss] one patch was omitted to be pushed to
branch-2.3---datapath: Fix recirc bug where skb is double freed
Hi, Jeremy,
Sorry for the delay. I don't think this patch is required for branch
2.3. As you may have noticed, this part of code
is different on branch 2.3. And it seems to work on my test.
Do you have the core dump from kernel crash? If yes, would you please
post the back trace?
Thanks,
Andy
On Sun, May 17, 2015 at 9:41 AM, Alex Wang <al...@nicira.com> wrote:
> Fwd to Andy,~
>
> On Sun, May 17, 2015 at 4:29 AM, zhangzhiming <zhangzhim...@yunshan.net.cn>
> wrote:
>>
>> Hi,
>>
>> I found one patch was omitted to be pushed to branch-2.3, which leads to
>> double freed skb.
>> Could someone to confirm the patch and submit it to branch-2.3?
>> Thanks!
>>
>> Here is the patch information:
>>
>>
>> commit 867e37ba00091b3e319c4c47c1598f1ae84dd32e
>> Author: Andy Zhou <az...@nicira.com>
>> Date: Mon Aug 25 15:18:19 2014 -0700
>>
>> datapath: Fix recirc bug where skb is double freed.
>>
>> If recirc action is the last action of a action list, the SKB triggers
>> the recirc will be freed twice. This patch fixes this bug.
>>
>> Reported-by: Justin Pettit <jpet...@nicira.com>
>> Signed-off-by: Andy Zhou <az...@nicira.com>
>>
>> diff --git a/datapath/actions.c b/datapath/actions.c
>> index ad22467..7f25553 100644
>> --- a/datapath/actions.c
>> +++ b/datapath/actions.c
>> @@ -809,7 +809,16 @@ static int execute_recirc(struct datapath *dp, struct
>> sk_buff *skb,
>> const struct nlattr *a, int rem)
>> {
>> struct sw_flow_key recirc_key;
>> - int err;
>> +
>> + if (!is_skb_flow_key_valid(skb)) {
>> + int err;
>> +
>> + err = ovs_flow_key_update(skb, OVS_CB(skb)->pkt_key);
>> + if (err)
>> + return err;
>> +
>> + }
>> + BUG_ON(!is_skb_flow_key_valid(skb));
>>
>> if (!last_action(a, rem)) {
>> /* Recirc action is the not the last action
>> @@ -820,19 +829,9 @@ static int execute_recirc(struct datapath *dp, struct
>> sk_buff *skb,
>> * continue on with the rest of the action list. */
>> if (!skb)
>> return 0;
>> - }
>>
>> - if (!is_skb_flow_key_valid(skb)) {
>> - err = ovs_flow_key_update(skb, OVS_CB(skb)->pkt_key);
>> - if (err) {
>> - kfree_skb(skb);
>> - return err;
>> - }
>> - }
>> - BUG_ON(!is_skb_flow_key_valid(skb));
>> -
>> - if (!last_action(a, rem))
>> flow_key_clone(skb, &recirc_key);
>> + }
>>
>> flow_key_set_recirc_id(skb, nla_get_u32(a));
>> ovs_dp_process_packet(skb, true);
>> @@ -897,6 +896,12 @@ static int do_execute_actions(struct datapath *dp,
>> struct sk_buff *skb,
>>
>> case OVS_ACTION_ATTR_RECIRC:
>> err = execute_recirc(dp, skb, a, rem);
>> + if (last_action(a, rem)) {
>> + /* If this is the last action, the skb has
>> + * been consumed or freed.
>> + * Return immediately. */
>> + return err;
>> + }
>> break;
>>
>> case OVS_ACTION_ATTR_SET:
>>
>> ________________________________
>> Jeremy Zhang
>>
>> _______________________________________________
>> discuss mailing list
>> discuss@openvswitch.org
>> http://openvswitch.org/mailman/listinfo/discuss
>>
>_______________________________________________
discuss mailing list
discuss@openvswitch.org
http://openvswitch.org/mailman/listinfo/discuss
