Sounds like a memory corruption. I am out of office for another week, so in the meantime if you might collect few postmortem dumps with reproductions, I will look at it when I return.
(I aim to try to reproduce by adding the necessary api calls to https://github.com/vpp-dev/apidump2py - because this way I can later enhance the test suite to cover that case) --a > On 6 Mar 2019, at 05:02, Raj <rajlistu...@gmail.com> wrote: > > Hello all, > > I am getting a core dump when adding MACIP ACL using API (using > honeycomb). My observation is that I can reproduce this core dump > reliably if I add about 300 MACIP ACL. I am on v18.10-27~ga0005702c > > I did some debugging and my observations is: > > In the function: > > void > vl_msg_api_handler_with_vm_node (api_main_t * am, > void *the_msg, vlib_main_t * vm, > vlib_node_runtime_t * node) > { > ... > ... > /* > * Special-case, so we can e.g. bounce messages off the vnet > * main thread without copying them... > */ > if (!(am->message_bounce[id])) > vl_msg_api_free (the_msg); > ... > } > > Control is reaching the special-case, and core dump is happening in > vl_msg_api_free function. > > Code flow is: > void_mem_api_handle_msg_i() > ->vl_msg_api_free (the_msg); > ->clib_mem_free (rv); > ->mspace_put (heap, p); > ->mspace_free (msp, object_header); > ->ok_magic(fm) > ->return (m->magic == mparams.magic); /* here it dumps > */ > > > > Following is my gdb session transcript: > > (gdb) bt > #0 0x00007ffff5fd9f98 in ok_magic (m=0x13131313cdbec9ad) at > /home/raj/vpp/src/vppinfra/dlmalloc.c:1618 > #1 0x00007ffff5fe271a in mspace_free (msp=0x130044010, > mem=0x1301c4ca0) at /home/raj/vpp/src/vppinfra/dlmalloc.c:4456 > #2 0x00007ffff5fe1b9d in mspace_put (msp=0x130044010, > p_arg=0x1301c4ca4) at /home/raj/vpp/src/vppinfra/dlmalloc.c:4291 > #3 0x00007ffff7b916a4 in clib_mem_free (p=0x1301c4ca4) at > /home/raj/vpp/src/vppinfra/mem.h:215 > #4 0x00007ffff7b922f6 in vl_msg_api_free (a=0x1301c4cb4) at > /home/raj/vpp/src/vlibmemory/memory_shared.c:291 > #5 0x00007ffff7bc325c in vl_msg_api_handler_with_vm_node > (am=0x7ffff7dd3d20 <api_main>, the_msg=0x1301c4cb4, vm=0x7ffff6952240 > <vlib > node=0x7fffb5264000) at /home/raj/vpp/src/vlibapi/api_shared.c:516 > #6 0x00007ffff7b8feb4 in void_mem_api_handle_msg_i (am=0x7ffff7dd3d20 > <api_main>, vm=0x7ffff6952240 <vlib_global_main>, node=0x7fffb > at /home/raj/vpp/src/vlibmemory/memory_api.c:692 > #7 0x00007ffff7b8ff23 in vl_mem_api_handle_msg_main > (vm=0x7ffff6952240 <vlib_global_main>, node=0x7fffb5264000) at > /home/raj/vpp/ > #8 0x00007ffff7baded4 in vl_api_clnt_process (vm=0x7ffff6952240 > <vlib_global_main>, node=0x7fffb5264000, f=0x0) at /home/raj/vpp/ > #9 0x00007ffff66ce32a in vlib_process_bootstrap (_a=140736236354592) > at /home/raj/vpp/src/vlib/main.c:1232 > #10 0x00007ffff5f5784c in clib_calljmp () from > /home/raj/vpp/build-root/install-vpp_debug-native/vpp/lib/libvppinfra.so.18.10 > #11 0x00007fffb55ffbf0 in ?? () > #12 0x00007ffff66ce455 in vlib_process_startup (vm=0xd52f22e80133b900, > p=0xffffffffffffffff, f=0x7fffb5264000) at /home/raj/vpp/sr > #13 0x0000000000000086 in ?? () > #14 0x00007ffff6952350 in vlib_global_main () from > /home/raj/vpp/build-root/install-vpp_debug-native/vpp/lib/libvlib.so.18.10 > #15 0x0003612097f3543e in ?? () > #16 0x00007fffb5264000 in ?? () > n ?? () > #18 0x00007fffb5ccf56c in ?? () > #19 0x0000000000000011 in ?? () > #20 0x00007fffb5ccf668 in ?? () > #21 0x00007fffb5264000 in ?? () > #22 0x00007fffb79d8294 in ?? () > #23 0x0000000000000000 in ?? () > > (gdb) f 2 > #2 0x00007ffff5fe1b9d in mspace_put (msp=0x130044010, > p_arg=0x1301c4ca4) at /home/raj/vpp/src/vppinfra/dlmalloc.c:4291 > 4291 mspace_free (msp, object_header); > > (gdb) p msp > $1 = (mspace) 0x130044010 > > (gdb) p *msp > Attempt to dereference a generic pointer. > > (gdb) p *(mstate)msp > $2 = {smallmap = 4096, treemap = 32768, dvsize = 0, topsize = > 15069712, least_addr = 0x130044000 "", dv = 0x0, top = 0x1301e4da0, > tri > release_checks = 4086, magic = 3735935678, smallbins = {0x0, 0x0, > 0x130044058, 0x130044058, 0x130044068, 0x130044068, 0x130044078, > 0x130044088, 0x130044098, 0x130044098, 0x1300440a8, 0x1300440a8, > 0x1300440b8, 0x1300440b8, 0x1300440c8, 0x1300440c8, 0x13005c5b0, > 0x1300440e8, 0x1300440f8, 0x1300440f8, 0x130044108, 0x130044108, > 0x1300652c0, 0x1300652c0, 0x130044128, 0x130044128, 0x130044138, > 0x130044148, 0x130044158, 0x130044158, 0x130044168, 0x130044168, > 0x130044178, 0x130044178, 0x130044188, 0x130044188, 0x1301c4ce0, > 0x1300441a8, 0x1300441b8, 0x1300441b8, 0x1300441c8, 0x1300441c8, > 0x1300441d8, 0x1300441d8, 0x1300441e8, 0x1300441e8, 0x1300441f8, > 0x130044208, 0x130044218, 0x130044218, 0x130044228, 0x130044228, > 0x130044238, 0x130044238, 0x130044248, 0x130044248}, treebins = > 0x1301c5cc0, 0x0 <repeats 16 times>}, footprint = 16777216, > max_footprint = 16777216, footprint_limit = 0, mflags = 15, mutex = 0 > size = 16777216, next = 0x0, sflags = 8}, extp = 0x0, exts = 0} > > (gdb) f 5 > #5 0x00007ffff7bc325c in vl_msg_api_handler_with_vm_node > (am=0x7ffff7dd3d20 <api_main>, the_msg=0x1301c4cb4, vm=0x7ffff6952240 > <vlib > node=0x7fffb5264000) at /home/raj/vpp/src/vlibapi/api_shared.c:516 > 516 vl_msg_api_free (the_msg); > > (gdb) p the_msg > $5 = (void *) 0x1301c4cb4 > > (gdb) p *((u16 *) the_msg) > $6 = 4883 > > (gdb) f 4 > #4 0x00007ffff7b922f6 in vl_msg_api_free (a=0x1301c4cb4) at > /home/raj/vpp/src/vlibmemory/memory_shared.c:291 > 291 clib_mem_free (rv); > > (gdb) p *rv > $12 = {q = 0x1313131313131313, data_len = 320017171, gc_mark_timestamp > = 320017171, data = 0x1301c4cb4 '\023' <repeats 200 times>...} > (gdb) f 0 > #0 0x00007ffff5fd9f98 in ok_magic (m=0x13131313cdbec9ad) at > /home/raj/vpp/src/vppinfra/dlmalloc.c:1618 > 1618 return (m->magic == mparams.magic); > > (gdb) p m->magic > Cannot access memory at address 0x13131313cdbec9ed > > (gdb) f 1 > #1 0x00007ffff5fe271a in mspace_free (msp=0x130044010, > mem=0x1301c4ca0) at /home/raj/vpp/src/vppinfra/dlmalloc.c:4456 > 4456 if (!ok_magic(fm)) { > > (gdb) p *(mstate)msp > $24 = {smallmap = 4096, treemap = 32768, dvsize = 0, topsize = > 15069712, least_addr = 0x130044000 "", dv = 0x0, top = 0x1301e4da0, tr > release_checks = 4086, magic = 3735935678, smallbins = {0x0, 0x0, > 0x130044058, 0x130044058, 0x130044068, 0x130044068, 0x130044078, > 0x130044088, 0x130044098, 0x130044098, 0x1300440a8, 0x1300440a8, > 0x1300440b8, 0x1300440b8, 0x1300440c8, 0x1300440c8, 0x13005c5b0, > 0x1300440e8, 0x1300440f8, 0x1300440f8, 0x130044108, 0x130044108, > 0x1300652c0, 0x1300652c0, 0x130044128, 0x130044128, 0x130044138, > 0x130044148, 0x130044158, 0x130044158, 0x130044168, 0x130044168, > 0x130044178, 0x130044178, 0x130044188, 0x130044188, 0x1301c4ce0, > 0x1300441a8, 0x1300441b8, 0x1300441b8, 0x1300441c8, 0x1300441c8, > 0x1300441d8, 0x1300441d8, 0x1300441e8, 0x1300441e8, 0x1300441f8, > 0x130044208, 0x130044218, 0x130044218, 0x130044228, 0x130044228, > 0x130044238, 0x130044238, 0x130044248, 0x130044248}, treebins = > 0x1301c5cc0, 0x0 <repeats 16 times>}, footprint = 16777216, > max_footprint = 16777216, footprint_limit = 0, mflags = 15, mutex = 0 > size = 16777216, next = 0x0, sflags = 8}, extp = 0x0, exts = 0} > > (gdb) p (*(mstate)msp).magic > $25 = 3735935678 > > (gdb) p (mstate)fm > $26 = (mstate) 0x13131313cdbec9ad > > (gdb) p fm.magic > Cannot access memory at address 0x13131313cdbec9ed > > (gdb) f 0 > #0 0x00007ffff5fd9f98 in ok_magic (m=0x13131313cdbec9ad) at > /home/raj/vpp/src/vppinfra/dlmalloc.c:1618 > 1618 return (m->magic == mparams.magic); > > (gdb) p m->magic > Cannot access memory at address 0x13131313cdbec9ed > > (gdb) > (gdb) p mparams.magic > $34 = 3735935678 > > Thanks and Regards, > > Raj > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > > View/Reply Online (#12438): https://lists.fd.io/g/vpp-dev/message/12438 > Mute This Topic: https://lists.fd.io/mt/30283387/675608 > Group Owner: vpp-dev+ow...@lists.fd.io > Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [ayour...@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=-
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#12439): https://lists.fd.io/g/vpp-dev/message/12439 Mute This Topic: https://lists.fd.io/mt/30283387/21656 Group Owner: vpp-dev+ow...@lists.fd.io Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
