(gdb) frame 1
#1 dlg_unref (dlg=dlg@entry=0x7f585c494b40, cnt=cnt@entry=1) at
dlg_hash.c:921
921 dlg_lock( d_table, d_entry);
(gdb) p *dlg
$1 = {ref = 793790803, next = 0xa0d4b4f20303032, prev =
0x504953203a616956, h_id = 808333871, h_entry = 1346655535, state =
774976288, lifetime = 775107122, init_ts = 775435825,
start_ts = 976303410, end_ts = 808857653, dflags = 1667592763, iflags
= 1702259045, sflags = 825441636, toroute = 858927662, toroute_name = {
s = 0x6172623b3135322e <Address 0x6172623b3135322e out of bounds>,
len = 1030251374}, from_rr_nb = 894132788, tl = {next =
0x726f70723b646262, prev = 0xa0d303630353d74,
timeout = 1836020294}, callid = {s = 0x20226e776f6e6b6e <Address
0x20226e776f6e6b6e out of bounds>, len = 1885958972}, from_uri = {
s = 0x7340444c4f74656e <Address 0x7340444c4f74656e out of bounds>,
len = 1999532137}, to_uri = {s = 0x743b3e74656e2e70 <Address
0x743b3e74656e2e70 out of bounds>,
len = 1631414113}, req_uri = {s = 0x540a0d3536343766 <Address
0x540a0d3536343766 out of bounds>, len = 1008745071}, tag = {{
s = 0x363233313431332b <Address 0x363233313431332b out of bounds>,
len = 892614711}, {s = 0x2e3836312e333232 <Address 0x2e3836312e333232
out of bounds>, len = 1043608370}},
cseq = {{s = 0x663330643473613d <Address 0x663330643473613d out of
bounds>, len = 224671543}, {s = 0x3534203a44492d6c <Address
0x3534203a44492d6c out of bounds>,
len = 909665638}}, route_set = {{s = 0x3433333435356635 <Address
0x3433333435356635 out of bounds>, len = 825582898}, {
s = 0x7340353762316435 <Address 0x7340353762316435 out of bounds>,
len = 1999532137}}, contact = {{s = 0x430a0d74656e2e70 <Address
0x430a0d74656e2e70 out of bounds>,
len = 980510035}, {s = 0x65530a0d45594220 <Address
0x65530a0d45594220 out of bounds>, len = 1919252082}}, bind_addr =
{0x70696f766c772e70, 0x6c410a0d74656e2e}, cbs = {
first = 0x564e49203a776f6c, types = 742741065}, profile_links =
0x4c45434e4143202c, vars = 0x4e4f4954504f202c}
On 07/15/2016 01:00 PM, Daniel-Constantin Mierla wrote:
> From the second crash, can you get:
>
> frame 1
>
> p *dlg
>
> So far it looks like either to a double free or some buffer overflow...
>
> Cheers,
> Daniel
>
>
> On 15/07/16 10:51, Dirk Teurlings - Signet B.V. wrote:
>> Just got another segfault.
>>
>> Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
>> Core was generated by `/usr/sbin/kamailio -f /etc/kamailio/kamailio.cfg
>> -P /var/run/kamailio/kamailio.'.
>> Program terminated with signal 11, Segmentation fault.
>> #0 atomic_get (v=0x7f6264d11378) at ../../mem/../atomic/atomic_common.h:74
>> 74 return atomic_get_int(&(v->val));
>> (gdb) bt
>> #0 atomic_get (v=0x7f6264d11378) at ../../mem/../atomic/atomic_common.h:74
>> #1 dlg_unref (dlg=dlg@entry=0x7f585c494b40, cnt=cnt@entry=1) at
>> dlg_hash.c:921
>> #2 0x00007f5855912802 in dlg_run_event_route
>> (dlg=dlg@entry=0x7f585c494b40, msg=msg@entry=0x7f587d4be8e8,
>> ostate=<optimized out>, nstate=<optimized out>) at dlg_handlers.c:1630
>> #3 0x00007f585591416a in dlg_onroute (req=0x7f587d4be8e8,
>> route_params=<optimized out>, param=<optimized out>) at dlg_handlers.c:1307
>> #4 0x00007f585965b0e2 in run_rr_callbacks
>> (req=req@entry=0x7f587d4be8e8, rr_param=rr_param@entry=0x7f58598677a0)
>> at rr_cb.c:96
>> #5 0x00007f58596452c5 in after_loose (_m=0x7f587d4be8e8, preloaded=0)
>> at loose.c:919
>> #6 0x000000000042b618 in do_action (h=h@entry=0x7ffd6e277fd0,
>> a=a@entry=0x7f587d264338, msg=msg@entry=0x7f587d4be8e8) at action.c:1060
>> #7 0x000000000042a10a in run_actions (h=h@entry=0x7ffd6e277fd0,
>> a=0x7f587d264338, msg=0x7f587d4be8e8) at action.c:1549
>> #8 0x0000000000437544 in run_actions_safe (h=h@entry=0x7ffd6e279500,
>> a=<optimized out>, msg=<optimized out>) at action.c:1614
>> #9 0x000000000053b2e8 in rval_get_int (h=0x7ffd6e279500, msg=<optimized
>> out>, i=0x7ffd6e278430, rv=rv@entry=0x7f587d264d58,
>> cache=cache@entry=0x0) at rvalue.c:912
>> #10 0x000000000054261c in rval_expr_eval_int (h=h@entry=0x7ffd6e279500,
>> msg=msg@entry=0x7f587d4be8e8, res=res@entry=0x7ffd6e278430,
>> rve=rve@entry=0x7f587d264d50) at rvalue.c:1910
>> #11 0x000000000042bc91 in do_action (h=h@entry=0x7ffd6e279500,
>> a=a@entry=0x7f587d268f88, msg=msg@entry=0x7f587d4be8e8) at action.c:1030
>> #12 0x000000000042a10a in run_actions (h=h@entry=0x7ffd6e279500,
>> a=0x7f587d268f88, msg=msg@entry=0x7f587d4be8e8) at action.c:1549
>> #13 0x000000000042bcf2 in do_action (h=h@entry=0x7ffd6e279500,
>> a=a@entry=0x7f587d2691e8, msg=msg@entry=0x7f587d4be8e8) at action.c:1049
>> #14 0x000000000042a10a in run_actions (h=h@entry=0x7ffd6e279500,
>> a=0x7f587d263f48, msg=msg@entry=0x7f587d4be8e8) at action.c:1549
>> #15 0x000000000042bde0 in do_action (h=h@entry=0x7ffd6e279500,
>> a=a@entry=0x7f587d073d70, msg=msg@entry=0x7f587d4be8e8) at action.c:678
>> #16 0x000000000042a10a in run_actions (h=h@entry=0x7ffd6e279500,
>> a=a@entry=0x7f587d071698, msg=msg@entry=0x7f587d4be8e8) at action.c:1549
>> #17 0x00000000004375d0 in run_top_route (a=0x7f587d071698,
>> msg=msg@entry=0x7f587d4be8e8, c=c@entry=0x0) at action.c:1635
>> #18 0x0000000000504386 in receive_msg (buf=<optimized out>,
>> len=<optimized out>, rcv_info=<optimized out>) at receive.c:240
>> #19 0x00000000005f5bd4 in udp_rcv_loop () at udp_server.c:495
>> #20 0x00000000004b2625 in main_loop () at main.c:1600
>> #21 0x0000000000427e2b in main (argc=<optimized out>, argv=<optimized
>> out>) at main.c:2616
>>
>>
>> Relevant logmessages before crash:
>> Jul 15 10:37:55 server /usr/sbin/kamailio[12426]: NOTICE: dialog
>> [dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
>> (0x7f585c4a6820 ref 4)
>> Jul 15 10:37:55 server /usr/sbin/kamailio[12397]: WARNING: dialog
>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>> route param '70f.b9d1' [3847:7579]
>> Jul 15 10:37:55 server /usr/sbin/kamailio[12395]: WARNING: dialog
>> [dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
>> 0x7f585c4a6820 [3847:7579] with clid
>> '4c41f08d317ecb9342b93f22738003f3@server' and tags 'as5f3a16b4' 'as71cb6036'
>> Jul 15 10:40:13 server /usr/sbin/kamailio[12378]: WARNING: dialog
>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>> route param 'eb6.1e21' [1726:4833]
>> Jul 15 10:40:13 server /usr/sbin/kamailio[12376]: WARNING: dialog
>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>> route param 'eb6.1e21' [1726:4833]
>> Jul 15 10:40:14 server /usr/sbin/kamailio[12377]: WARNING: dialog
>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>> route param 'eb6.1e21' [1726:4833]
>> Jul 15 10:40:16 server /usr/sbin/kamailio[12377]: WARNING: dialog
>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>> route param 'eb6.1e21' [1726:4833]
>> Jul 15 10:40:16 server /usr/sbin/kamailio[12396]: WARNING: dialog
>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>> route param 'eb6.1e21' [1726:4833]
>> Jul 15 10:41:34 server /usr/sbin/kamailio[12396]: ERROR: sl
>> [sl_funcs.c:363]: sl_reply_error(): ERROR: sl_reply_error used: I'm
>> terribly sorry, server error occurred (1/SL)
>> Jul 15 10:41:34 server /usr/sbin/kamailio[12396]: ERROR: tm
>> [t_reply.c:533]: _reply_light(): ERROR: _reply_light: can't generate 487
>> reply when a final 487 was sent out
>> Jul 15 10:41:34 server /usr/sbin/kamailio[12396]: ERROR: tm
>> [t_lookup.c:1471]: t_unref(): ERROR: t_unref: generation of a delayed
>> stateful reply failed
>> Jul 15 10:42:25 server /usr/sbin/kamailio[12426]: NOTICE: dialog
>> [dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
>> (0x7f585c49d5b0 ref 4)
>> Jul 15 10:42:25 server /usr/sbin/kamailio[12426]: NOTICE: dialog
>> [dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
>> (0x7f585c604f18 ref 4)
>> Jul 15 10:42:25 server /usr/sbin/kamailio[12426]: NOTICE: dialog
>> [dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
>> (0x7f585c494b40 ref 4)
>> Jul 15 10:42:25 server /usr/sbin/kamailio[12383]: WARNING: dialog
>> [dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
>> 0x7f585c604f18 [2396:9046] with clid
>> '1b3ff5f0246fb7e82ed949544bcccbba@192.168.10.233:5060' and tags
>> 'as4d83d6f8' '5788A162-2557E04D-3E86ED15'
>> Jul 15 10:42:25 server /usr/sbin/kamailio[12395]: WARNING: dialog
>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>> route param '6b3.c6b' [950:2924]
>> Jul 15 10:42:25 server kernel: [209851.262461] kamailio[12376]: segfault
>> at 7f6264d11378 ip 00007f585592a908 sp 00007ffd6e277330 error 4 in
>> dialog.so[7f58558e0000+88000]
>> Jul 15 10:42:25 server /usr/sbin/kamailio[12394]: WARNING: dialog
>> [dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
>> 0x7f585c49d5b0 [950:2924] with clid
>> '45fe86ce065f5543342e51ad355d1b75@server' and tags 'as152f7465' 'as4d03f77d'
>> Jul 15 10:42:26 server /usr/sbin/kamailio[12431]: CRITICAL: <core>
>> [pass_fd.c:275]: receive_fd(): EOF on 32
>> Jul 15 10:42:26 server /usr/sbin/kamailio[12370]: ALERT: <core>
>> [main.c:739]: handle_sigs(): child process 12376 exited by a signal 11
>> Jul 15 10:42:26 server /usr/sbin/kamailio[12370]: ALERT: <core>
>> [main.c:742]: handle_sigs(): core was generated
>> Jul 15 10:42:26 server /usr/sbin/kamailio[12370]: INFO: <core>
>> [main.c:754]: handle_sigs(): terminating due to SIGCHLD
>>
>>
>> Cheers,
>> Dirk
>>
>>
>> On 07/15/2016 10:06 AM, Dirk Teurlings - Signet B.V. wrote:
>>> Hi,
>>>
>>> Running Kamailio on Debian from the Kamailio repository with 4.4.2
>>> stable (unpatched). Getting some random segfaults with it now, here's
>>> the relevant backtrace from the generated core.
>>>
>>> Core was generated by `/usr/sbin/kamailio -f /etc/kamailio/kamailio.cfg
>>> -P /var/run/kamailio/kamailio.'.
>>> Program terminated with signal 11, Segmentation fault.
>>> #0 run_dlg_callbacks (type=type@entry=64, dlg=dlg@entry=0x7fceb400e2f0,
>>> req=req@entry=0x7fced4f093c8, rpl=rpl@entry=0x0, dir=<optimized out>,
>>> dlg_data=dlg_data@entry=0x0) at dlg_cb.c:253
>>> 253 if ( (cb->types)&type ) {
>>> (gdb) bt
>>> #0 run_dlg_callbacks (type=type@entry=64, dlg=dlg@entry=0x7fceb400e2f0,
>>> req=req@entry=0x7fced4f093c8, rpl=rpl@entry=0x0, dir=<optimized out>,
>>> dlg_data=dlg_data@entry=0x0) at dlg_cb.c:253
>>> #1 0x00007fcead3648f9 in dlg_terminated (dir=<optimized out>,
>>> dlg=0x7fceb400e2f0, req=0x7fced4f093c8) at dlg_handlers.c:368
>>> #2 dlg_onroute (req=0x7fced4f093c8, route_params=<optimized out>,
>>> param=<optimized out>) at dlg_handlers.c:1354
>>> #3 0x00007fceb10ab0e2 in run_rr_callbacks
>>> (req=req@entry=0x7fced4f093c8, rr_param=rr_param@entry=0x7fceb12b77a0)
>>> at rr_cb.c:96
>>> #4 0x00007fceb10952c5 in after_loose (_m=0x7fced4f093c8, preloaded=0)
>>> at loose.c:919
>>> #5 0x000000000042b618 in do_action (h=h@entry=0x7ffeb0b3ed80,
>>> a=a@entry=0x7fced4cb4338, msg=msg@entry=0x7fced4f093c8) at action.c:1060
>>> #6 0x000000000042a10a in run_actions (h=h@entry=0x7ffeb0b3ed80,
>>> a=0x7fced4cb4338, msg=0x7fced4f093c8) at action.c:1549
>>> #7 0x0000000000437544 in run_actions_safe (h=h@entry=0x7ffeb0b402b0,
>>> a=<optimized out>, msg=<optimized out>) at action.c:1614
>>> #8 0x000000000053b2e8 in rval_get_int (h=0x7ffeb0b402b0, msg=<optimized
>>> out>, i=0x7ffeb0b3f1e0, rv=rv@entry=0x7fced4cb4d58,
>>> cache=cache@entry=0x0) at rvalue.c:912
>>> #9 0x000000000054261c in rval_expr_eval_int (h=h@entry=0x7ffeb0b402b0,
>>> msg=msg@entry=0x7fced4f093c8, res=res@entry=0x7ffeb0b3f1e0,
>>> rve=rve@entry=0x7fced4cb4d50) at rvalue.c:1910
>>> #10 0x000000000042bc91 in do_action (h=h@entry=0x7ffeb0b402b0,
>>> a=a@entry=0x7fced4cb8f88, msg=msg@entry=0x7fced4f093c8) at action.c:1030
>>> #11 0x000000000042a10a in run_actions (h=h@entry=0x7ffeb0b402b0,
>>> a=0x7fced4cb8f88, msg=msg@entry=0x7fced4f093c8) at action.c:1549
>>> #12 0x000000000042bcf2 in do_action (h=h@entry=0x7ffeb0b402b0,
>>> a=a@entry=0x7fced4cb91e8, msg=msg@entry=0x7fced4f093c8) at action.c:1049
>>> #13 0x000000000042a10a in run_actions (h=h@entry=0x7ffeb0b402b0,
>>> a=0x7fced4cb3f48, msg=msg@entry=0x7fced4f093c8) at action.c:1549
>>> #14 0x000000000042bde0 in do_action (h=h@entry=0x7ffeb0b402b0,
>>> a=a@entry=0x7fced4ac3d70, msg=msg@entry=0x7fced4f093c8) at action.c:678
>>> #15 0x000000000042a10a in run_actions (h=h@entry=0x7ffeb0b402b0,
>>> a=a@entry=0x7fced4ac1698, msg=msg@entry=0x7fced4f093c8) at action.c:1549
>>> #16 0x00000000004375d0 in run_top_route (a=0x7fced4ac1698,
>>> msg=msg@entry=0x7fced4f093c8, c=c@entry=0x0) at action.c:1635
>>> #17 0x0000000000504386 in receive_msg (buf=<optimized out>,
>>> len=<optimized out>, rcv_info=<optimized out>) at receive.c:240
>>> #18 0x00000000005f5bd4 in udp_rcv_loop () at udp_server.c:495
>>> #19 0x00000000004b2625 in main_loop () at main.c:1600
>>> #20 0x0000000000427e2b in main (argc=<optimized out>, argv=<optimized
>>> out>) at main.c:2616
>>>
>>>
>>> And from syslog the relevant messages before this dump:
>>>
>>> Jul 15 08:55:03 server /usr/sbin/kamailio[16470]: WARNING: dialog
>>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>>> route param 'd4c.26d1' [3149:7522]
>>> Jul 15 08:56:01 server /usr/sbin/kamailio[16481]: WARNING: dialog
>>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>>> route param 'fc.99f1' [207:8089]
>>> Jul 15 08:56:27 server /usr/sbin/kamailio[16470]: CRITICAL: dialog
>>> [dlg_timer.c:200]: update_dlg_timer(): Trying to update a bogus dlg
>>> tl=0x7fceb3f7d920 tl->next=(nil) tl->prev=(nil)
>>> Jul 15 08:56:27 server /usr/sbin/kamailio[16470]: ERROR: dialog
>>> [dlg_handlers.c:1377]: dlg_onroute(): failed to update dialog lifetime
>>> Jul 15 08:57:01 server /usr/sbin/kamailio[16482]: ERROR: db_mysql
>>> [km_dbase.c:128]: db_mysql_submit_query(): driver error on query:
>>> Duplicate entry '9584-3854-435' for key 'hash_index' (1062)
>>> Jul 15 08:57:01 server /usr/sbin/kamailio[16482]: ERROR: <core>
>>> [db_query.c:181]: db_do_raw_query(): error while submitting query
>>> Jul 15 08:57:01 server /usr/sbin/kamailio[16482]: ERROR: sqlops
>>> [sql_api.c:265]: sql_do_query(): cannot do the query [INSERT INTO
>>> `dialog_extra` (`h_i]
>>> Jul 15 08:57:01 server /usr/sbin/kamailio[16482]: ERROR: auth
>>> [api.c:119]: auth_check_hdr_md5(): auth:pre_auth: Credentials are not
>>> filled properly
>>> Jul 15 08:57:01 server /usr/sbin/kamailio[16483]: ERROR: auth
>>> [api.c:119]: auth_check_hdr_md5(): auth:pre_auth: Credentials are not
>>> filled properly
>>> Jul 15 08:57:54 server /usr/sbin/kamailio[16506]: NOTICE: dialog
>>> [dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
>>> (0x7fceb3f64470 ref 4)
>>> Jul 15 08:57:54 server /usr/sbin/kamailio[16473]: WARNING: dialog
>>> [dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
>>> 0x7fceb3f64470 [1182:5803] with clid
>>> '09ad128753e2535d24bde58e3d7eda04@192.168.10.232:5060' and tags
>>> 'as1b497b34' '5788890C-EC6F55F-3E86ED0C'
>>> Jul 15 08:57:54 server /usr/sbin/kamailio[16469]: ERROR: dialog
>>> [dlg_handlers.c:334]: dlg_terminated_confirmed(): failed to get dialog
>>> from params!
>>> Jul 15 08:58:49 server /usr/sbin/kamailio[16467]: WARNING: dialog
>>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>>> route param '6d2.2581' [726:6226]
>>> Jul 15 08:59:24 server /usr/sbin/kamailio[16506]: NOTICE: dialog
>>> [dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
>>> (0x7fceb400e2f0 ref 4)
>>> Jul 15 08:59:25 server /usr/sbin/kamailio[16464]: WARNING: dialog
>>> [dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
>>> route param '3e4.b5c1' [1251:7259]
>>> Jul 15 08:59:25 server /usr/sbin/kamailio[16465]: WARNING: dialog
>>> [dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
>>> 0x7fceb400e2f0 [1251:7259] with clid '87791a#015#012Call-ID:
>>> 25750e286a5654361ef9405d72edbc' and tags '' 'as148f41b1'
>>> Jul 15 08:59:25 server kernel: [203670.830521] kamailio[16465] general
>>> protection ip:7fcead34b3a5 sp:7ffeb0b3e220 error:0 in
>>> dialog.so[7fcead330000+88000]
>>> Jul 15 08:59:26 server /usr/sbin/kamailio[16511]: CRITICAL: <core>
>>> [pass_fd.c:275]: receive_fd(): EOF on 33
>>> Jul 15 08:59:26 server /usr/sbin/kamailio[16458]: ALERT: <core>
>>> [main.c:739]: handle_sigs(): child process 16465 exited by a signal 11
>>> Jul 15 08:59:26 server /usr/sbin/kamailio[16458]: ALERT: <core>
>>> [main.c:742]: handle_sigs(): core was generated
>>> Jul 15 08:59:26 server /usr/sbin/kamailio[16458]: INFO: <core>
>>> [main.c:754]: handle_sigs(): terminating due to SIGCHLD
>>>
>>> Any insight would be appreciated!
>>>
>>> Cheers,
>>> Dirk
>>>
>>> _______________________________________________
>>> SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
>>> sr-users@lists.sip-router.org
>>> http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
>>>
>> _______________________________________________
>> SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
>> sr-users@lists.sip-router.org
>> http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
>
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
