Hi, I'm running into a deadlock using openssl 1.0.0h. it's a multi-threaded ssl proxy based on openssl and libevent, but i think the problem is much more related to openssl than libevent.
there are 4 worker threads (5, 4, 3, 2), 1 external report thread (6), i think we can ignore this one, and 1 main thread. Incoming connections will be dispatched to these 4 worker threads. A X509_STORE object is shared by these 4 worker thread for peer certificate verification. here is the gdb output when deadlock happened. the OS is linux 2.6.35. I can not figure out how this deadlock could happen and will be glad to provide more information if required. any help will be appreciated. (gdb) thread apply all bt Thread 6 (Thread 0x7f2b9f13d950 (LWP 1280)): #0 0x00007f2b9fa1f3d8 in epoll_wait () from /lib/libc.so.6 #1 0x00007f2ba0fc15c5 in epoll_dispatch (base=0x8be4f40, tv=<optimized out>) at epoll.c:407 #2 0x00007f2ba0fafb26 in event_base_loop (base=0x8be4f40, flags=<optimized out>) at event.c:1603 #3 0x000000000040f98d in CTRL_main (arg=0x61c010) at ctrl.c:134 #4 0x00007f2ba03363c4 in start_thread () from /lib/libpthread.so.0 #5 0x00007f2b9fa1eefd in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 5 (Thread 0x7f2b9e93c950 (LWP 1281)): #0 0x00007f2ba033cfb4 in __lll_lock_wait () from /lib/libpthread.so.0 #1 0x00007f2ba0338a62 in _L_lock_52 () from /lib/libpthread.so.0 #2 0x00007f2ba03383a5 in pthread_mutex_lock () from /lib/libpthread.so.0 #3 0x00007f2ba12e41f5 in X509_PUBKEY_get () from /usr/lib/libcrypto.so.1.0.0 #4 0x00007f2ba1302282 in ?? () from /usr/lib/libcrypto.so.1.0.0 #5 0x00007f2ba1302d6d in X509_verify_cert () from /usr/lib/libcrypto.so.1.0.0 #6 0x00007f2ba077b4c2 in SYAN_X509_STORE_verify () from /usr/lib/libsyan2.so #7 0x0000000000409b5d in SSLX_verify_callback (ctx=<optimized out>, c=0x61c010, x=0x7f2b8c12dfa0) at sslx.c:139 #8 0x00007f2ba15fa2ae in ssl_verify_cert_chain () from /usr/lib/libssl.so.1.0.0 #9 0x00007f2ba15cad53 in ssl3_get_client_certificate () from /usr/lib/libssl.so.1.0.0 #10 0x00007f2ba15cefb7 in ssl3_accept () from /usr/lib/libssl.so.1.0.0 #11 0x00007f2ba0d9b6ce in do_handshake (bev_ssl=0x7f2b8805c740) at bufferevent_openssl.c:990 #12 0x00007f2ba0d9ba28 in be_openssl_handshakeeventcb (fd=<optimized out>, what=0, ptr=0x7f2b8805c740) at bufferevent_openssl.c:1046 #13 0x00007f2ba0fafe5c in event_process_active_single_queue (activeq=<optimized out>, base=<optimized out>) at event.c:1346 #14 event_process_active (base=<optimized out>) at event.c:1416 #15 event_base_loop (base=0x8be5bd0, flags=<optimized out>) at event.c:1617 #16 0x00007f2ba0994af5 in ?? () from /usr/lib/libevext.so #17 0x00007f2ba03363c4 in start_thread () from /lib/libpthread.so.0 #18 0x00007f2b9fa1eefd in clone () from /lib/libc.so.6 #19 0x0000000000000000 in ?? () Thread 4 (Thread 0x7f2b9e13b950 (LWP 1282)): #0 0x00007f2ba033cfb4 in __lll_lock_wait () from /lib/libpthread.so.0 #1 0x00007f2ba0338a62 in _L_lock_52 () from /lib/libpthread.so.0 #2 0x00007f2ba03383a5 in pthread_mutex_lock () from /lib/libpthread.so.0 #3 0x00007f2ba1257077 in CRYPTO_add_lock () from /usr/lib/libcrypto.so.1.0.0 #4 0x00007f2ba12dbc23 in ?? () from /usr/lib/libcrypto.so.1.0.0 #5 0x00007f2ba12d5f4a in EVP_VerifyFinal () from /usr/lib/libcrypto.so.1.0.0 #6 0x00007f2ba12e1fb6 in ASN1_item_verify () from /usr/lib/libcrypto.so.1.0.0 #7 0x00007f2ba1302299 in ?? () from /usr/lib/libcrypto.so.1.0.0 #8 0x00007f2ba1302d6d in X509_verify_cert () from /usr/lib/libcrypto.so.1.0.0 #9 0x00007f2ba077b4c2 in SYAN_X509_STORE_verify () from /usr/lib/libsyan2.so #10 0x0000000000409b5d in SSLX_verify_callback (ctx=<optimized out>, c=0x61c010, x=0x8e3c370) at sslx.c:139 #11 0x00007f2ba15fa2ae in ssl_verify_cert_chain () from /usr/lib/libssl.so.1.0.0 #12 0x00007f2ba15cad53 in ssl3_get_client_certificate () from /usr/lib/libssl.so.1.0.0 #13 0x00007f2ba15cefb7 in ssl3_accept () from /usr/lib/libssl.so.1.0.0 #14 0x00007f2ba0d9b6ce in do_handshake (bev_ssl=0x7f2b8c059990) at bufferevent_openssl.c:990 #15 0x00007f2ba0d9ba28 in be_openssl_handshakeeventcb (fd=<optimized out>, what=0, ptr=0x7f2b8c059990) at bufferevent_openssl.c:1046 #16 0x00007f2ba0fafe5c in event_process_active_single_queue (activeq=<optimized out>, base=<optimized out>) at event.c:1346 #17 event_process_active (base=<optimized out>) at event.c:1416 #18 event_base_loop (base=0x8be8310, flags=<optimized out>) at event.c:1617 #19 0x00007f2ba0994af5 in ?? () from /usr/lib/libevext.so #20 0x00007f2ba03363c4 in start_thread () from /lib/libpthread.so.0 #21 0x00007f2b9fa1eefd in clone () from /lib/libc.so.6 #22 0x0000000000000000 in ?? () Thread 3 (Thread 0x7f2b9d93a950 (LWP 1283)): #0 0x00007f2ba033cfb4 in __lll_lock_wait () from /lib/libpthread.so.0 #1 0x00007f2ba0338a62 in _L_lock_52 () from /lib/libpthread.so.0 #2 0x00007f2ba03383a5 in pthread_mutex_lock () from /lib/libpthread.so.0 #3 0x00007f2ba1257077 in CRYPTO_add_lock () from /usr/lib/libcrypto.so.1.0.0 #4 0x00007f2ba12d6299 in EVP_PKEY_free () from /usr/lib/libcrypto.so.1.0.0 #5 0x00007f2ba12e4204 in X509_PUBKEY_get () from /usr/lib/libcrypto.so.1.0.0 #6 0x00007f2ba1302282 in ?? () from /usr/lib/libcrypto.so.1.0.0 #7 0x00007f2ba1302d6d in X509_verify_cert () from /usr/lib/libcrypto.so.1.0.0 #8 0x00007f2ba077b4c2 in SYAN_X509_STORE_verify () from /usr/lib/libsyan2.so #9 0x0000000000409b5d in SSLX_verify_callback (ctx=<optimized out>, c=0x61c010, x=0x7f2b880d99a0) at sslx.c:139 #10 0x00007f2ba15fa2ae in ssl_verify_cert_chain () from /usr/lib/libssl.so.1.0.0 #11 0x00007f2ba15cad53 in ssl3_get_client_certificate () from /usr/lib/libssl.so.1.0.0 #12 0x00007f2ba15cefb7 in ssl3_accept () from /usr/lib/libssl.so.1.0.0 #13 0x00007f2ba0d9b6ce in do_handshake (bev_ssl=0x8c72330) at bufferevent_openssl.c:990 #14 0x00007f2ba0d9ba28 in be_openssl_handshakeeventcb (fd=<optimized out>, what=0, ptr=0x8c72330) at bufferevent_openssl.c:1046 #15 0x00007f2ba0fafe5c in event_process_active_single_queue (activeq=<optimized out>, base=<optimized out>) at event.c:1346 #16 event_process_active (base=<optimized out>) at event.c:1416 #17 event_base_loop (base=0x8beaa50, flags=<optimized out>) at event.c:1617 ---Type <return> to continue, or q <return> to quit--- #18 0x00007f2ba0994af5 in ?? () from /usr/lib/libevext.so #19 0x00007f2ba03363c4 in start_thread () from /lib/libpthread.so.0 #20 0x00007f2b9fa1eefd in clone () from /lib/libc.so.6 #21 0x0000000000000000 in ?? () Thread 2 (Thread 0x7f2b97fff950 (LWP 1284)): #0 0x00007f2ba033cfb4 in __lll_lock_wait () from /lib/libpthread.so.0 #1 0x00007f2ba0338a62 in _L_lock_52 () from /lib/libpthread.so.0 #2 0x00007f2ba03383a5 in pthread_mutex_lock () from /lib/libpthread.so.0 #3 0x00007f2ba1257077 in CRYPTO_add_lock () from /usr/lib/libcrypto.so.1.0.0 #4 0x00007f2ba12d6299 in EVP_PKEY_free () from /usr/lib/libcrypto.so.1.0.0 #5 0x00007f2ba12db7ad in EVP_PKEY_CTX_free () from /usr/lib/libcrypto.so.1.0.0 #6 0x00007f2ba12d5fa3 in EVP_VerifyFinal () from /usr/lib/libcrypto.so.1.0.0 #7 0x00007f2ba12e1fb6 in ASN1_item_verify () from /usr/lib/libcrypto.so.1.0.0 #8 0x00007f2ba1302299 in ?? () from /usr/lib/libcrypto.so.1.0.0 #9 0x00007f2ba1302d6d in X509_verify_cert () from /usr/lib/libcrypto.so.1.0.0 #10 0x00007f2ba077b4c2 in SYAN_X509_STORE_verify () from /usr/lib/libsyan2.so #11 0x0000000000409b5d in SSLX_verify_callback (ctx=<optimized out>, c=0x61c010, x=0x7f2b9811dd30) at sslx.c:139 #12 0x00007f2ba15fa2ae in ssl_verify_cert_chain () from /usr/lib/libssl.so.1.0.0 #13 0x00007f2ba15cad53 in ssl3_get_client_certificate () from /usr/lib/libssl.so.1.0.0 #14 0x00007f2ba15cefb7 in ssl3_accept () from /usr/lib/libssl.so.1.0.0 #15 0x00007f2ba0d9b6ce in do_handshake (bev_ssl=0x8c379b0) at bufferevent_openssl.c:990 #16 0x00007f2ba0d9ba28 in be_openssl_handshakeeventcb (fd=<optimized out>, what=0, ptr=0x8c379b0) at bufferevent_openssl.c:1046 #17 0x00007f2ba0fafe5c in event_process_active_single_queue (activeq=<optimized out>, base=<optimized out>) at event.c:1346 #18 event_process_active (base=<optimized out>) at event.c:1416 #19 event_base_loop (base=0x8bed1e0, flags=<optimized out>) at event.c:1617 #20 0x00007f2ba0994af5 in ?? () from /usr/lib/libevext.so #21 0x00007f2ba03363c4 in start_thread () from /lib/libpthread.so.0 #22 0x00007f2b9fa1eefd in clone () from /lib/libc.so.6 #23 0x0000000000000000 in ?? () Thread 1 (Thread 0x7f2ba1a2a6f0 (LWP 1279)): #0 0x00007f2b9fa1f3d8 in epoll_wait () from /lib/libc.so.6 #1 0x00007f2ba0fc15c5 in epoll_dispatch (base=0x8be52c0, tv=<optimized out>) at epoll.c:407 #2 0x00007f2ba0fafb26 in event_base_loop (base=0x8be52c0, flags=<optimized out>) at event.c:1603 #3 0x000000000040735c in ETUNNEL_main (opt=0x61c010) at etunnel.c:508 #4 0x00007f2b9f97453d in __libc_start_main () from /lib/libc.so.6 #5 0x0000000000406d99 in _start () ______________________________________________________________________ OpenSSL Project http://www.openssl.org User Support Mailing List openssl-users@openssl.org Automated List Manager majord...@openssl.org
