Greetings OpenSSL Users list, In the TR-069 WAN management space, the plethora of user devices in the home have to support SSL-based communications with the server (over the WAN). Recently, one of our integration customers has been subjecting their client to WAN congestion simulation using WANem (wanem.sourceforge.net) on request of an operator for testing. What has been found is that under high packet loss (approximate 50%) and worse, non-SSL operation works fine, but SSL communications will randomly end up stuck on a blocked call, regardless of using select() first, socket timeouts, etc. The usual suspect call that is at the top of most of the stack backtraces is __read_nocancel(). The "bad" blocking behavior has been experienced using OpenSSL 0.9.7f through 0.9.8k.
The question to the list is how to stop the blocked call behavior so device communication threads don't get stuck? Said differently, if non-SSL isn't blocking, why is SSL getting stuck? Thanks for any suggestions (besides a watchdog time/observer) that can help correct the problem. Background: the TR-069 client uses the gSoap system that in turn calls OpenSSL. The communications to the server at HTTP/SOAP based using SSL or non_SSL. The problems are being experienced in Linux 2.6.x systems 32-bit and 64-bit, on MIPS and AMD processors; i.e. both embedded Linux systems and normal development systems. WANem is configured for T1 link, 100ms delay, 10ms jitter, and 40% to 50% packet loss. gSoap uses a select() call with timeout prior to calling SSL_read. In addition, I added code to set SO_RCVTIMEO and SO_SNDTIMEO to 60 seconds on the socket. Various stack backtraces from entry into gSoap are presented below, each one was captured from a core file produced from kill -3'ing the hung up client. They are just representative of the problem happening from several different entry points into OpenSSL. (gdb) bt 1 #0 0x0000003b8d40bf7b in __read_nocancel () from /lib64/libpthread.so.0 #1 0x0000003b91499091 in BIO_new_socket () from /lib64/libcrypto.so.6 #2 0x0000003b9149766f in BIO_read () from /lib64/libcrypto.so.6 #3 0x0000003f9642047d in ssl3_read_n () from /lib64/libssl.so.6 #4 0x0000003f964209dd in ssl3_read_bytes () from /lib64/libssl.so.6 #5 0x0000003f9641de64 in ssl3_shutdown () from /lib64/libssl.so.6 #6 0x0000000000455961 in tcp_disconnect (soap=0x596960) at gsoap/stdsoap2.c:4013 #7 0x0000000000455c9c in soap_closesock (soap=0x596960) at gsoap/stdsoap2.c:4069 gdb) bt 2 #0 0x0000003b8d40bf7b in __read_nocancel () from /lib64/libpthread.so.0 #1 0x0000003b91499091 in BIO_new_socket () from /lib64/libcrypto.so.6 #2 0x0000003b9149766f in BIO_read () from /lib64/libcrypto.so.6 #3 0x0000003f9642047d in ssl3_read_n () from /lib64/libssl.so.6 #4 0x0000003f96420a7f in ssl3_read_bytes () from /lib64/libssl.so.6 #5 0x0000003f964216c6 in ssl3_get_message () from /lib64/libssl.so.6 #6 0x0000003f9641c5f6 in ssl3_get_server_hello () from /lib64/libssl.so.6 #7 0x0000003f9641d1b6 in ssl3_connect () from /lib64/libssl.so.6 #8 0x0000003f9642334f in ssl23_connect () from /lib64/libssl.so.6 #9 0x0000000000453bbc in tcp_connect (soap=0x596960, endpoint=0x595de0 "https://10.2.2.22:8443/dps/TR069";, host=0x5997f0 "10.2.2.22", port=8443) at gsoap/stdsoap2.c:3459 #10 0x000000000046d8e8 in soap_connect_command (soap=0x596960, http_command=2000, endpoint=0x595de0 "https://10.2.2.22:8443/dps/TR069";, action=0x476f2f "") at gsoap/stdsoap2.c:12234 #11 0x000000000046d689 in soap_connect (soap=0x596960, endpoint=0x595de0 "https://10.2.2.22:8443/dps/TR069";, action=0x476f2f "") at gsoap/stdsoap2.c:12195 gdb) bt 3 #0 0x0000003b8d40bf7b in __read_nocancel () from /lib64/libpthread.so.0 #1 0x00000000004a8beb in sock_read () #2 0x00000000004a7d6b in BIO_read () #3 0x0000000000476fce in ssl23_read_bytes () #4 0x0000000000476344 in ssl23_connect () #5 0x00000000004528bc in tcp_connect (soap=0x6a8b40, endpoint=0x68dc40 "https://10.2.2.22:8443/dps/TR069";, host=0x6ab9d0 "10.2.2.22", port=8443) at gsoap/stdsoap2.c:3459 #6 0x000000000046c5e8 in soap_connect_command (soap=0x6a8b40, http_command=2000, endpoint=0x68dc40 "https://10.2.2.22:8443/dps/TR069";, action=0x52432f "") at gsoap/stdsoap2.c:12234 #7 0x000000000046c389 in soap_connect (soap=0x6a8b40, endpoint=0x68dc40 "https://10.2.2.22:8443/dps/TR069";, action=0x52432f "") at gsoap/stdsoap2.c:12195 (gdb) bt 4 #0 0x0000000000496332 in CRYPTO_lock () #1 0x00000000004eec8f in ssleay_rand_add () #2 0x00000000004907ec in ssl3_connect () #3 0x00000000004945da in ssl3_write_bytes () #4 0x000000000044918a in fsend (soap=0x6a8b60, s=0x6a8f40 "POST /dps/TR069 HTTP/1.1\r\nHost: 10.2.2.22:8443\r\nUser-Agent: gSOAP/2.7\r\nContent-Type: text/xml; charset=utf-8\r\nContent-Length: \ 2393\r\nConnection: keep-alive\r\nSOAPAction: \"\"\r\n\r\n\" xmlns:xsi=\"http://www.w3";..., n=174) at gsoap/stdsoap2.c:468 #5 0x000000000044984f in soap_flush_raw (soap=0x6a8b60, s=0x6a8f40 "POST /dps/TR069 HTTP/1.1\r\nHost: 10.2.2.22:8443\r\nUser-Agent: gSOAP/2.7\r\nContent-Type: text/xml; charset=utf-8\r\nContent-Length: \ 2393\r\nConnection: keep-alive\r\nSOAPAction: \"\"\r\n\r\n\" xmlns:xsi=\"http://www.w3";..., n=174) at gsoap/stdsoap2.c:669 #6 0x000000000044957f in soap_flush (soap=0x6a8b60) at gsoap/stdsoap2.c:635 #7 0x0000000000458c8c in soap_end_send (soap=0x6a8b60) at gsoap/stdsoap2.c:5365 (gdb) bt 5 #0 0x0000003b8c36d478 in free () from /lib64/libc.so.6 #1 0x0000000000496d0d in CRYPTO_free () #2 0x00000000004af9d6 in EVP_MD_CTX_cleanup () #3 0x00000000004eedc3 in ssleay_rand_add () #4 0x00000000004909fc in ssl3_connect () #5 0x00000000004947ea in ssl3_write_bytes () #6 0x000000000044918a in fsend (soap=0x6a8da0, s=0x6a9180 "POST /dps/TR069 HTTP/1.1\r\nHost: 10.2.2.22:8443\r\nUser-Agent: gSOAP/2.7\r\nContent-Type: text/xml; charset=utf-8\r\nContent-Length: \ 2393\r\nConnection: keep-alive\r\nSOAPAction: \"\"\r\n\r\n\" xmlns:xsi=\"http://www.w3";..., n=174) at gsoap/stdsoap2.c:468 #7 0x000000000044984f in soap_flush_raw (soap=0x6a8da0, s=0x6a9180 "POST /dps/TR069 HTTP/1.1\r\nHost: 10.2.2.22:8443\r\nUser-Agent: gSOAP/2.7\r\nContent-Type: text/xml; charset=utf-8\r\nContent-Length: \ 2393\r\nConnection: keep-alive\r\nSOAPAction: \"\"\r\n\r\n\" xmlns:xsi=\"http://www.w3";..., n=174) at gsoap/stdsoap2.c:669 #8 0x000000000044957f in soap_flush (soap=0x6a8da0) at gsoap/stdsoap2.c:635 #9 0x0000000000458e9c in soap_end_send (soap=0x6a8da0) at gsoap/stdsoap2.c:5399 (gdb) bt 6 #0 0x0000003b8d40bf7b in __read_nocancel () from /lib64/libpthread.so.0 #1 0x00000000004a8e0b in sock_read () #2 0x00000000004a7f8b in BIO_read () #3 0x000000000049411d in ssl3_read_n () #4 0x0000000000494c60 in ssl3_read_bytes () #5 0x0000000000495b44 in ssl3_get_message () #6 0x000000000048fea1 in ssl3_get_server_hello () #7 0x0000000000490a66 in ssl3_connect () #8 0x00000000004947da in ssl3_write_bytes () #9 0x0000000000449194 in fsend (soap=0x6a8de0, s=0x6a91c0 "POST /dps/TR069 HTTP/1.1\r\nHost: 10.2.2.22:8443\r\nUser-Agent: gSOAP/2.7\r\nContent-Type: text/xml; charset=utf-8\r\nContent-Length: \ 2393\r\nConnection: keep-alive\r\nSOAPAction: \"\"\r\n\r\n\" xmlns:xsi=\"http://www.w3";..., n=174) at gsoap/stdsoap2.c:470 #10 0x0000000000449859 in soap_flush_raw (soap=0x6a8de0, s=0x6a91c0 "POST /dps/TR069 HTTP/1.1\r\nHost: 10.2.2.22:8443\r\nUser-Agent: gSOAP/2.7\r\nContent-Type: text/xml; charset=utf-8\r\nContent-Length: \ 2393\r\nConnection: keep-alive\r\nSOAPAction: \"\"\r\n\r\n\" xmlns:xsi=\"http://www.w3";..., n=174) at gsoap/stdsoap2.c:671 #11 0x0000000000449589 in soap_flush (soap=0x6a8de0) at gsoap/stdsoap2.c:637 #12 0x0000000000458e8c in soap_end_send (soap=0x6a8de0) at gsoap/stdsoap2.c:5399 Thanks for any help!! -- Mark Laubach Chief Technology Officer Dimark Technologies, Inc. www.dimark.com m...@dimark.com ______________________________________________________________________ OpenSSL Project http://www.openssl.org User Support Mailing List openssl-users@openssl.org Automated List Manager majord...@openssl.org
