details: https://github.com/nginx/nginx/commit/6a134dfd4888fc3850d22294687cfb3940994c69 branches: master commit: 6a134dfd4888fc3850d22294687cfb3940994c69 user: Sergey Kandaurov <pluk...@nginx.com> date: Thu, 13 Feb 2025 17:00:56 +0400 description: QUIC: using QUIC API introduced in OpenSSL 3.5.
Similarly to the QUIC API originated in BoringSSL, this API allows to register custom TLS callbacks for an external QUIC implementation. See the SSL_set_quic_tls_cbs manual page for details. Due to a different approach used in OpenSSL 3.5, handling of CRYPTO frames was streamlined to always write an incoming CRYPTO buffer to the crypto context. Using SSL_provide_quic_data(), this results in transient allocation of chain links and buffers for CRYPTO frames received in order. Testing didn't reveal performance degradation of QUIC handshakes, https://github.com/nginx/nginx/pull/646 provides specific results. --- auto/lib/openssl/conf | 10 +- src/event/quic/ngx_event_quic.h | 5 +- src/event/quic/ngx_event_quic_connection.h | 5 + src/event/quic/ngx_event_quic_ssl.c | 411 +++++++++++++++++++++++++---- 4 files changed, 383 insertions(+), 48 deletions(-) diff --git a/auto/lib/openssl/conf b/auto/lib/openssl/conf index f4b00ebd6..3068cae36 100644 --- a/auto/lib/openssl/conf +++ b/auto/lib/openssl/conf @@ -147,11 +147,17 @@ else if [ $USE_OPENSSL_QUIC = YES ]; then - ngx_feature="OpenSSL QUIC support" + ngx_feature="OpenSSL QUIC API" ngx_feature_name="NGX_QUIC" - ngx_feature_test="SSL_set_quic_method(NULL, NULL)" + ngx_feature_test="SSL_set_quic_tls_cbs(NULL, NULL, NULL)" . auto/feature + if [ $ngx_found = no ]; then + ngx_feature="BoringSSL-like QUIC API" + ngx_feature_test="SSL_set_quic_method(NULL, NULL)" + . auto/feature + fi + if [ $ngx_found = no ]; then ngx_feature="OpenSSL QUIC compatibility" ngx_feature_test="SSL_CTX_add_custom_ext(NULL, 0, 0, diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h index 50a5c214e..d95d3d85b 100644 --- a/src/event/quic/ngx_event_quic.h +++ b/src/event/quic/ngx_event_quic.h @@ -12,7 +12,10 @@ #include <ngx_core.h> -#ifdef SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION +#ifdef OSSL_RECORD_PROTECTION_LEVEL_NONE +#define NGX_QUIC_OPENSSL_API 1 + +#elif (defined SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION) #define NGX_QUIC_QUICTLS_API 1 #elif (defined OPENSSL_IS_BORINGSSL || defined LIBRESSL_VERSION_NUMBER) diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index 856512118..33922cf80 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -301,6 +301,11 @@ struct ngx_quic_connection_s { unsigned key_phase:1; unsigned validated:1; unsigned client_tp_done:1; + +#if (NGX_QUIC_OPENSSL_API) + unsigned read_level:2; + unsigned write_level:2; +#endif }; diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index 6ce926c81..e961c80cd 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -18,6 +18,23 @@ #define NGX_QUIC_MAX_BUFFERED 65535 +#if (NGX_QUIC_OPENSSL_API) + +static int ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn, + const unsigned char *data, size_t len, size_t *consumed, void *arg); +static int ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn, + const unsigned char **data, size_t *bytes_read, void *arg); +static int ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn, + size_t bytes_read, void *arg); +static int ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t level, + int direction, const unsigned char *secret, size_t secret_len, void *arg); +static int ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn, + const unsigned char *params, size_t params_len, void *arg); +static int ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert, + void *arg); + +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ + static ngx_inline ngx_uint_t ngx_quic_map_encryption_level( enum ssl_encryption_level_t ssl_level); @@ -39,9 +56,270 @@ static int ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, static int ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn); static int ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t ssl_level, uint8_t alert); + +#endif + static ngx_int_t ngx_quic_handshake(ngx_connection_t *c); -static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, - ngx_uint_t level); +static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level); + + +#if (NGX_QUIC_OPENSSL_API) + +static int +ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn, + const unsigned char *data, size_t len, size_t *consumed, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_chain_t *out; + unsigned int alpn_len; + ngx_quic_frame_t *frame; + const unsigned char *alpn_data; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_send len:%uz", len); + + qc = ngx_quic_get_connection(c); + + *consumed = 0; + + SSL_get0_alpn_selected(ssl_conn, &alpn_data, &alpn_len); + + if (alpn_len == 0) { + qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_NO_APPLICATION_PROTOCOL); + qc->error_reason = "missing ALPN extension"; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic missing ALPN extension"); + return 1; + } + + if (!qc->client_tp_done) { + /* RFC 9001, 8.2. QUIC Transport Parameters Extension */ + qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_MISSING_EXTENSION); + qc->error_reason = "missing transport parameters"; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "missing transport parameters"); + return 1; + } + + ctx = ngx_quic_get_send_ctx(qc, qc->write_level); + + out = ngx_quic_copy_buffer(c, (u_char *) data, len); + if (out == NGX_CHAIN_ERROR) { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + frame = ngx_quic_alloc_frame(c); + if (frame == NULL) { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + frame->data = out; + frame->level = qc->write_level; + frame->type = NGX_QUIC_FT_CRYPTO; + frame->u.crypto.offset = ctx->crypto_sent; + frame->u.crypto.length = len; + + ctx->crypto_sent += len; + *consumed = len; + + ngx_quic_queue_frame(qc, frame); + + return 1; +} + + +static int +ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn, + const unsigned char **data, size_t *bytes_read, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_buf_t *b; + ngx_chain_t *cl; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_recv_rcd"); + + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, qc->read_level); + + for (cl = ctx->crypto.chain; cl; cl = cl->next) { + b = cl->buf; + + if (b->sync) { + /* hole */ + + *bytes_read = 0; + + break; + } + + *data = b->pos; + *bytes_read = b->last - b->pos; + + break; + } + + return 1; +} + + +static int +ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn, size_t bytes_read, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_chain_t *cl; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_release_rcd len:%uz", bytes_read); + + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, qc->read_level); + + cl = ngx_quic_read_buffer(c, &ctx->crypto, bytes_read); + if (cl == NGX_CHAIN_ERROR) { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + ngx_quic_free_chain(c, cl); + + return 1; +} + + +static int +ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t ssl_level, + int direction, const unsigned char *secret, size_t secret_len, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_uint_t level; + const SSL_CIPHER *cipher; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_yield_secret() level:%uD", ssl_level); +#ifdef NGX_QUIC_DEBUG_CRYPTO + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic %s secret len:%uz %*xs", + direction ? "write" : "read", secret_len, + secret_len, secret); +#endif + + qc = ngx_quic_get_connection(c); + cipher = SSL_get_current_cipher(ssl_conn); + + switch (ssl_level) { + case OSSL_RECORD_PROTECTION_LEVEL_NONE: + level = NGX_QUIC_ENCRYPTION_INITIAL; + break; + case OSSL_RECORD_PROTECTION_LEVEL_EARLY: + level = NGX_QUIC_ENCRYPTION_EARLY_DATA; + break; + case OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE: + level = NGX_QUIC_ENCRYPTION_HANDSHAKE; + break; + default: /* OSSL_RECORD_PROTECTION_LEVEL_APPLICATION */ + level = NGX_QUIC_ENCRYPTION_APPLICATION; + break; + } + + if (ngx_quic_keys_set_encryption_secret(c->log, direction, qc->keys, level, + cipher, secret, secret_len) + != NGX_OK) + { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + if (direction) { + qc->write_level = level; + + } else { + qc->read_level = level; + } + + return 1; +} + + +static int +ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn, + const unsigned char *params, size_t params_len, void *arg) +{ + ngx_connection_t *c = arg; + + u_char *p, *end; + ngx_quic_tp_t ctp; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_got_transport_params() len:%uz", + params_len); + + qc = ngx_quic_get_connection(c); + + /* defaults for parameters not sent by client */ + ngx_memcpy(&ctp, &qc->ctp, sizeof(ngx_quic_tp_t)); + + p = (u_char *) params; + end = p + params_len; + + if (ngx_quic_parse_transport_params(p, end, &ctp, c->log) != NGX_OK) { + qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR; + qc->error_reason = "failed to process transport parameters"; + + return 1; + } + + if (ngx_quic_apply_transport_params(c, &ctp) != NGX_OK) { + return 1; + } + + qc->client_tp_done = 1; + + return 1; +} + + +static int +ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_alert() alert:%d", (int) alert); + + /* already closed on regular shutdown */ + + qc = ngx_quic_get_connection(c); + if (qc == NULL) { + return 1; + } + + qc->error = NGX_QUIC_ERR_CRYPTO(alert); + qc->error_reason = "handshake failed"; + + return 1; +} + + +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ static ngx_inline ngx_uint_t @@ -340,13 +618,14 @@ ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, return 1; } +#endif + ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_frame_t *frame) { uint64_t last; - ngx_chain_t *cl; ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; ngx_quic_crypto_frame_t *f; @@ -385,41 +664,18 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, return NGX_OK; } - if (f->offset == ctx->crypto.offset) { - if (ngx_quic_crypto_provide(c, frame->data, pkt->level) != NGX_OK) { - return NGX_ERROR; - } - - if (ngx_quic_handshake(c) != NGX_OK) { - return NGX_ERROR; - } - - ngx_quic_skip_buffer(c, &ctx->crypto, last); - - } else { - if (ngx_quic_write_buffer(c, &ctx->crypto, frame->data, f->length, - f->offset) - == NGX_CHAIN_ERROR) - { - return NGX_ERROR; - } + if (ngx_quic_write_buffer(c, &ctx->crypto, frame->data, f->length, + f->offset) + == NGX_CHAIN_ERROR) + { + return NGX_ERROR; } - cl = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1); - - if (cl) { - if (ngx_quic_crypto_provide(c, cl, pkt->level) != NGX_OK) { - return NGX_ERROR; - } - - if (ngx_quic_handshake(c) != NGX_OK) { - return NGX_ERROR; - } - - ngx_quic_free_chain(c, cl); + if (ngx_quic_crypto_provide(c, pkt->level) != NGX_OK) { + return NGX_ERROR; } - return NGX_OK; + return ngx_quic_handshake(c); } @@ -528,13 +784,24 @@ ngx_quic_handshake(ngx_connection_t *c) static ngx_int_t -ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, - ngx_uint_t level) +ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level) { +#if (NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API) + ngx_buf_t *b; - ngx_chain_t *cl; + ngx_chain_t *out, *cl; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; enum ssl_encryption_level_t ssl_level; + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, level); + + out = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1); + if (out == NGX_CHAIN_ERROR) { + return NGX_ERROR; + } + switch (level) { case NGX_QUIC_ENCRYPTION_INITIAL: ssl_level = ssl_encryption_initial; @@ -562,6 +829,10 @@ ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, } } + ngx_quic_free_chain(c, out); + +#endif + return NGX_OK; } @@ -569,14 +840,40 @@ ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, ngx_int_t ngx_quic_init_connection(ngx_connection_t *c) { - u_char *p; - size_t clen; - ssize_t len; - ngx_str_t dcid; - ngx_ssl_conn_t *ssl_conn; - ngx_quic_socket_t *qsock; - ngx_quic_connection_t *qc; - static SSL_QUIC_METHOD quic_method; + u_char *p; + size_t clen; + ssize_t len; + ngx_str_t dcid; + ngx_ssl_conn_t *ssl_conn; + ngx_quic_socket_t *qsock; + ngx_quic_connection_t *qc; + +#if (NGX_QUIC_OPENSSL_API) + static const OSSL_DISPATCH qtdis[] = { + + { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_SEND, + (void (*)(void)) ngx_quic_cbs_send }, + + { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RECV_RCD, + (void (*)(void)) ngx_quic_cbs_recv_rcd }, + + { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RELEASE_RCD, + (void (*)(void)) ngx_quic_cbs_release_rcd }, + + { OSSL_FUNC_SSL_QUIC_TLS_YIELD_SECRET, + (void (*)(void)) ngx_quic_cbs_yield_secret }, + + { OSSL_FUNC_SSL_QUIC_TLS_GOT_TRANSPORT_PARAMS, + (void (*)(void)) ngx_quic_cbs_got_transport_params }, + + { OSSL_FUNC_SSL_QUIC_TLS_ALERT, + (void (*)(void)) ngx_quic_cbs_alert }, + + { 0, NULL } + }; +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ + static SSL_QUIC_METHOD quic_method; +#endif qc = ngx_quic_get_connection(c); @@ -588,6 +885,20 @@ ngx_quic_init_connection(ngx_connection_t *c) ssl_conn = c->ssl->connection; +#if (NGX_QUIC_OPENSSL_API) + + if (SSL_set_quic_tls_cbs(ssl_conn, qtdis, c) == 0) { + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, + "quic SSL_set_quic_tls_cbs() failed"); + return NGX_ERROR; + } + + if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) { + SSL_set_quic_tls_early_data_enabled(ssl_conn, 1); + } + +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ + if (!quic_method.send_alert) { #if (NGX_QUIC_BORINGSSL_API) quic_method.set_read_secret = ngx_quic_set_read_secret; @@ -610,6 +921,8 @@ ngx_quic_init_connection(ngx_connection_t *c) if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) { SSL_set_quic_early_data_enabled(ssl_conn, 1); } +#endif + #endif qsock = ngx_quic_get_socket(c); @@ -641,11 +954,19 @@ ngx_quic_init_connection(ngx_connection_t *c) "quic transport parameters len:%uz %*xs", len, len, p); #endif +#if (NGX_QUIC_OPENSSL_API) + if (SSL_set_quic_tls_transport_params(ssl_conn, p, len) == 0) { + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, + "quic SSL_set_quic_tls_transport_params() failed"); + return NGX_ERROR; + } +#else if (SSL_set_quic_transport_params(ssl_conn, p, len) == 0) { ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "quic SSL_set_quic_transport_params() failed"); return NGX_ERROR; } +#endif #ifdef OPENSSL_IS_BORINGSSL if (SSL_set_quic_early_data_context(ssl_conn, p, clen) == 0) { _______________________________________________ nginx-devel mailing list nginx-devel@nginx.org https://mailman.nginx.org/mailman/listinfo/nginx-devel
