On 7/23/2024 12:30 PM, Mini Hawthorne wrote:
# HG changeset patch
# User Mini Hawthorne <m...@f5.com>
# Date 1721762842 0
# Tue Jul 23 19:27:22 2024 +0000
# Node ID 8eee61e223bb7cb7475e50b866fd6b9a83fa5fa0
# Parent 59ac183dfee8e9641563e043eb19480d91dd7cc0
SSL: object caching.
Added ngx_openssl_cache_module, which indexes a type-aware object cache.
It maps an id to a unique instance, and provides references to it, which
are dropped when the cycle's pool is destroyed. Also, for those objects
that can be cached, valid references may be pulled from cycle->old_cycle.
The cache will be used in subsequent patches.
diff --git a/auto/modules b/auto/modules
--- a/auto/modules
+++ b/auto/modules
@@ -1307,10 +1307,11 @@ fi
if [ $USE_OPENSSL = YES ]; then
ngx_module_type=CORE
- ngx_module_name=ngx_openssl_module
+ ngx_module_name="ngx_openssl_module ngx_openssl_cache_module"
ngx_module_incs=
ngx_module_deps=src/event/ngx_event_openssl.h
ngx_module_srcs="src/event/ngx_event_openssl.c
+ src/event/ngx_event_openssl_cache.c
src/event/ngx_event_openssl_stapling.c"
ngx_module_libs=
ngx_module_link=YES
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -83,7 +83,8 @@
#endif
-typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t;
+typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t;
+typedef struct ngx_ssl_cache_type_s ngx_ssl_cache_type_t;
typedef struct {
@@ -233,6 +234,9 @@ ngx_int_t ngx_ssl_ocsp_get_status(ngx_co
void ngx_ssl_ocsp_cleanup(ngx_connection_t *c);
ngx_int_t ngx_ssl_ocsp_cache_init(ngx_shm_zone_t *shm_zone, void *data);
+void *ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
+ ngx_ssl_cache_type_t *type, char **err, ngx_str_t *id, void *data);
+
ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file);
ngx_array_t *ngx_ssl_preserve_passwords(ngx_conf_t *cf,
ngx_array_t *passwords);
diff --git a/src/event/ngx_event_openssl_cache.c
b/src/event/ngx_event_openssl_cache.c
new file mode 100644
--- /dev/null
+++ b/src/event/ngx_event_openssl_cache.c
@@ -0,0 +1,277 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+typedef struct {
+ ngx_rbtree_node_t node;
+ ngx_str_t id;
+
+ ngx_ssl_cache_type_t *type;
+ void *value;
+} ngx_ssl_cache_node_t;
+
+
+typedef void *(*ngx_ssl_cache_create_pt)(ngx_str_t *id, char **err, void
*data);
+typedef void (*ngx_ssl_cache_free_pt)(void *data);
+typedef void *(*ngx_ssl_cache_ref_pt)(char **err, void *data);
+
+
+struct ngx_ssl_cache_type_s {
+ const char *name;
+
+ ngx_ssl_cache_create_pt create;
+ ngx_ssl_cache_free_pt free;
+ ngx_ssl_cache_ref_pt ref;
+};
+
+
+typedef struct {
+ ngx_rbtree_t rbtree;
+ ngx_rbtree_node_t sentinel;
+} ngx_ssl_cache_t;
+
+
+static void *ngx_openssl_cache_create_conf(ngx_cycle_t *cycle);
+static void ngx_ssl_cache_cleanup(void *data);
+static void ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp,
+ ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
+
+static ngx_ssl_cache_node_t *ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache,
+ ngx_ssl_cache_type_t *type, ngx_str_t *id, uint32_t hash);
+
+
+static ngx_core_module_t ngx_openssl_cache_module_ctx = {
+ ngx_string("openssl_cache"),
+ ngx_openssl_cache_create_conf,
+ NULL
+};
+
+
+ngx_module_t ngx_openssl_cache_module = {
+ NGX_MODULE_V1,
+ &ngx_openssl_cache_module_ctx, /* module context */
+ NULL, /* module directives */
+ NGX_CORE_MODULE, /* module type */
+ NULL, /* init master */
+ NULL, /* init module */
+ NULL, /* init process */
+ NULL, /* init thread */
+ NULL, /* exit thread */
+ NULL, /* exit process */
+ NULL, /* exit master */
+ NGX_MODULE_V1_PADDING
+};
+
+
+static void *
+ngx_openssl_cache_create_conf(ngx_cycle_t *cycle)
+{
+ ngx_ssl_cache_t *cache;
+ ngx_pool_cleanup_t *cln;
+
+ cache = ngx_pcalloc(cycle->pool, sizeof(ngx_ssl_cache_t));
+ if (cache == NULL) {
+ return NULL;
+ }
+
+ cln = ngx_pool_cleanup_add(cycle->pool, 0);
+ if (cln == NULL) {
+ return NULL;
+ }
+
+ cln->handler = ngx_ssl_cache_cleanup;
+ cln->data = cache;
+
+ ngx_rbtree_init(&cache->rbtree, &cache->sentinel,
+ ngx_ssl_cache_node_insert);
+
+ return cache;
+}
+
+
+static void
+ngx_ssl_cache_cleanup(void *data)
+{
+ ngx_ssl_cache_t *cache = data;
+
+ ngx_rbtree_t *tree;
+ ngx_rbtree_node_t *node;
+ ngx_ssl_cache_node_t *cn;
+
+ tree = &cache->rbtree;
+
+ if (tree->root == tree->sentinel) {
+ return;
+ }
+
+ for (node = ngx_rbtree_min(tree->root, tree->sentinel);
+ node;
+ node = ngx_rbtree_next(tree, node))
+ {
+ cn = ngx_rbtree_data(node, ngx_ssl_cache_node_t, node);
+
+ if (cn->type != NULL && cn->value != NULL) {
+ cn->type->free(cn->value);
+ }
+ }
+}
+
+
+static void
+ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp,
+ ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel)
+{
+ ngx_rbtree_node_t **p;
+ ngx_ssl_cache_node_t *n, *t;
+
+ for ( ;; ) {
+
+ n = ngx_rbtree_data(node, ngx_ssl_cache_node_t, node);
+ t = ngx_rbtree_data(temp, ngx_ssl_cache_node_t, node);
+
+ if (node->key != temp->key) {
+
+ p = (node->key < temp->key) ? &temp->left : &temp->right;
+
+ } else if (n->type != t->type) {
+
+ p = (n->type < t->type) ? &temp->left : &temp->right;
+
+ } else {
+
+ p = (ngx_memn2cmp(n->id.data, t->id.data, n->id.len, t->id.len)
+ < 0) ? &temp->left : &temp->right;
+ }
+
+ if (*p == sentinel) {
+ break;
+ }
+
+ temp = *p;
+ }
+
+ *p = node;
+ node->parent = temp;
+ node->left = sentinel;
+ node->right = sentinel;
+ ngx_rbt_red(node);
+}
+
+
+void *
+ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
+ ngx_ssl_cache_type_t *type, char **err, ngx_str_t *id, void *data)
+{
+ void *value;
+ uint32_t hash;
+ ngx_ssl_cache_t *cache;
+ ngx_ssl_cache_node_t *cn;
+
+ value = NULL;
+
+ hash = ngx_murmur_hash2(id->data, id->len);
+
+ cache = (ngx_ssl_cache_t *) ngx_get_conf(cycle->conf_ctx,
+ ngx_openssl_cache_module);
+
+ if (ngx_process == NGX_PROCESS_WORKER
+ || ngx_process == NGX_PROCESS_SINGLE)
+ {
+ return type->create(id, err, data);
+ }
This check kept bothering me, so I figured we can try to improve it.
The logic for configuration time and per-connection fetches is quite
different and we're always aware of the context we're working in. It
would be cleaner to have separate fetch implementations, like that:
void *
ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_ssl_cache_type_t *type,
char **err,
ngx_str_t *id, void *data)
{
...
}
void *
ngx_ssl_cache_connection_fetch(ngx_ssl_cache_type_t *type, char **err,
ngx_str_t *id, void *data)
{
return type->create(id, err, data);
}
Full diff is attached, but it needs to be properly split between patches
2-6.
As a bonus, the change makes it easier to isolate any future logic for
dynamic certificate caching. It was noticeable on our review of early
prototypes how combining both code paths in the `ngx_ssl_cache_fetch`
code made the function unnecessarily complicated.
+
+ cn = ngx_ssl_cache_lookup(cache, type, id, hash);
+
+ if (cn == NULL) {
+ cn = ngx_palloc(pool, sizeof(ngx_ssl_cache_node_t) + id->len + 1);
+ if (cn == NULL) {
+ return NULL;
+ }
+
+ cn->node.key = hash;
+ cn->id.data = (u_char *)(cn + 1);
+ cn->id.len = id->len;
+ cn->type = type;
+ cn->value = NULL;
+
+ ngx_cpystrn(cn->id.data, id->data, id->len + 1);
+
+ ngx_rbtree_insert(&cache->rbtree, &cn->node);
+ }
+
+ /* try to use a reference from the cache */
+
+ if (cn->value != NULL) {
+ value = type->ref(err, cn->value);
+ }
+
+ if (value == NULL) {
+ value = type->create(id, err, data);
+ }
+
+ if (value != NULL && cn->value == NULL) {
+ /* we have a value and the node needs one; try to reference it */
+ cn->value = type->ref(err, value);
+ }
+
+ return value;
+}
+
+
+static ngx_ssl_cache_node_t *
+ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type,
+ ngx_str_t *id, uint32_t hash)
+{
+ ngx_int_t rc;
+ ngx_rbtree_node_t *node, *sentinel;
+ ngx_ssl_cache_node_t *cn;
+
+ node = cache->rbtree.root;
+ sentinel = cache->rbtree.sentinel;
+
+ while (node != sentinel) {
+
+ if (hash < node->key) {
+ node = node->left;
+ continue;
+ }
+
+ if (hash > node->key) {
+ node = node->right;
+ continue;
+ }
+
+ /* hash == node->key */
+
+ cn = (ngx_ssl_cache_node_t *) node;
+
+ if ((ngx_uint_t) type < (ngx_uint_t) cn->type) {
+ node = node->left;
+ continue;
+ }
+
+ if ((ngx_uint_t) type > (ngx_uint_t) cn->type) {
+ node = node->right;
+ continue;
+ }
+
+ /* type == cn->type */
+
+ rc = ngx_memn2cmp(id->data, cn->id.data, id->len, cn->id.len);
+
+ if (rc == 0) {
+ return cn;
+ }
+
+ node = (rc < 0) ? node->left : node->right;
+ }
+
+ return NULL;
+}
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel
diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index e3f9c7bcc..ed8c8feff 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -439,8 +439,7 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl,
ngx_str_t *cert,
STACK_OF(X509) *chain;
ngx_ssl_name_t *name;
- chain = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_cert,
- &err, cert, NULL);
+ chain = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_cert, &err, cert, NULL);
if (chain == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
@@ -534,8 +533,7 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl,
ngx_str_t *cert,
}
#endif
- pkey = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_key, &err,
- key, passwords);
+ pkey = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_key, &err, key, passwords);
if (pkey == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
@@ -568,8 +566,8 @@ ngx_ssl_connection_certificate(ngx_connection_t *c,
ngx_pool_t *pool,
EVP_PKEY *pkey;
STACK_OF(X509) *chain;
- chain = ngx_ssl_cache_fetch((ngx_cycle_t *) ngx_cycle, c->pool,
- &ngx_ssl_cache_cert, &err, cert, NULL);
+ chain = ngx_ssl_cache_connection_fetch(&ngx_ssl_cache_cert, &err, cert,
+ NULL);
if (chain == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
@@ -609,8 +607,8 @@ ngx_ssl_connection_certificate(ngx_connection_t *c,
ngx_pool_t *pool,
#endif
- pkey = ngx_ssl_cache_fetch((ngx_cycle_t *) ngx_cycle, c->pool,
- &ngx_ssl_cache_key, &err, key, passwords);
+ pkey = ngx_ssl_cache_connection_fetch(&ngx_ssl_cache_key, &err, key,
+ passwords);
if (pkey == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, c->log, 0,
@@ -686,8 +684,7 @@ ngx_ssl_client_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl,
ngx_str_t *cert,
}
xs = SSL_CTX_get_cert_store(ssl->ctx);
- xsk = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_ca, &err,
- cert, NULL);
+ xsk = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_ca, &err, cert, NULL);
if (xsk == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
@@ -792,8 +789,7 @@ ngx_ssl_trusted_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl,
ngx_str_t *cert,
}
xs = SSL_CTX_get_cert_store(ssl->ctx);
- xsk = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_ca, &err,
- cert, NULL);
+ xsk = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_ca, &err, cert, NULL);
if (xsk == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
@@ -860,8 +856,7 @@ ngx_ssl_crl(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *crl)
return NGX_ERROR;
}
- xcsk = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_crl,
- &err, crl, NULL);
+ xcsk = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_crl, &err, crl, NULL);
if (xcsk == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index 490f995b9..7bf79a017 100644
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -234,8 +234,10 @@ ngx_int_t ngx_ssl_ocsp_get_status(ngx_connection_t *c,
const char **s);
void ngx_ssl_ocsp_cleanup(ngx_connection_t *c);
ngx_int_t ngx_ssl_ocsp_cache_init(ngx_shm_zone_t *shm_zone, void *data);
-void *ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
- ngx_ssl_cache_type_t *type, char **err, ngx_str_t *id, void *data);
+void *ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_ssl_cache_type_t *type,
+ char **err, ngx_str_t *id, void *data);
+void *ngx_ssl_cache_connection_fetch(ngx_ssl_cache_type_t *type, char **err,
+ ngx_str_t *id, void *data);
ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file);
ngx_array_t *ngx_ssl_preserve_passwords(ngx_conf_t *cf,
diff --git a/src/event/ngx_event_openssl_cache.c
b/src/event/ngx_event_openssl_cache.c
index 7e207c916..64eae9c9c 100644
--- a/src/event/ngx_event_openssl_cache.c
+++ b/src/event/ngx_event_openssl_cache.c
@@ -219,8 +219,8 @@ ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp,
void *
-ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
- ngx_ssl_cache_type_t *type, char **err, ngx_str_t *id, void *data)
+ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_ssl_cache_type_t *type, char **err,
+ ngx_str_t *id, void *data)
{
void *value;
uint32_t hash;
@@ -231,19 +231,13 @@ ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
hash = ngx_murmur_hash2(id->data, id->len);
- cache = (ngx_ssl_cache_t *) ngx_get_conf(cycle->conf_ctx,
+ cache = (ngx_ssl_cache_t *) ngx_get_conf(cf->cycle->conf_ctx,
ngx_openssl_cache_module);
- if (ngx_process == NGX_PROCESS_WORKER
- || ngx_process == NGX_PROCESS_SINGLE)
- {
- return type->create(id, err, data);
- }
-
cn = ngx_ssl_cache_lookup(cache, type, id, hash);
if (cn == NULL) {
- cn = ngx_palloc(pool, sizeof(ngx_ssl_cache_node_t) + id->len + 1);
+ cn = ngx_palloc(cf->pool, sizeof(ngx_ssl_cache_node_t) + id->len + 1);
if (cn == NULL) {
return NULL;
}
@@ -278,6 +272,14 @@ ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
}
+void *
+ngx_ssl_cache_connection_fetch(ngx_ssl_cache_type_t *type, char **err,
+ ngx_str_t *id, void *data)
+{
+ return type->create(id, err, data);
+}
+
+
static ngx_ssl_cache_node_t *
ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type,
ngx_str_t *id, uint32_t hash)
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel
