Hello everyone,
This patch implements a new backend directive to control hash-based load
balancing when servers are at the 'maxconn' limit or have a full queue. See
https://github.com/haproxy/haproxy/issues/2893 for context.
Thank you!
>From e518ee79a56319c9781e62005da13f2c0064399b Mon Sep 17 00:00:00 2001
From: psavalle <psaval...@protonmail.com>
Date: Fri, 21 Mar 2025 11:27:21 +0100
Subject: [PATCH] MEDIUM: lb-chash: add directive hash-preserve-affinity
When using hash-based load balancing, requests are always assigned to the
server corresponding
to the hash bucket for the balancing key, without taking maxconn or maxqueue
into account, unlike
in other load balancing methods like 'first'. This adds a new backend directive
that can be used
to take maxconn and possibly maxqueue in that context. This can be used when
hashing is desired
to achieve cache locality, but sending requests to a different server is
preferable to queuing
for a long time or failing requests when the initial server is saturated.
By default, affinity is preserved as was the case previously. When
'hash-preserve-affinity' is
set to 'maxqueue', servers are considered successively in the order of the hash
ring until a
server that does not have a full queue is found.
When 'maxconn' is set on a server, queueing cannot be disabled, as 'maxqueue=0'
means unlimited.
To support picking a different server when a server is at 'maxconn'
irrespective of the queue,
'hash-preserve-affinity' can be set to 'maxconn'.
---
doc/configuration.txt | 31 +++++++++-
include/haproxy/proxy-t.h | 8 ++-
reg-tests/balance/balance-hash-maxconn.vtc | 52 ++++++++++++++++
reg-tests/balance/balance-hash-maxqueue.vtc | 68 +++++++++++++++++++++
src/cfgparse-listen.c | 28 +++++++++
src/lb_chash.c | 13 +++-
tests/conf/test-hash-preseve-affinity.cfg | 52 ++++++++++++++++
7 files changed, 249 insertions(+), 3 deletions(-)
create mode 100644 reg-tests/balance/balance-hash-maxconn.vtc
create mode 100644 reg-tests/balance/balance-hash-maxqueue.vtc
create mode 100644 tests/conf/test-hash-preseve-affinity.cfg
diff --git a/doc/configuration.txt b/doc/configuration.txt
index 8eb8db06f..b161419ac 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -7950,8 +7950,37 @@ hash-type <method> <function> <modifier>
default function is "sdbm", the selection of a function should be based on
the range of the values being hashed.
- See also : "balance", "hash-balance-factor", "server"
+ See also : "balance", "hash-balance-factor", "hash-preserve-affinity",
"server"
+hash-preserve-affinity { always | maxconn | maxqueue }
+ Specify a method for assigning streams to servers with hash load balancing
when
+ servers are satured or have a full queue.
+
+ May be used in the following contexts: http
+
+ May be used in sections: defaults | frontend | listen | backend
+ yes | no | yes | yes
+
+ The following values can be specified:
+
+ - "always" : this is the default stategy. A stream is assigned to a
server
+ based on hashing irrespective of whether the server is
currently
+ saturated.
+
+ - "maxconn" : when selected, servers that have "maxconn" set and are
currently
+ saturated will be skipped. Another server will be picked by
+ following the hashing ring. This has no effect on servers
that do
+ not set "maxconn". If all servers are saturated, the
request is
+ enqueued to the last server in the hash ring before the
initially
+ selected server.
+
+ - "maxqueue" : when selected, servers that have "maxconn" set, "maxqueue"
set
+ to a non-zero value (limited queue size) and currently have
a
+ full queue will be skipped. Another server will be picked by
+ following the hashing ring. This has no effect on servers
that
+ do not set both "maxconn" and "maxqueue".
+
+ See also : "maxconn", "maxqueue", "hash-balance-factor"
http-after-response <action> <options...> [ { if | unless } <condition> ]
Access control for all Layer 7 responses (server, applet/service and internal
diff --git a/include/haproxy/proxy-t.h b/include/haproxy/proxy-t.h
index 9762ab166..5ad420129 100644
--- a/include/haproxy/proxy-t.h
+++ b/include/haproxy/proxy-t.h
@@ -180,7 +180,13 @@ enum PR_SRV_STATE_FILE {
#define PR_O3_LOGF_HOST_APPEND 0x00000080
#define PR_O3_LOGF_HOST 0x000000F0
-/* unused: 0x00000100 to 0x80000000 */
+/* bits for hash-preserve-affinity */
+#define PR_O3_HASHAFNTY_ALWS 0x00000000 /* always preserve hash affinity */
+#define PR_O3_HASHAFNTY_MAXCONN 0x00000100 /* preserve hash affinity until
maxconn is reached */
+#define PR_O3_HASHAFNTY_MAXQUEUE 0x00000200 /* preserve hash affinity until
maxqueue is reached */
+#define PR_O3_HASHAFNTY_MASK 0x00000300 /* mask for hash-preserve-affinity
*/
+
+/* unused: 0x00000400 to 0x80000000 */
/* end of proxy->options3 */
/* Cookie settings for pr->ck_opts */
diff --git a/reg-tests/balance/balance-hash-maxconn.vtc
b/reg-tests/balance/balance-hash-maxconn.vtc
new file mode 100644
index 000000000..ae3cd5377
--- /dev/null
+++ b/reg-tests/balance/balance-hash-maxconn.vtc
@@ -0,0 +1,52 @@
+vtest "Test for balance URI with hash-preserve-affinity maxconn"
+feature ignore_unknown_macro
+
+server s0 {
+ rxreq
+ delay 0.5
+ txresp -hdr "Server: s0"
+} -start
+
+server s1 {
+ rxreq
+ delay 0.5
+ txresp -hdr "Server: s1"
+} -start
+
+haproxy h1 -arg "-L A" -conf {
+ defaults
+ mode http
+ timeout server "${HAPROXY_TEST_TIMEOUT-5s}"
+ timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
+ timeout client "${HAPROXY_TEST_TIMEOUT-5s}"
+
+ listen px
+ bind "fd@${px}"
+ balance uri
+ hash-preserve-affinity maxconn
+ hash-type consistent
+
+ server srv0 ${s0_addr}:${s0_port} maxconn 1
+ server srv1 ${s1_addr}:${s1_port} maxconn 1
+
+} -start
+
+client c1 -connect ${h1_px_sock} {
+ txreq -url "/test-url"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.Server ~ s1
+} -start
+
+delay 0.1
+
+# s1 is saturated, request should be assigned to s0
+client c2 -connect ${h1_px_sock} {
+ txreq -url "/test-url"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.Server ~ s0
+} -start
+
+client c1 -wait
+client c2 -wait
diff --git a/reg-tests/balance/balance-hash-maxqueue.vtc
b/reg-tests/balance/balance-hash-maxqueue.vtc
new file mode 100644
index 000000000..147eebe5d
--- /dev/null
+++ b/reg-tests/balance/balance-hash-maxqueue.vtc
@@ -0,0 +1,68 @@
+vtest "Test for balance URI with hash-preserve-affinity maxqueue"
+feature ignore_unknown_macro
+
+server s0 {
+ rxreq
+ delay 0.5
+ txresp -hdr "Server: s0"
+} -repeat 2 -start
+
+server s1 {
+ rxreq
+ delay 0.5
+ txresp -hdr "Server: s1"
+} -repeat 2 -start
+
+haproxy h1 -arg "-L A" -conf {
+ defaults
+ mode http
+ timeout server "${HAPROXY_TEST_TIMEOUT-5s}"
+ timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
+ timeout client "${HAPROXY_TEST_TIMEOUT-5s}"
+
+ listen px
+ bind "fd@${px}"
+ balance uri
+ hash-preserve-affinity maxqueue
+ hash-type consistent
+
+ server srv0 ${s0_addr}:${s0_port} maxconn 1 maxqueue 1
+ server srv1 ${s1_addr}:${s1_port} maxconn 1 maxqueue 1
+
+} -start
+
+client c1 -connect ${h1_px_sock} {
+ txreq -url "/test-url"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.Server ~ s1
+} -start
+
+client c1b -connect ${h1_px_sock} {
+ txreq -url "/test-url"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.Server ~ s1
+} -start
+
+delay 0.1
+
+# s1 is saturated, requests should be assigned to s0
+client c2 -connect ${h1_px_sock} {
+ txreq -url "/test-url"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.Server ~ s0
+} -start
+
+client c2b -connect ${h1_px_sock} {
+ txreq -url "/test-url"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.Server ~ s0
+} -start
+
+client c1 -wait
+client c1b -wait
+client c2 -wait
+client c2b -wait
diff --git a/src/cfgparse-listen.c b/src/cfgparse-listen.c
index 2dd2a4897..a9f445508 100644
--- a/src/cfgparse-listen.c
+++ b/src/cfgparse-listen.c
@@ -2533,6 +2533,34 @@ int cfg_parse_listen(const char *file, int linenum, char
**args, int kwm)
goto out;
}
}
+ else if (strcmp(args[0], "hash-preserve-affinity") == 0) {
+ if (warnifnotcap(curproxy, PR_CAP_BE, file, linenum, args[0],
NULL))
+ err_code |= ERR_WARN;
+
+ if (strcmp(args[1], "always") == 0) {
+ curproxy->options3 &= ~PR_O3_HASHAFNTY_MASK;
+ curproxy->options3 |= PR_O3_HASHAFNTY_ALWS;
+ if (alertif_too_many_args_idx(0, 1, file, linenum,
args, &err_code))
+ goto out;
+ }
+ else if (strcmp(args[1], "maxconn") == 0) {
+ curproxy->options3 &= ~PR_O3_HASHAFNTY_MASK;
+ curproxy->options3 |= PR_O3_HASHAFNTY_MAXCONN;
+ if (alertif_too_many_args_idx(0, 1, file, linenum,
args, &err_code))
+ goto out;
+ }
+ else if (strcmp(args[1], "maxqueue") == 0) {
+ curproxy->options3 &= ~PR_O3_HASHAFNTY_MASK;
+ curproxy->options3 |= PR_O3_HASHAFNTY_MAXQUEUE;
+ if (alertif_too_many_args_idx(0, 1, file, linenum,
args, &err_code))
+ goto out;
+ }
+ else {
+ ha_alert("parsing [%s:%d] : '%s' only supports
'always', 'maxconn', 'maxqueue'.\n", file, linenum, args[0]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+ }
else if (strcmp(args[0], "monitor") == 0) {
if (curproxy->cap & PR_CAP_DEF) {
ha_alert("parsing [%s:%d] : '%s' not allowed in
'defaults' section.\n", file, linenum, args[0]);
diff --git a/src/lb_chash.c b/src/lb_chash.c
index 784a27af1..f5b075ab3 100644
--- a/src/lb_chash.c
+++ b/src/lb_chash.c
@@ -404,6 +404,7 @@ struct server *chash_get_server_hash(struct proxy *p,
unsigned int hash, const s
struct eb_root *root;
unsigned int dn, dp;
int loop;
+ int hashafnty;
HA_RWLOCK_RDLOCK(LBPRM_LOCK, &p->lbprm.lock);
@@ -449,7 +450,17 @@ struct server *chash_get_server_hash(struct proxy *p,
unsigned int hash, const s
}
loop = 0;
- while (nsrv == avoid || (p->lbprm.hash_balance_factor &&
!chash_server_is_eligible(nsrv))) {
+ hashafnty = p->options3 & PR_O3_HASHAFNTY_MASK;
+
+ while (nsrv == avoid ||
+ (p->lbprm.hash_balance_factor &&
!chash_server_is_eligible(nsrv)) ||
+ (hashafnty == PR_O3_HASHAFNTY_MAXCONN &&
+ nsrv->maxconn &&
+ nsrv->served >= srv_dynamic_maxconn(nsrv)) ||
+ (hashafnty == PR_O3_HASHAFNTY_MAXQUEUE &&
+ nsrv->maxconn &&
+ nsrv->maxqueue &&
+ nsrv->served + nsrv->queueslength >=
srv_dynamic_maxconn(nsrv) + nsrv->maxqueue)) {
next = eb32_next(next);
if (!next) {
next = eb32_first(root);
diff --git a/tests/conf/test-hash-preseve-affinity.cfg
b/tests/conf/test-hash-preseve-affinity.cfg
new file mode 100644
index 000000000..1aa0e2c80
--- /dev/null
+++ b/tests/conf/test-hash-preseve-affinity.cfg
@@ -0,0 +1,52 @@
+# This is a test configuration for "hash-preserve-affinity" parameter
+global
+ log 127.0.0.1 local0
+
+defaults
+ mode http
+ timeout client 10s
+ timeout server 10s
+ timeout connect 10s
+
+listen vip1
+ log global
+ option httplog
+ bind :8001
+ mode http
+ maxconn 100
+ balance url_param foo
+ server srv1 127.0.0.1:80
+ server srv2 127.0.0.1:80
+
+listen vip2
+ log global
+ option httplog
+ bind :8002
+ mode http
+ maxconn 100
+ balance url_param foo check_post
+ server srv1 127.0.0.1:80
+ server srv2 127.0.0.1:80
+ hash-preserve-affinity always
+
+listen vip3
+ log global
+ option httplog
+ bind :8003
+ mode http
+ maxconn 100
+ balance url_param foo check_post
+ server srv1 127.0.0.1:80
+ server srv2 127.0.0.1:80
+ hash-preserve-affinity maxconn
+
+listen vip4
+ log global
+ option httplog
+ bind :8004
+ mode http
+ maxconn 100
+ balance url_param foo check_post
+ server srv1 127.0.0.1:80
+ server srv2 127.0.0.1:80
+ hash-preserve-affinity maxqueue
--
2.39.5 (Apple Git-154)
