We have 3x Supermicro Intel Dual Xeon E5-2620v3 powered systems with 32GB
ECC
memory, 4x 10 Gigabit Ethernet NICs (Intel X520-DA2), and 2x Gigabit
Ethernet
onboard NICs connected towards a Virtual Chassis of a Juniper EX 4550
Ethernet
Switch, running OpenBSD 5.8 with all (11) patches.
We want to use these 3 systems as loadbalancers, 2x 10GE (trunk0, LACP)
inbound,
2x 10GE (trunk1, LACP) outbound, 2x 1GE (trunk2, LACP) for Pfsync.
LB-1 shares a public IP with LB-2, and LB-2 and LB-3 do the same (via
CARP). We
use relayd for Loadbalancing the traffic towards 3 backend servers, all
they
currently do is serving a HTTP 200 OK response.
When we load tested one LB's HTTP performance alone with wrk - we get about
40k
req/s when testing with one machine in the same network as a client, and
more
than 100k req/s when testing with 3 client machines. Doing the test with
HTTPS
brings the performance down to 1400 req/s, and it does not matter if using
more
or less clients, the total number of req/s stays almost the same.
The overall load of the systems is low (below 2-3), memory utilization is
low as well.
As we don't have experience with OpenBSD and relayd we can only compare
these
numbers to FreeBSD and HAproxy, which we used in our previous setup. Our
configuration files are listed below - we would be happy about any comment
how
to improve the HTTPS performance.
################################################################################
# OpenBSD sysctl.conf
net.inet.carp.preempt=1
### Tried with and without the following settings - with some effect
kern.bufcachepercent=90
kern.maxfiles=200000
kern.maxproc=50000
kern.maxclusters=32768
machdep.allowaperture=2
net.inet.ip.forwarding=1
net.inet.ip.ifq.maxlen=8192
net.inet.ip.mtudisc=0
net.inet.tcp.rfc3390=1
net.inet.tcp.mssdflt=1440
################################################################################
# OpenBSD relayd.conf
ip4_244 = "xx.xx.xx.244"
ip4_245 = "xx.xx.xx.245"
tracker5 = "10.5.3.34"
tracker6 = "10.5.3.42"
tracker7 = "10.5.3.50"
interval 10
table <trackers> { $tracker5, $tracker6, $tracker7 }
prefork 12
http protocol https {
### TCP performance options
tcp { nodelay, sack, socket buffer 65536, backlog 128 }
match request header append "X-Forwarded-For" value "$REMOTE_ADDR"
match request header append "X-Forwarded-By" \
value "$SERVER_ADDR:$SERVER_PORT"
match request header set "Connection" value "close"
tls { no tlsv1.0, ciphers HIGH }
tls session cache disable # tried enabling/disabling -> no effect
}
relay wwwssl {
listen on $ip4_244 port 443 tls
listen on $ip4_245 port 443 tls
protocol "https"
forward to <trackers> port 8083 mode loadbalance check tcp
}
relay www {
listen on $ip4_244 port 80
listen on $ip4_245 port 80
forward to <trackers> port 8083 mode loadbalance check tcp
}
################################################################################
# OpenBSD: pf.conf
tcp_services = "{ domain, www, https }"
udp_services = "{ domain }"
tcp_public_services = "{ www, https }"
icmp_types = "{ echorep, echoreq, unreach}"
icmp6_types = "{ echorep, echoreq, unreach, timex, paramprob, routersol,
routeradv, neighbrsol, neighbradv, redir }"
pfsync_int = trunk2 # Pfsync interface
int_if = trunk1 # DMZ (internal) interface
ext_if = trunk0 # External CARP interface
# Increase limits
set limit { states 100000, src-nodes 100000, table-entries 2000000 }
# Optimizations
set optimization aggressive
set timeout { adaptive.end 120000, interval 2, tcp.tsdiff 5, tcp.first 5,
tcp.closing 5, tcp.closed 5, tcp.finwait 5, tcp.established 4200} # tried
with
# and without - very small effect
# See pf.conf(5) and /etc/examples/pf.conf
anchor "relayd/*"
set reassemble yes
set block-policy drop
set loginterface $ext_if
set skip on lo
set skip on $int_if
set skip on $pfsync_int
# Scrub incoming
match in all scrub (no-df max-mss 1440)
# Block everything by default
block all
# Activate spoofing protection
block in quick from urpf-failed
# Allow main service of this host
pass out proto tcp to port $tcp_services keep state
pass in proto tcp to port $tcp_public_services keep state
pass proto udp to port $udp_services keep state
# Pass CARP and pfsync
pass proto carp keep state (no-sync)
pass quick proto pfsync keep state (no-sync)
# SSH backup channel from Wooga office
pass in on trunk0 inet proto tcp from 185.74.12.0/22 to any port 22 keep
state (no-sync)
# Allow pings for Pingdom status checks
pass on trunk0 inet proto icmp icmp-type $icmp_types keep state (no-sync)
pass on trunk0 inet6 proto icmp6 icmp6-type $icmp6_types keep state
(no-sync)
