I have built test implementation 2 machine pf/carp cluster and I am
receiving odd (or so I think) results with pfsync.
carp1 is wan, carp2 is LAN.
Both of these firewalls are connected back to back using a crossover cable
on em0.
When I "ifconfig carp2 down" the backup machine assumes the master position
(which is expected).
While monitoring pf states using pftop, it looks like the state tables are
identical for tcp sessions (expected).
However, when I failover to the backup machine the tcp sessions for
workstations on the 192.168.10.0/24 network
get killed (not expected).
Using netstat I can see the <n> failed state lookup/inserts counter go up.
What I am using to test:
A workstation behind these firewalls with the default gateway of
192.168.10.5 (internal carp address) browses to
a website that requires authentication, logins in and browses pages.
Then failover the master , then browse to the same pages, which redirects
to the login page.
Is this expected behavior?
I do believe this is a scenario which pfsync should allow a transparent
failover without killing off tcp sessions?
But I could be wrong :)
Any help would be greatly appreciated.
Machine 1:
-------------------------------------------------------------------------
ifconfig:
xl0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:50:04:a4:1b:20
media: Ethernet autoselect (10baseT half-duplex)
status: active
inet 192.168.10.2 netmask 0xffffff00 broadcast 192.168.10.255
inet6 fe80::250:4ff:fea4:1b20%xl0 prefixlen 64 scopeid 0x1
dc0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:04:5a:8d:eb:21
groups: egress
media: Ethernet autoselect (10baseT half-duplex)
status: active
inet 172.16.10.173 netmask 0xffffff00 broadcast 172.16.10.255
inet6 fe80::204:5aff:fe8d:eb21%dc0 prefixlen 64 scopeid 0x2
em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:08:74:1f:16:24
media: Ethernet autoselect (1000baseT
full-duplex,master,rxpause,txpause)
status: active
inet 10.10.10.2 netmask 0xffffff00 broadcast 10.10.10.255
inet6 fe80::208:74ff:fe1f:1624%em0 prefixlen 64 scopeid 0x3
.....
pfsync0: flags=41<UP,RUNNING> mtu 1460
pfsync: syncdev: em0 syncpeer: 10.10.10.3 maxupd: 128
groups: carp pfsync
pflog0: flags=141<UP,RUNNING,PROMISC> mtu 33208
groups: pflog
carp1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:00:5e:00:01:01
carp: MASTER carpdev dc0 vhid 1 advbase 1 advskew 10
groups: carp
inet6 fe80::200:5eff:fe00:101%carp1 prefixlen 64 scopeid 0x6
inet 172.16.10.175 netmask 0xffffff00 broadcast 172.16.10.255
carp2: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:00:5e:00:01:02
carp: MASTER carpdev xl0 vhid 2 advbase 1 advskew 10
groups: carp
inet6 fe80::200:5eff:fe00:102%carp2 prefixlen 64 scopeid 0x7
inet 192.168.10.5 netmask 0xffffff00 broadcast 192.168.10.255
/etc/pf.conf
#external interfaces
ext_if="dc0"
carp_ext="carp1"
#internal interfaces
int_if="xl0"
sync_if="em0"
carp_int="carp2"
icmp_types="echoreq"
set block-policy return
set loginterface $ext_if
set skip on lo
scrub in
nat on $ext_if from !($ext_if) -> ($ext_if)
#filters
block in log
pass out keep state
antispoof quick for { lo $int_if }
pass quick on $sync_if proto pfsync keep state (no-sync)
pass on { $int_if, $ext_if } proto carp keep state (no-sync)
#allow ssh to itchy/scratchy/krusty
pass in on $ext_if inet proto tcp from any to {$ext_if $carp_ext} \
port 22 keep state
#allow ping to itchy/scratchy/krusty
pass in inet proto icmp all icmp-type $icmp_types keep state
pass in quick on $int_if
netstat -sp pfsync
pfsync:
6355 packets received (IPv4)
0 packets received (IPv6)
0 packets discarded for bad interface
0 packets discarded for bad ttl
0 packets shorter than header
0 packets discarded for bad version
0 packets discarded for bad HMAC
0 packets discarded for bad action
0 packets discarded for short packet
0 states discarded for bad values
4 stale states
2688 failed state lookup/inserts
6509 packets sent (IPv4)
0 packets sent (IPv6)
0 send failed due to mbuf memory error
0 send error
Machine 2:
---------------------------------------------------------------------------
ifconfig:
xl0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:50:04:a4:95:3b
groups: egress
media: Ethernet autoselect (10baseT half-duplex)
status: active
inet 172.16.10.172 netmask 0xffffff00 broadcast 172.16.10.255
inet6 fe80::250:4ff:fea4:953b%xl0 prefixlen 64 scopeid 0x1
xl1: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:10:4b:6b:64:86
media: Ethernet autoselect (10baseT half-duplex)
status: active
inet 192.168.10.3 netmask 0xffffff00 broadcast 192.168.10.255
inet6 fe80::210:4bff:fe6b:6486%xl1 prefixlen 64 scopeid 0x2
em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:08:74:33:d1:a4
media: Ethernet autoselect (1000baseT full-duplex,rxpause,txpause)
status: active
inet 10.10.10.3 netmask 0xffffff00 broadcast 10.10.10.255
inet6 fe80::208:74ff:fe33:d1a4%em0 prefixlen 64 scopeid 0x3
.....
pfsync0: flags=41<UP,RUNNING> mtu 1460
pfsync: syncdev: em0 syncpeer: 10.10.10.2 maxupd: 128
groups: carp pfsync
pflog0: flags=141<UP,RUNNING,PROMISC> mtu 33208
groups: pflog
carp1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:00:5e:00:01:01
carp: BACKUP carpdev xl0 vhid 1 advbase 1 advskew 100
groups: carp
inet6 fe80::200:5eff:fe00:101%carp1 prefixlen 64 scopeid 0x6
inet 172.16.10.175 netmask 0xffffff00 broadcast 172.16.10.255
carp2: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:00:5e:00:01:02
carp: BACKUP carpdev xl1 vhid 2 advbase 1 advskew 100
groups: carp
inet6 fe80::200:5eff:fe00:102%carp2 prefixlen 64 scopeid 0x7
inet 192.168.10.5 netmask 0xffffff00 broadcast 192.168.10.255
/etc/pf.conf:
#external interfaces
ext_if="xl0"
carp_ext="carp1"
#internal lan
int_if="xl1"
carp_int="carp2"
sync_if="em0"
icmp_types="echoreq"
set block-policy return
set loginterface $ext_if
set skip on lo
scrub in
nat on $ext_if from !($ext_if) -> ($ext_if)
#filters
block in log
pass out keep state
antispoof quick for { lo $int_if }
pass quick on $sync_if proto pfsync keep state (no-sync)
pass on { $int_if, $ext_if } proto carp keep state (no-sync)
#allow ssh to itchy/scratchy/krusty
pass in on $ext_if inet proto tcp from any to {$ext_if $carp_ext} \
port 22 keep state
#allow ping to itchy/scratchy/krusty
pass in inet proto icmp all icmp-type $icmp_types keep state
pass in quick on $int_if
netstat -sp pfsync
pfsync:
6618 packets received (IPv4)
0 packets received (IPv6)
0 packets discarded for bad interface
0 packets discarded for bad ttl
0 packets shorter than header
0 packets discarded for bad version
0 packets discarded for bad HMAC
0 packets discarded for bad action
0 packets discarded for short packet
0 states discarded for bad values
4 stale states
7717 failed state lookup/inserts
6526 packets sent (IPv4)
0 packets sent (IPv6)
0 send failed due to mbuf memory error
0 send error
