Hi,
03.04.2011 22:42, ruslan usifov wrote:
> You need some tuning from both sides.
> First, (at least some versions of) ietd needs to be blocked (-j DROP)
> with iptables on restarts. That means, you should block all incoming and
> outgoing packets (later is more important) before ietd stop and unblock
> all after it starts. I use home-brew stateful RA for this, which blocks
> (DROP) all traffic to/from VIP in slave mode and passes it to a later
> decision (no -j) in master mode.
>
>
>
> Thanks for reply, But how to implemet this from pacemaker. Or i must to
> modify inet.d scripts?
You can try attached.
This was a first trial to write RA... It may contain some wrong logic,
especially with scores, but it works for me.
It is intended to be colocated with IP-address resource:
colocation col1 inf: FW:Master ip
order ord1 inf: ip:start FW:promote
Vladislav
#!/bin/bash
#
#
# OCF Resource Agent compliant script
#
# Copyright (c) 2010 Vitalativ S.R.O.,
# Copyright (c) 2010 Vladislav Bogdanov
#
# All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# Further, this software is distributed without any warranty that it is
# free of the rightful claim of any third person regarding infringement
# or the like. Any license provided herein, whether implied or
# otherwise, applies only to this software file. Patent licenses, if
# any, provided herein do not apply to combinations of this program with
# other software, or any other product whatsoever.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write the Free Software Foundation,
# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
#
#
# OCF instance parameters
# OCF_RESKEY_vip
# OCF_RESKEY_CRM_meta_clone_max
# OCF_RESKEY_CRM_meta_clone_node_max
# OCF_RESKEY_CRM_meta_master_max
# OCF_RESKEY_CRM_meta_master_node_max
#######################################################################
# Initialization:
: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
export LANG=C LANGUAGE=C LC_ALL=C
# Defaults according to "Configuration 1.0 Explained",
# "Multi-state resource configuration options"
: ${OCF_RESKEY_CRM_meta_clone_node_max=1}
: ${OCF_RESKEY_CRM_meta_master_max=1}
: ${OCF_RESKEY_CRM_meta_master_node_max=1}
OCF_RESKEY_allow_action_default="accept"
: ${OCF_RESKEY_allow_action=${OCF_RESKEY_allow_action_default}}
OCF_RESKEY_deny_action_default="drop"
: ${OCF_RESKEY_deny_action=${OCF_RESKEY_deny_action_default}}
#######################################################################
# for debugging this RA
DEBUG_LOG_DIR=/tmp/vipfw.ocf.ra.debug
DEBUG_LOG=$DEBUG_LOG_DIR/log
USE_DEBUG_LOG=false
ls_stat_is_dir_0700_root() {
set -- $(command ls -ldn "$1" 2>/dev/null);
[[ $1/$3 = drwx?-??-?/0 ]]
}
# try to avoid symlink vuln.
if ls_stat_is_dir_0700_root $DEBUG_LOG_DIR &&
[[ -w "$DEBUG_LOG" && ! -L "$DEBUG_LOG" ]]
then
USE_DEBUG_LOG=true
exec 9>>"$DEBUG_LOG"
date >&9
echo "$*" >&9
env | grep OCF_ | sort >&9
else
exec 9>/dev/null
fi
# end of debugging aid
#######################################################################
meta_data() {
cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="VIPfirewall">
<version>0.1</version>
<longdesc lang="en">
This resource agent manages firewall rules for Virtual IP addresses
as a master/slave resource, blocking all traffic to/from VIP in Slave state and
allowing
it in a Master state. Main purpose of this to be written is support for
seamless IET iSCSI
target resource (backed by DRBD) migration.
</longdesc>
<shortdesc lang="en">Master/Slave OCF Resource Agent for Virtual IP address
blocking by OS firewall</shortdesc>
<parameters>
<parameter name="vip" unique="1" required="1">
<longdesc lang="en">
Virtual IP address (CIDR prefixes are accepted too).
</longdesc>
<shortdesc lang="en">ip addresses</shortdesc>
<content type="string"/>
</parameter>
<parameter name="allow_action" required="0" unique="0">
<longdesc lang="en">
How should iptables be configured on promote. Must be one of "accept" or "pass",
first one will explicitely permit traffic to/from VIP, second will pass it
further to chains.
Default is "accept".
</longdesc>
<shortdesc lang="en">iptables action on promote</shortdesc>
<content type="string" default="${OCF_RESKEY_allow_action_default}"/>
</parameter>
<parameter name="deny_action" required="0" unique="0">
<longdesc lang="en">
How should iptables be configured on start/demote. Must be one of "drop" or
"reject",
first one will silently drop traffic to/from VIP, second will result in ICMP
"port-unreachable" messages.
Default is "drop".
</longdesc>
<shortdesc lang="en">iptables action on start/demote</shortdesc>
<content type="string" default="${OCF_RESKEY_deny_action_default}"/>
</parameter>
</parameters>
<actions>
<action name="start" timeout="5" />
<action name="promote" timeout="5" />
<action name="demote" timeout="5" />
<action name="notify" timeout="5" />
<action name="stop" timeout="5" />
<action name="monitor" depth="0" timeout="5" interval="5" start-delay="1m"
role="Slave" />
<action name="monitor" depth="0" timeout="5" interval="5" start-delay="1m"
role="Master" />
<action name="meta-data" timeout="5" />
<action name="validate-all" timeout="5" />
</actions>
</resource-agent>
END
}
do_cmd() {
# Run a command, return its exit code, capture any output, and log
# everything if appropriate.
local cmd="$*" cmd_out ret
ocf_log debug "$OCF_RESOURCE_INSTANCE ($OCF_RESKEY_vip): Calling $cmd"
cmd_out=$( "$@" )
ret=$?
if [ $ret != 0 ]; then
ocf_log err "$OCF_RESOURCE_INSTANCE ($OCF_RESKEY_vip): Called
$cmd"
ocf_log err "$OCF_RESOURCE_INSTANCE ($OCF_RESKEY_vip): Exit
code $ret"
ocf_log err "$OCF_RESOURCE_INSTANCE ($OCF_RESKEY_vip): Command
output: $cmd_out"
else
ocf_log debug "$OCF_RESOURCE_INSTANCE ($OCF_RESKEY_vip): Exit
code $ret"
ocf_log debug "$OCF_RESOURCE_INSTANCE ($OCF_RESKEY_vip):
Command output: $cmd_out"
fi
echo "$cmd_out"
return $ret
}
vipfw_set_master_score() {
# Use quiet mode (-Q) to quench logging. Actual score updates
# will get logged by attrd anyway
do_cmd ${HA_SBIN_DIR}/crm_master -Q -l reboot -v $1
}
vipfw_remove_master_score() {
do_cmd ${HA_SBIN_DIR}/crm_master -l reboot -D
}
vipfw_update_master_score() {
# We have no any preferences yet
# FIXME: Shouldn't we check for presence of VIP locally?
# VAL: not sure.
vipfw_set_master_score 1000
return $OCF_SUCCESS
}
is_firewall_enabled() {
# VAL: pros and cons with using dedicated TABLE?
test -f /proc/net/ip_tables_names
}
#######################################################################
vipfw_usage() {
echo "\
usage: $0 {start|stop|monitor|validate-all|promote|demote|notify|meta-data}
Expects to have a fully populated OCF RA-compliant environment set."
}
VIPFW_POLICY_UNKNOWN=0
VIPFW_POLICY_ACCEPT=1
VIPFW_POLICY_PASS=2
VIPFW_POLICY_DROP=3
VIPFW_POLICY_REJECT=4
vipfw_get_policy() {
local policy
set -- `iptables -nL $1 | \
grep -E
"$2[[:space:]]+all[[:space:]]+--[[:space:]]+$3[[:space:]]+$4[[:space:]]+$"| \
head -n1`
case $1 in
ACCEPT)
policy=$VIPFW_POLICY_ACCEPT
;;
DROP)
policy=$VIPFW_POLICY_DROP
;;
REJECT)
policy=$VIPFW_POLICY_REJECT
;;
all) # No target specified and second field is catched instead of first
one
policy=$VIPFW_POLICY_PASS
;;
*)
policy=$VIPFW_POLICY_UNKNOWN
;;
esac
return $policy
}
vipfw_status() {
local rc policy_input policy_output
local catch_regexp="^(ACCEPT|DROP|REJECT|[[:space:]]+)"
rc=$OCF_NOT_RUNNING
if ! is_firewall_enabled ; then
return $rc
fi
case $1 in
non_pass)
catch_regexp="^(ACCEPT|DROP|REJECT)"
;;
*)
;;
esac
regexp_vip=${OCF_RESKEY_vip//./\\.} # Prepare dots for regexp
regexp_vip=${regexp_vip/\/32/} # Strip /32 if any to comply with
iptables output
regexp_all="0\.0\.0\.0/0"
vipfw_get_policy "INPUT" ${catch_regexp} ${regexp_all} ${regexp_vip}
policy_input=$?
vipfw_get_policy "OUTPUT" ${catch_regexp} ${regexp_vip} ${regexp_all}
policy_output=$?
if (( $policy_input != $policy_output )) ; then
return $OCF_ERR_GENERIC # ??? FIXME!!!
fi
case $policy_input in
$VIPFW_POLICY_ACCEPT)
if [ ${OCF_RESKEY_allow_action} == "accept" ] ; then
rc=$OCF_RUNNING_MASTER
fi
;;
$VIPFW_POLICY_PASS)
if [ ${OCF_RESKEY_allow_action} == "pass" ] ; then
# Recursion, check next rule after 'pass' one
vipfw_status "non_pass"
case $? in
$OCF_SUCCESS)
# We have 'drop' or' reject' rule right after
'pass' rule and
# that rule is configured by ourselves.
# That means that traffic will be blocked.
# Fail for now.
rc=$OCF_ERR_GENERIC
;;
$OCF_NOT_RUNNING)
# We didn't find any more rules except our
'pass' rule.
# This means that we are configured correctly,
independently
# of correct or broken default iptables setup.
# If traffic is blocked by later rules then
that's not our fault.
rc=$OCF_RUNNING_MASTER
;;
$OCF_ERR_GENERIC)
# Hmmm. Something is really broken.
rc=$OCF_ERR_GENERIC
;;
esac
fi
;;
$VIPFW_POLICY_DROP)
if [ ${OCF_RESKEY_deny_action} == "drop" ] ; then
rc=$OCF_SUCCESS
fi
;;
$VIPFW_POLICY_REJECT)
if [ ${OCF_RESKEY_deny_action} == "reject" ] ; then
rc=$OCF_SUCCESS
fi
;;
*)
;;
esac
return $rc
}
vipfw_monitor() {
local status
vipfw_status
status=$?
vipfw_update_master_score
return $status
}
# This will return after first successful operation
do_iptables() {
local rc=1
local count=0
# Call iptables from within a loop, because iptables is known to fail
# on concurrent operations
# iptables returns 2 on command line params error, stop in that case too
# VAL: you are paranoid...
until (( rc != 1 )) || (( count > 10 )) ; do # We are in bash, right?
# First try to be quiet
# Yes, no debug. But no flood in logs too.
if [ $USE_DEBUG_LOG == true ] ; then
do_cmd $IPTABLES "$@"
else
$IPTABLES "$@" >/dev/null 2>&1
fi
rc=$?
(( count++ ))
done
if (( count > 10 )) ; then
# Try once more and log error
do_cmd $IPTABLES "$@"
rc=$?
fi
return $rc
}
# This will run command in loop until first error
do_iptables_all() {
local rc=0
local count=0
# Call iptables from within a loop, because iptables is known to fail
# on concurrent operations
# iptables returns 2 on command line params error
until (( rc != 0 )) ; do
# First try to be quiet
# Yes, no debug. But no flood in logs too.
if [ $USE_DEBUG_LOG == true ] ; then
do_cmd $IPTABLES "$@"
else
$IPTABLES "$@" >/dev/null 2>&1
fi
rc=$?
(( count++ ))
done
if (( count > 1 )) ; then # We succeded at least once
return 0
else
return $rc
fi
}
vipfw_op() {
local op op1
local target target1
local rc_in
local rc_out
local rc
case $1 in
# It is not safe to use "-R" because the only way to replace rule is by
its rule number
# But, that number can change between read and replace operations. So I
use insert/delete.
add|block|unblock)
# We should add our rules very early in list to avoid default
setup with stateful firewall
# break things
op="-I"
target="-j DROP"
case $1 in
unblock)
if [ ${OCF_RESKEY_allow_action} == "pass" ] ; then
target=""
else
target="-j ACCEPT"
fi
;;
*)
;;
esac
do_iptables $op INPUT -d $OCF_RESKEY_vip $target
rc_in=$?
do_iptables $op OUTPUT -s $OCF_RESKEY_vip $target
rc_out=$?
if (( $rc_in )) || (( $rc_out )) ; then
rc=1
else
rc=0
fi
;;
*)
;;
esac
case $1 in
block|unblock|delete)
# be safe on deletion: delete by rule spec, not by rule number
op="-D"
target="-j DROP"
case $1 in
block)
# It is not safe to use "-R" because the only way to
replace rule is by its rule number
# But, that number can change between read and replace
operations
if [ ${OCF_RESKEY_allow_action} == "pass" ] ; then
target=""
else
target="-j ACCEPT"
fi
;;
*)
;;
esac
# Delete all matching rules (they can remain from previous runs)
# We are not interested in result, because this is only a
cleanup.
# FIXME: Nope, that's wrong. We MUST ensure that all DROP rules
are deleted if
# we use 'pass' mode'
do_iptables_all $op INPUT -d $OCF_RESKEY_vip $target
do_iptables_all $op OUTPUT -s $OCF_RESKEY_vip $target
;;
*)
;;
esac
return $rc
}
vipfw_start() {
local rc
local status
local first_try=true
rc=$OCF_ERR_GENERIC
if ! is_firewall_enabled; then
ocf_log err "Firewall is not started, unable to block
VIPs."$'\n';
return $OCF_ERR_INSTALLED
fi
# Keep trying to bring up the resource;
# wait for the CRM to time us out if this fails
while :; do
vipfw_status
status=$?
case "$status" in
$OCF_SUCCESS)
rc=$OCF_SUCCESS
break
;;
$OCF_NOT_RUNNING)
vipfw_op add
;;
$OCF_RUNNING_MASTER)
ocf_log warn "$OCF_RESKEY_vip already promoted,
demoting."
vipfw_op block
;;
$OCF_ERR_GENERIC)
vipfw_op block
;;
esac
$first_try || sleep 1
first_try=false
done
# in case someone does not configure monitor,
# we must at least call it once after start.
vipfw_update_master_score
return $rc
}
vipfw_promote() {
local rc
local status
local first_try=true
rc=$OCF_ERR_GENERIC
# Keep trying to promote the resource;
# wait for the CRM to time us out if this fails
while :; do
vipfw_status
status=$?
case "$status" in
$OCF_SUCCESS)
vipfw_op unblock
;;
$OCF_NOT_RUNNING)
ocf_log error "Trying to promote a resource that was
not started"
break
;;
$OCF_RUNNING_MASTER)
rc=$OCF_SUCCESS
break
;;
$OCF_ERR_GENERIC)
vipfw_op unblock
;;
esac
$first_try || sleep 1
first_try=false
done
return $rc
}
vipfw_demote() {
local rc
local status
local first_try=true
rc=$OCF_ERR_GENERIC
# Keep trying to demote the resource;
# wait for the CRM to time us out if this fails
while :; do
vipfw_status
status=$?
case "$status" in
$OCF_SUCCESS)
rc=$OCF_SUCCESS
break
;;
$OCF_NOT_RUNNING)
ocf_log error "Trying to demote a resource that was not
started"
break
;;
$OCF_RUNNING_MASTER)
vipfw_op block
;;
$OCF_ERR_GENERIC)
vipfw_op block
;;
esac
$first_try || sleep 1
first_try=false
done
return $rc
}
vipfw_stop() {
local rc=$OCF_ERR_GENERIC
local first_try=true
# Keep trying to bring down the resource;
# wait for the CRM to time us out if this fails
while :; do
vipfw_status
status=$?
case "$status" in
$OCF_SUCCESS)
vipfw_op delete
;;
$OCF_NOT_RUNNING)
rc=$OCF_SUCCESS
break
;;
$OCF_RUNNING_MASTER)
ocf_log warn "$OCF_RESKEY_vip still Primary, demoting."
vipfw_op block
;;
$OCF_ERR_GENERIC)
vipfw_op block
vipfw_op delete
;;
esac
$first_try || sleep 1
first_try=false
done
# do not let old master scores laying around.
# they may confuse crm if this node was set to standby.
vipfw_remove_master_score
return $rc
}
vipfw_notify() {
local n_type=$OCF_RESKEY_CRM_meta_notify_type
local n_op=$OCF_RESKEY_CRM_meta_notify_operation
# FIXME: Nothing to do?
return $OCF_SUCCESS
}
# "macro" to be able to give useful error messages
# on clone resource configuration error.
meta_expect()
{
local what=$1 whatvar=OCF_RESKEY_CRM_meta_${1//-/_} op=$2 expect=$3
local val=${!whatvar}
if [[ -n $val ]]; then
# [, not [[, or it won't work ;)
[ $val $op $expect ] && return
fi
ocf_log err "meta parameter misconfigured, expected $what $op $expect,
but found ${val:-unset}."
exit $OCF_ERR_CONFIGURED
}
ls_stat_is_block_maj_147() {
set -- $(command ls -L -l "$1" 2>/dev/null)
[[ $1 = b* ]] && [[ $5 == 147,* ]]
}
check_crm_feature_set()
{
set -- ${OCF_RESKEY_crm_feature_set//[!0-9]/ }
local a=${1:-0} b=${2:-0} c=${3:-0}
(( a > 3 )) ||
(( a == 3 && b > 0 )) ||
(( a == 3 && b == 0 && c > 0 )) ||
ocf_log warn "You may be disappointed: This RA is intended for
pacemaker 1.0 or better!"
}
vipfw_validate_all() {
check_binary $IPTABLES
# XXX I really take cibadmin, sed, grep, etc. for granted.
check_crm_feature_set
# Check clone and M/S options.
meta_expect clone-max -le 2
meta_expect clone-node-max = 1
meta_expect master-node-max = 1
meta_expect master-max = 1
case "$OCF_RESKEY_vip" in
"")
ocf_log err "No Virtual IP address specified!"
return $OCF_ERR_CONFIGURED
;;
*[!./0-9]*)
ocf_log err "IP address must only contain [./0-9]"
return $OCF_ERR_CONFIGURED
esac
case "${OCF_RESKEY_allow_action}" in
"pass"|"accept")
;;
*)
ocf_log err "allow_action must be one of \"accept\" or \"pass\""
return $OCF_ERR_CONFIGURED
;;
esac
case "${OCF_RESKEY_deny_action}" in
"drop"|"reject")
;;
*)
ocf_log err "deny_action must be one of \"drop\" or \"reject\""
return $OCF_ERR_CONFIGURED
;;
esac
return $OCF_SUCCESS
}
#######################################################################
if [ $# != 1 ]; then
vipfw_usage
exit $OCF_ERR_ARGS
fi
# if $__OCF_ACTION = monitor, but meta_interval not set,
# this is a "probe". we could change behaviour.
: ${OCF_RESKEY_CRM_meta_interval=0}
case $__OCF_ACTION in
meta-data)
meta_data
exit $OCF_SUCCESS
;;
usage)
vipfw_usage
exit $OCF_SUCCESS
esac
if $USE_DEBUG_LOG ; then
exec 2>&9
set -x
fi
# Everything except usage and meta-data must pass the validate test
vipfw_validate_all || exit
case $__OCF_ACTION in
start)
vipfw_start
;;
stop)
vipfw_stop
;;
notify)
vipfw_notify
;;
promote)
vipfw_promote
;;
demote)
vipfw_demote
;;
status)
vipfw_status
;;
monitor)
vipfw_monitor
;;
validate-all)
;;
*)
vipfw_usage
exit $OCF_ERR_UNIMPLEMENTED
esac
# exit code is the exit code (return code) of the last command (shell function)
_______________________________________________
Pacemaker mailing list: Pacemaker@oss.clusterlabs.org
http://oss.clusterlabs.org/mailman/listinfo/pacemaker
Project Home: http://www.clusterlabs.org
Getting started: http://www.clusterlabs.org/doc/Cluster_from_Scratch.pdf
Bugs: http://developerbugs.linux-foundation.org/enter_bug.cgi?product=Pacemaker
