diff --git a/crm/test/2node-fail.sh.in b/crm/test/2node-fail.sh.in
index 263f0ed7ed..6253cd869e 100644
--- a/crm/test/2node-fail.sh.in
+++ b/crm/test/2node-fail.sh.in
@@ -1,339 +1,339 @@
#!/bin/bash
#
# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
testdir=@libdir@/heartbeat/crmtest
. ${testdir}/helper.sh || exit 1
CRM_ERR_SHUTDOWN=0
test_nodes=2
function 2node_fail_test() {
test_type=$1
fail_pieces=$2
fail_node=$3
good_node=$4
do_failback=$5
accelerated=$6
if [ $fail_node = $test_node_1 ]; then
moved_rsc=rsc1
else
moved_rsc=rsc2
fi
#----
echo -ne "\033]0;$test_type: Iteration $iteration of $repeats\007"
crm-cleanup
do_cmd echo "#############################"
do_cmd echo "$test_type: Iteration $iteration of $repeats"
# make *sure* theres nothing left over from last time
#----
do_cmd echo "wait for HA to start on ${test_node_1}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \
-s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" \
-s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \
-e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_1} failed."
#----
do_cmd echo "wait for CRMd to start on ${test_node_1}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}" \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert "CRMd startup on ${test_node_1} failed."
do_cmd wait_for_state S_IDLE 3 $test_node_1
cts_assert "S_IDLE not reached on $test_node_1 (startup)!"
#----
do_cmd echo Create the first constraint and wait for S_IDLE
rsc=rsc1
uuid1=`uuidgen`
uuid2=`uuidgen`
uuid3=`uuidgen`
node_xml="'<rsc_location id=\"${uuid1}\" rsc=\"${rsc}\">
<rule id=\"${uuid2}\" result=\"can\"/>
<rule id=\"${uuid3}\" score=\"1000\" boolean_op=\"or\">
<expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_1}\"/>
</rule>
</rsc_location>'"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_constraint_adv $test_node_1 $node_xml
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert Adding constraint1 did not pass
#----
do_cmd echo Create the second constraint and wait for S_IDLE
rsc=rsc2
uuid1=`uuidgen`
uuid2=`uuidgen`
uuid3=`uuidgen`
node_xml="'<rsc_location id=\"${uuid1}\" rsc=\"${rsc}\">
<rule id=\"${uuid2}\" result=\"can\"/>
<rule id=\"${uuid3}\" score=\"1000\" boolean_op=\"or\">
<expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_2}\"/>
</rule>
</rsc_location>'"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_constraint_adv $test_node_1 $node_xml
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert Adding constraint2 did not pass
#----
do_cmd echo Create the first resource and wait for S_IDLE after start
args="<nvpair name=\"1\" value=\"${ip_rsc_1}\"/>"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - ignore $args
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
-s "crmd(.*) Performing op start(.*) on rsc1" \
-s "crmd(.*) Resource state: rsc1(.*) after start"
cts_assert Adding rsc1 did not pass
#----
do_cmd echo Create the second resource and wait for S_IDLE after start
args="<nvpair name=\"1\" value=\"${ip_rsc_2}\"/>"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - ignore $args
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
-s "crmd(.*) Performing op start(.*) on rsc2" \
-s "crmd(.*) Resource state: rsc2(.*) after start"
cts_assert Adding rsc2 did not pass
#----
do_cmd echo Various sanity checks - stage 1
do_cmd wait_for_state S_IDLE 3 $test_node_1
cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!"
do_cmd is_running rsc1 $test_node_1
cts_assert "rsc1 NOT running"
do_cmd is_running rsc2 $test_node_1
cts_assert "rsc2 NOT running"
do_cmd is_dc $test_node_1
cts_assert "$test_node_1 is supposed to be the DC"
do_cmd is_running rsc1 $test_node_1 x$test_node_1
cts_assert_false "rsc1 IS running on x$test_node_1"
do_cmd is_running rsc1 $test_node_1 $test_node_1
cts_assert "rsc1 NOT running on $test_node_1"
do_cmd is_running rsc2 $test_node_1 $test_node_1
cts_assert "rsc2 NOT running on $test_node_1"
#----
do_cmd echo "wait for HA to start on $test_node_2"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" \
-s "${test_node_2} heartbeat(.*): info: Starting (.*)lrmd" \
-e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_2} failed."
#----
do_cmd echo "wait for CRMd to start on $test_node_2"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \
-s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" \
-s "${test_node_2} crmd(.*) State transition \"S_PENDING\" \-> \"S_NOT_DC\""
cts_assert "CRMd startup on ${test_node_2} failed."
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3500 \
-s "${test_node_1} crmd(.*) State transition(.*) \-> \"S_IDLE\"" \
-s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" \
-s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}"
cts_assert "rsc2 was not transferred to ${test_node_2} on startup."
#----
do_cmd echo Various sanity checks - stage 2
do_cmd wait_for_state S_NOT_DC 30 $test_node_2
cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!"
do_cmd wait_for_state S_IDLE 30 $test_node_1
cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!"
do_cmd is_running rsc1 $test_node_1
cts_assert "rsc1 NOT running"
do_cmd is_running rsc2 $test_node_1
cts_assert "rsc2 NOT running"
do_cmd is_running rsc1 $test_node_1 $test_node_1
cts_assert "rsc1 NOT running on $test_node_1"
do_cmd is_running rsc2 $test_node_1 $test_node_2
cts_assert "rsc2 NOT running on $test_node_2"
#----
is_dc $fail_node 2>&1 > /dev/null
test_for_election=$?
do_cmd echo Killing $fail_pieces on $fail_node
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $ADMIN_USER $fail_node "killall -9 $fail_pieces" &
if [ $test_for_election = 0 ]; then
do_cmd echo Killed the DC... checking for DC Failover
- do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 50000 \
+ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 60000 \
-s "${good_node} crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \
-s "${good_node} crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert "Transition of the DC from ${fail_node} to ${good_node} failed."
else
do_cmd echo Killed slave node... checking the DC noticed
- do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 50000 \
+ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 60000 \
-s "${good_node} crmd(.*) State transition (.*) \-> \"S_POLICY_ENGINE\"" \
-s "${good_node} crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert "Failure of slave node ${fail_node} was not noticed on the DC (${good_node})."
fi
- do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 50000 \
+ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 60000 \
-s "${god_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" \
-s "crmd(.*) Resource state: ${moved_rsc}(.*) after start] on ${good_node}"
cts_assert "Move of ${moved_rsc} to ${good_node} failed."
#----
do_cmd echo Various sanity checks - stage 3
do_cmd wait_for_state S_IDLE 60 $good_node
cts_assert "S_IDLE not reached on $good_node after kill!"
do_cmd is_running rsc1 $good_node
cts_assert "rsc1 NOT running on $good_node"
do_cmd is_running rsc2 $good_node
cts_assert "rsc2 NOT running on $good_node"
do_cmd is_running rsc1 $good_node $fail_node
cts_assert_false "rsc1 IS running on $fail_node"
do_cmd is_running rsc2 $good_node $good_node
cts_assert "rsc2 NOT running on $good_node"
#----
if [ $do_failback = 1 ]; then
do_cmd echo "Re-Starting on failed node $fail_node"
if [ "$fail_pieces" = "crmd" ]; then
do_cmd echo "HA still running, skipping restart"
else
do_cmd echo "wait for HA to start on $fail_node"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $fail_node $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" &
- do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
+ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2000 \
-s "${fail_node} ccm(.*) Hostname: ${fail_node}" \
-s "${fail_node} heartbeat(.*): info: Starting (.*)lrmd" \
-e "${fail_node} heartbeat(.*) Client (.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${fail_node} failed."
fi
#----
do_cmd echo "wait for CRMd to start on $fail_node"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $fail_node $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
-s "${fail_node} crmd(.*)FSA Hostname: ${fail_node}" \
-s "${fail_node} crmd(.*) State transition \"S_PENDING\" \-> \"S_NOT_DC\""
cts_assert "CRMd startup on ${fail_node} failed."
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
-s "${good_node} crmd(.*) State transition(.*) \-> \"S_IDLE\"" \
-s "${fail_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" \
-s "crmd(.*) Resource state: ${moved_rsc}(.*) after start] on ${fail_node}"
cts_assert "$moved_rsc was not transferred to ${fail_node} on startup."
do_cmd echo Various sanity checks - stage 4
do_cmd wait_for_state S_NOT_DC 30 $fail_node
cts_assert "S_NOT_DC not reached on $fail_node (restart)!"
do_cmd wait_for_state S_IDLE 30 $good_node
cts_assert "S_IDLE not reached on $good_node (restart)!"
do_cmd is_running rsc1 $test_node_1
cts_assert "rsc1 NOT running"
do_cmd is_running rsc2 $test_node_1
cts_assert "rsc2 NOT running"
do_cmd is_running rsc1 $test_node_1 $test_node_1
cts_assert "rsc1 NOT running on $test_node_1"
do_cmd is_running rsc2 $test_node_1 $test_node_2
cts_assert "rsc2 NOT running on $test_node_2"
fi
#----
do_cmd echo "test ${test_type}: PASSED"
}
while [ $iteration -lt $repeats ]; do
iteration=`expr $iteration + 1`
echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007"
echo "########### $test_type : Begining iteration $iteration of $repeats ###########"
(
2node_fail_test 2node__fail_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 0 0
2node_fail_test 2node__fail_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 0 0
2node_fail_test 2node__failback_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 1 0
2node_fail_test 2node__failback_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 1 0
2node_fail_test 2node__fail_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 0 0
2node_fail_test 2node__fail_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 0 0
2node_fail_test 2node__failback_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 1 0
2node_fail_test 2node__failback_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 1 0
#2node_fail_test 2node__fail_slave_LRM "lrmd" ${test_node_2} ${test_node_1}
#2node_fail_test 2node__fail_DC_LRM "lrmd" ${test_node_1} ${test_node_2}
echo "test suite: PASSED"
)
done
diff --git a/crm/test/2node.sh.in b/crm/test/2node.sh.in
index b068e08e02..495eb3f449 100644
--- a/crm/test/2node.sh.in
+++ b/crm/test/2node.sh.in
@@ -1,255 +1,259 @@
#!/bin/bash
#
# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
testdir=@libdir@/heartbeat/crmtest
. ${testdir}/helper.sh || exit 1
CRM_ERR_SHUTDOWN=0
test_nodes=2
while [ $iteration -lt $repeats ]; do
iteration=`expr $iteration + 1`
echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007"
echo "########### Begining iteration $iteration of $repeats ###########"
(
# make *sure* theres nothing left over from last time
crm-cleanup
#----
do_cmd echo "wait for HA to start on ${test_node_1}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \
-s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" \
-s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \
-e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_1} failed."
#----
do_cmd echo "wait for CRMd to start on ${test_node_1}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}" \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert "CRMd startup on ${test_node_1} failed."
do_cmd wait_for_state S_IDLE 3 $test_node_1
cts_assert "S_IDLE not reached on $test_node_1 (startup)!"
#----
do_cmd echo Create the first constraint and wait for S_IDLE
rsc=rsc1
uuid1=`uuidgen`
uuid2=`uuidgen`
uuid3=`uuidgen`
node_xml="'<rsc_location id=\"${uuid1}\" rsc=\"${rsc}\">
<rule id=\"${uuid2}\" result=\"can\"/>
<rule id=\"${uuid3}\" score=\"1000\" boolean_op=\"or\">
<expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_1}\"/>
</rule>
</rsc_location>'"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_constraint_adv $test_node_1 $node_xml
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert Adding constraint1 did not pass
#----
do_cmd echo Create the second constraint and wait for S_IDLE
rsc=rsc2
uuid1=`uuidgen`
uuid2=`uuidgen`
uuid3=`uuidgen`
node_xml="'<rsc_location id=\"${uuid1}\" rsc=\"${rsc}\">
<rule id=\"${uuid2}\" result=\"can\"/>
<rule id=\"${uuid3}\" score=\"1000\" boolean_op=\"or\">
<expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_2}\"/>
</rule>
</rsc_location>'"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_constraint_adv $test_node_1 $node_xml
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert Adding constraint2 did not pass
#----
do_cmd echo Create the first resource and wait for S_IDLE after start
args="<nvpair name=\"1\" value=\"${ip_rsc_1}\"/>"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
-s "crmd(.*) Performing op start(.*) on rsc1" \
-s "crmd(.*) Resource state: rsc1(.*) after start"
cts_assert Adding rsc1 did not pass
#----
do_cmd echo Create the second resource and wait for S_IDLE after start
args="<nvpair name=\"1\" value=\"${ip_rsc_2}\"/>"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
-s "crmd(.*) Performing op start(.*) on rsc2" \
-s "crmd(.*) Resource state: rsc2(.*) after start"
cts_assert Adding rsc2 did not pass
#----
do_cmd echo Various sanity checks - stage 1
do_cmd wait_for_state S_IDLE 3 $test_node_1
cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!"
do_cmd is_running rsc1 $test_node_1
cts_assert "rsc1 NOT running"
do_cmd is_running rsc2 $test_node_1
cts_assert "rsc2 NOT running"
do_cmd is_dc $test_node_1
cts_assert "$test_node_1 is supposed to be the DC"
do_cmd is_running rsc1 $test_node_1 x$test_node_1
cts_assert_false "rsc1 IS running on x$test_node_1"
do_cmd is_running rsc1 $test_node_1 $test_node_1
cts_assert "rsc1 NOT running on $test_node_1"
do_cmd is_running rsc2 $test_node_1 $test_node_1
cts_assert "rsc2 NOT running on $test_node_1"
#----
do_cmd echo "wait for HA to start on $test_node_2"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" \
-s "${test_node_2} heartbeat(.*): info: Starting (.*)lrmd" \
-e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_2} failed."
#----
do_cmd echo "wait for CRMd to start on $test_node_2"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \
-s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" \
-s "${test_node_2} crmd(.*) State transition \"S_PENDING\" \-> \"S_NOT_DC\""
cts_assert "CRMd startup on ${test_node_2} failed."
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3500 \
-s "${test_node_1} crmd(.*) State transition(.*) \-> \"S_IDLE\"" \
-s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" \
-s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}"
cts_assert "rsc2 was not transferred to ${test_node_2} on startup."
#----
do_cmd echo Various sanity checks - stage 2
do_cmd wait_for_state S_NOT_DC 30 $test_node_2
cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!"
do_cmd wait_for_state S_IDLE 30 $test_node_1
cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!"
do_cmd is_running rsc1 $test_node_1
cts_assert "rsc1 NOT running"
do_cmd is_running rsc2 $test_node_1
cts_assert "rsc2 NOT running"
do_cmd is_running rsc1 $test_node_1 $test_node_1
cts_assert "rsc1 NOT running on $test_node_1"
+do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
+ -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}"
do_cmd is_running rsc2 $test_node_1 $test_node_2
cts_assert "rsc2 NOT running on $test_node_2"
#----
do_cmd echo Shutdown $test_node_1
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \
-s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \
-s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_NOT_DC\""
cts_assert "Transition of the DC from ${test_node_1} to ${test_node_2} failed."
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \
-s "${test_node_1} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \
-e "${test_node_1} crmd(.*)Timer I_TERMINATE just popped" \
-s "${test_node_1} crmd(.*)\[crmd\] stopped"
cts_assert "Shutdown of ${test_node_1} failed"
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
-s "crmd(.*) Performing op start(.*) on rsc1" \
-s "crmd(.*) Resource state: rsc1(.*) after start"
cts_assert "rsc1 was not transferred to ${test_node_2} on shutdown."
#----
do_cmd echo Various sanity checks - stage 3
do_cmd wait_for_state S_IDLE 60 $test_node_2
cts_assert "S_IDLE not reached on $test_node_2 after shutdown!"
do_cmd is_running rsc1 $test_node_2
cts_assert "rsc1 NOT running"
do_cmd is_running rsc2 $test_node_2
cts_assert "rsc2 NOT running"
+do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
+ -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_2}"
do_cmd is_running rsc1 $test_node_2 $test_node_1
cts_assert_false "rsc1 IS running on $test_node_1"
do_cmd is_running rsc2 $test_node_2 $test_node_2
cts_assert "rsc2 NOT running on $test_node_2"
#----
do_cmd echo Shutdown $test_node_2
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2
do_cmd wait_for_state S_PENDING 30 $test_node_2
cts_assert "S_PENDING not reached on $test_node_2!"
#----
do_cmd echo Escalate the shutdown
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "${test_node_2} crmd(.*)State transition \"S_PENDING\" \-> \"S_STOPPING\""\
-s "${test_node_2} crmd(.*)\[crmd\] stopped"
cts_assert "Shutdown of ${test_node_2} failed."
#----
do_cmd remote_cmd $CRMD_USER $test_node_2 killall -9 crmd
echo "test: PASSED"
)
done
diff --git a/crm/test/3node-rapid.sh.in b/crm/test/3node-rapid.sh.in
index 1485e40c03..f365decee5 100644
--- a/crm/test/3node-rapid.sh.in
+++ b/crm/test/3node-rapid.sh.in
@@ -1,290 +1,296 @@
#!/bin/bash
#
# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
testdir=@libdir@/heartbeat/crmtest
. ${testdir}/helper.sh || exit 1
CRM_ERR_SHUTDOWN=0
test_nodes=3
while [ $iteration -lt $repeats ]; do
iteration=`expr $iteration + 1`
echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007"
echo "########### Begining iteration $iteration of $repeats ###########"
(
# make *sure* theres nothing left over from last time
crm-cleanup
#----
do_cmd echo "wait for HA to start on ${test_node_1}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null"
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \
-s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" \
-s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \
-e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_1} failed."
#----
do_cmd echo "wait for HA to start on ${test_node_2}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null"
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \
-s "${test_node_2} ccm(.*): info: Hostname: ${test_node_2}" \
-s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" \
-e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_2} failed."
#----
do_cmd echo "wait for HA to start on ${test_node_3}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null"
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \
-s "${test_node_3} ccm(.*): info: Hostname: ${test_node_3}" \
-s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" \
-e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_3} failed."
#----
do_cmd echo "wait for CRMd to start on ${test_node_1}, ${test_node_2} and ${test_node_3}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" &
do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" &
do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \
-s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}"
cts_assert "CRMd startup on ${test_node_1} failed."
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \
-s "${test_node_2} crmd(.*): info:(.*)FSA Hostname: ${test_node_2}"
cts_assert "CRMd startup on ${test_node_2} failed."
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \
-s "${test_node_3} crmd(.*): info:(.*)FSA Hostname: ${test_node_3}"
cts_assert "CRMd startup on ${test_node_3} failed."
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 25000 \
-s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
-s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" \
-s "${test_node_3} crmd(.*) State transition (.*) \-> \"S_NOT_DC\""
cts_assert "CRMd did not start up into correct modes."
#----
do_cmd echo Various sanity checks - stage 1
do_cmd is_dc $test_node_1 $test_node_1
cts_assert "$test_node_1 is supposed to be the DC"
do_cmd wait_for_state S_IDLE 3 DC $test_node_1
cts_assert "S_IDLE not reached on the DC!"
#----
do_cmd echo Create the first constraint and wait for S_IDLE
rsc=rsc1
uuid1=`uuidgen`
uuid2=`uuidgen`
uuid3=`uuidgen`
uuid4=`uuidgen`
node_xml="'<rsc_location id=\"${uuid1}\" rsc=\"${rsc}\">
<rule id=\"${uuid2}\" result=\"can\"/>
<rule id=\"${uuid3}\" score=\"1000\" boolean_op=\"or\">
<expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_1}\"/>
</rule>
<rule id=\"${uuid4}\" score=\"100\" boolean_op=\"or\">
<expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_2}\"/>
</rule>
</rsc_location>'"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_constraint_adv $test_node_1 $node_xml
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert Adding constraint1 did not pass
#----
do_cmd echo Create the second constraint and wait for S_IDLE
rsc=rsc2
uuid1=`uuidgen`
uuid2=`uuidgen`
uuid3=`uuidgen`
uuid4=`uuidgen`
node_xml="'<rsc_location id=\"${uuid1}\" rsc=\"${rsc}\">
<rule id=\"${uuid2}\" result=\"can\"/>
<rule id=\"${uuid3}\" score=\"1000\" boolean_op=\"or\">
<expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_2}\"/>
</rule>
<rule id=\"${uuid4}\" score=\"100\" boolean_op=\"or\">
<expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_1}\"/>
</rule>
</rsc_location>'"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_constraint_adv $test_node_1 $node_xml
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert Adding constraint2 did not pass
#----
do_cmd echo Create the first resource and wait for S_IDLE after start
args="<nvpair name=\"1\" value=\"${ip_rsc_1}\"/>"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
-s "crmd(.*) Performing op start(.*) on rsc1" \
-s "crmd(.*) Resource state: rsc1(.*) after start"
cts_assert Adding rsc1 did not pass
#----
do_cmd echo Create the second resource and wait for S_IDLE after start
args="<nvpair name=\"1\" value=\"${ip_rsc_2}\"/>"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
-s "crmd(.*) Performing op start(.*) on rsc2" \
-s "crmd(.*) Resource state: rsc2(.*) after start"
cts_assert Adding rsc2 did not pass
#----
do_cmd echo Various sanity checks - stage 2
do_cmd is_dc $test_node_1 $test_node_1
cts_assert "$test_node_1 is supposed to be the DC"
do_cmd wait_for_state S_IDLE 3 DC $test_node_1
cts_assert "S_IDLE not reached on the DC!"
do_cmd is_running rsc1 $test_node_1 $test_node_1
cts_assert "rsc1 NOT running on $test_node_1"
do_cmd is_running rsc2 $test_node_1 $test_node_2
cts_assert "rsc2 NOT running on $test_node_2"
#----
do_cmd echo Shutdown $test_node_1
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
-s "crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \
-s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_NOT_DC\""
cts_assert "Transition of the DC away from ${test_node_1} failed."
# -s "crmd(.*) State transition (.*) \->" \
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
-s "${test_node_1} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \
-s "${test_node_1} crmd(.*)\[crmd\] stopped" \
-e "${test_node_1} crmd(.*)Timer I_TERMINATE just popped"
cts_assert "Shutdown of ${test_node_1} failed"
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
-s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\""
cts_assert "No DC Declared"
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
-s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \
-s "crmd(.*)State transition(.*) \-> \"S_IDLE\""
cts_assert "DC not idle"
#----
do_cmd echo Various sanity checks - stage 3
do_cmd is_dc $test_node_2 $test_node_2
cts_assert "$test_node_2 is supposed to be the DC"
do_cmd wait_for_state S_IDLE 3 $test_node_2
cts_assert "S_IDLE not reached on $test_node_2!"
-do_cmd is_running rsc1 $test_node_2
+do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
+ -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_2}"
+do_cmd is_running rsc1 $test_node_2 $test_node_2
cts_assert "rsc1 NOT running on $test_node_2"
do_cmd is_running rsc2 $test_node_2 $test_node_2
cts_assert "rsc2 NOT running on $test_node_2"
#----
do_cmd echo Shutdown $test_node_2
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \
-s "crmd(.*) State transition \"S_NOT_DC\" \-> \"S_ELECTION\"" \
-s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\""
cts_assert "Transition of the DC away from ${test_node_2} failed."
# -s "crmd(.*) State transition(.*) \->" \
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 6000 \
-s "${test_node_2} crmd(.*)State transition" \
-s "${test_node_2} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \
-s "${test_node_2} crmd(.*)\[crmd\] stopped" \
-e "${test_node_2} crmd(.*)Timer I_TERMINATE just popped"
cts_assert "Shutdown of ${test_node_2} failed"
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
-s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \
-s "crmd(.*)State transition(.*) \-> \"S_IDLE\""
cts_assert "No DC Declared"
#----
do_cmd echo Various sanity checks - stage 4
do_cmd is_dc $test_node_3 $test_node_3
cts_assert "$test_node_3 is supposed to be the DC"
do_cmd wait_for_state S_IDLE 3 $test_node_3
cts_assert "S_IDLE not reached on $test_node_3!"
-do_cmd is_running rsc1 $test_node_3
+do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
+ -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_3}"
+do_cmd is_running rsc1 $test_node_3 $test_node_3
cts_assert "rsc1 NOT running on $test_node_3"
+do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
+ -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_3}"
do_cmd is_running rsc2 $test_node_3 $test_node_3
cts_assert "rsc2 NOT running on $test_node_3"
#----
do_cmd echo Shutdown $test_node_3
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 &
do_cmd wait_for_state S_PENDING 30 $test_node_3
cts_assert "S_PENDING not reached on $test_node_3!"
#----
do_cmd echo Escalate the shutdown
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
-s "${test_node_3} crmd(.*)State transition \"S_PENDING\" \-> \"S_STOPPING\"" \
-s "${test_node_3} crmd(.*)\[crmd\] stopped"
cts_assert "Shutdown of ${test_node_3} failed."
#----
do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd
echo "test: PASSED"
)
done
diff --git a/crm/test/Makefile.am b/crm/test/Makefile.am
index 3133e013cd..ed7b331143 100644
--- a/crm/test/Makefile.am
+++ b/crm/test/Makefile.am
@@ -1,32 +1,34 @@
#
# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
MAINTAINERCLEANFILES = Makefile.in
testdir = $(libdir)/heartbeat/crmtest
-test_SCRIPTS = helper.sh testutils.pl stop.sh \
- 1node.sh 2node.sh 2node-fail.sh 3node.sh 3node-rapid.sh 3node-fail.sh
+test_SCRIPTS = helper.sh testutils.pl stop.sh cib_suite.sh \
+ 1node.sh 2node.sh 2node-fail.sh \
+ 3node.sh 3node-rapid.sh 3node-fail.sh
-EXTRA_DIST = helper.sh testutils.pl stop.sh \
- 1node.sh 2node.sh 2node-fail.sh 3node.sh 3node-rapid.sh 3node-fail.sh
+EXTRA_DIST = helper.sh testutils.pl stop.sh cib_suite.sh \
+ 1node.sh 2node.sh 2node-fail.sh \
+ 3node.sh 3node-rapid.sh 3node-fail.sh
install-exec-local:
$(mkinstalldirs) $(DESTDIR)/$(testdir)
-chown $(HA_CCMUSER) $(DESTDIR)/$(testdir)
-chgrp $(HA_APIGROUP) $(DESTDIR)/$(testdir)
-chmod g+w $(DESTDIR)/$(testdir)
diff --git a/crm/test/cib_suite.sh.in b/crm/test/cib_suite.sh.in
new file mode 100644
index 0000000000..f41afb7827
--- /dev/null
+++ b/crm/test/cib_suite.sh.in
@@ -0,0 +1,536 @@
+#!/bin/bash
+#
+# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+
+testdir=@libdir@/heartbeat/crmtest
+. ${testdir}/helper.sh || exit 1
+
+erase_sync_1="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_1\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes/>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+erase_sync_2="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_2\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes/>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+erase_sync_3="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_3\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes/>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+create_1="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_1\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+create_2="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_2\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+create_3="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_3\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+create_local_1="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_1\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ <node uname=\"$test_node_3\" description=\"test node: $test_node_3\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+create_local_2="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_2\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+create_local_3="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_3\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+bump_1="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_1\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ <node uname=\"$test_node_3\" description=\"test node: $test_node_3\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+bump_2="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_2\">
+ <cib version=\"1\" generated=\"true\" generation=\"1\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+bump_3="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_3\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+bump_remote_1="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_1\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ <node uname=\"$test_node_3\" description=\"test node: $test_node_3\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+bump_remote_2="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_2\">
+ <cib version=\"1\" generated=\"true\" generation=\"2\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+bump_remote_3="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_3\">
+ <cib version=\"1\" generated=\"true\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+sync_from_1="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_1\">
+ <cib version=\"1\" generated=\"true\" generation=\"2\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+sync_from_2="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_2\">
+ <cib version=\"1\" generated=\"true\" generation=\"2\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+sync_from_3="<?xml version=\"1.0\"?>
+<cib_fragment section=\"all\" generated_on=\"$test_node_3\">
+ <cib version=\"1\" generated=\"true\" generation=\"2\">
+ <configuration>
+ <crm_config/>
+ <nodes>
+ <node uname=\"$test_node_1\" description=\"test node: $test_node_1\" type=\"member\"/>
+ <node uname=\"$test_node_2\" description=\"test node: $test_node_2\" type=\"member\"/>
+ </nodes>
+ <resources/>
+ <constraints/>
+ </configuration>
+ <status/>
+ </cib>
+</cib_fragment>
+"
+
+# make *sure* theres nothing left over from last time
+crm-cleanup
+
+function compare_cibs()
+{
+ cib_cmp_base=$1; shift
+ cib_cmp_host=$1; shift
+ cib_cmp_opts=$*
+ cib_cmp_output=${cib_cmp_base}.out
+ cib_cmp_expected=${cib_cmp_base}.exp
+ cib_cmp_failed=$test_dump_dir/test.txt
+ cib_cmp_diff_opts="--ignore-all-space -U 1 -u"
+
+ do_cmd echo ssh $CRMD_USER@$cib_cmp_host "$HALIB_DIR/cibadmin -Q ${cib_cmp_opts}"
+ remote_cmd $CRMD_USER $cib_cmp_host "$HALIB_DIR/cibadmin -Q ${cib_cmp_opts}" > $cib_cmp_output
+ cts_assert "compare_cibs(): Could not perform query: opts=$cib_cmp_opts"
+
+ gres ' timestamp=\".*\"' "" ${cib_cmp_output}
+ gres ' id=\"[0123456789abcdef-]*\"' "" ${cib_cmp_output}
+
+ # this is the value stored in a variable named $cib_cmp_base
+ echo -e "${!cib_cmp_base}" > $cib_cmp_expected
+
+# if [ "$create_mode" = "true" -a ! -f $cib_cmp_expected ]; then
+# cp "$cib_cmp_output" "$cib_cmp_expected"
+# fi
+
+ if [ -f $cib_cmp_expected ]; then
+ diff $cib_cmp_diff_opts -q $cib_cmp_expected $cib_cmp_output >/dev/null
+ rc=$?
+ fi
+
+# if [ "$create_mode" = "true" ]; then
+# echo "Test $cib_cmp_base... Created expected output"
+# el
+ if [ ! -f $cib_cmp_expected ]; then
+ echo "==== Raw results for CIB test ($cib_cmp_base) ====" >> $cib_cmp_failed
+ cat $cib_cmp_output 2>/dev/null >> $cib_cmp_failed
+ cat $cib_cmp_output
+
+ elif [ "$rc" = 0 ]; then
+ do_cmd echo "Test $cib_cmp_base... Passed";
+ elif [ "$rc" = 1 ]; then
+ do_cmd echo "Test $cib_cmp_base... * Failed";
+ diff $cib_cmp_diff_opts $cib_cmp_expected $cib_cmp_output 2>/dev/null >> $cib_cmp_failed
+ diff $cib_cmp_diff_opts $cib_cmp_expected $cib_cmp_output
+ else
+ do_cmd echo "Test $cib_cmp_base... Error (diff: $rc)";
+ do_cmd echo "==== Raw results for test ($cib_cmp_base) ====" >> $cib_cmp_failed
+ cat $cib_cmp_output 2>/dev/null >> $cib_cmp_failed
+ cat $cib_cmp_output
+ fi
+
+ rm $cib_cmp_output .gres.$cib_cmp_output $cib_cmp_expected
+
+ do_cts_assert $rc 0 "diff for $cib_cmp_base found differences"
+}
+
+
+#----
+
+do_cmd echo "wait for HA to start"
+crm_log_pos=$(stat -L -c %s $logfile)
+do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" &
+do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" &
+do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" &
+do_cmd ${testdir}/testutils.pl -l ${logfile} -p $crm_log_pos --search -a -m 1500 \
+ -s "${test_node_1} heartbeat(.*) info: Starting(.*)cib" \
+ -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" \
+ -s "${test_node_2} heartbeat(.*) info: Starting(.*)cib" \
+ -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" \
+ -s "${test_node_3} heartbeat(.*) info: Starting(.*)cib" \
+ -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast"
+cts_assert "Startup of Heartbeat on ${test_node_1}, ${test_node_2} and ${test_node_3} failed."
+
+#----
+
+do_cmd echo "is master instance?"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -M"
+do_cts_assert $? 35 "CIB service on $test_node_1 should not be in master mode by default"
+
+#----
+
+do_cmd echo "local query"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -l"
+cts_assert "Could not query local CIB service on $test_node_1"
+
+#----
+
+do_cmd echo "queries with hostname"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -h $test_node_2"
+cts_assert "Could not query CIB service on $test_node_2 from $test_node_1"
+
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -l -h $test_node_1"
+cts_assert "Could not query CIB service on $test_node_1 from $test_node_1"
+
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -h $test_node_1"
+cts_assert "Could not query CIB service on $test_node_1 from $test_node_1 without -l flag"
+
+#----
+
+do_cmd echo "make sure all instances are slaves"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -r"
+cts_assert "Could not set CIB services to slaves"
+
+#----
+
+do_cmd echo "make local the master instance"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -w"
+cts_assert "Could not set CIB service on $test_node_1 to master"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -M"
+cts_assert "Could not set CIB service on $test_node_1 to master"
+
+#----
+
+do_cmd echo "erase CIB contents and sync"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -E"
+cts_assert "Could not erase CIB contents"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -S"
+cts_assert "Could not sync CIB to all nodes"
+
+#----
+
+do_cmd echo "verify the CIB contents"
+compare_cibs erase_sync_1 $test_node_1 -l
+compare_cibs erase_sync_2 $test_node_1 -h $test_node_2
+compare_cibs erase_sync_3 $test_node_1 -l -h $test_node_3
+
+#----
+
+do_cmd echo "make some nodes in the CIB"
+do_cmd make_node $test_node_1 $test_node_1 "member"
+cts_assert "Could not create $test_node_1"
+do_cmd make_node $test_node_1 $test_node_2 "member"
+cts_assert "Could not create $test_node_2"
+
+#----
+
+do_cmd echo "verify the CIB contents"
+compare_cibs create_1 $test_node_1 -l
+compare_cibs create_2 $test_node_2 -l
+compare_cibs create_3 $test_node_3 -l
+
+#----
+
+do_cmd make_node_local $test_node_1 $test_node_3 "member"
+cts_assert "Could not create $test_node_3 locally"
+
+#----
+
+do_cmd echo "verify the CIB contents on all nodes"
+compare_cibs create_local_1 $test_node_1 -l
+compare_cibs create_local_2 $test_node_2 -l
+compare_cibs create_local_3 $test_node_3 -l
+
+#----
+
+do_cmd echo "make all slaves"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -r"
+cts_assert "Could not set CIB services to slaves"
+
+do_cmd echo "make node2 master"
+do_cmd remote_cmd $CRMD_USER $test_node_2 "$HALIB_DIR/cibadmin -w"
+cts_assert "Could not set CIB service on $test_node_1 to master"
+
+#----
+
+do_cmd echo "bump master twice (so we can use it in a sync later)"
+
+#----
+
+do_cmd remote_cmd $CRMD_USER $test_node_2 "$HALIB_DIR/cibadmin -B"
+cts_assert "Could not perform bump on $test_node_2"
+
+do_cmd echo "verify the CIB contents on all nodes"
+compare_cibs bump_1 $test_node_1 -l
+compare_cibs bump_2 $test_node_2 -l
+compare_cibs bump_3 $test_node_3 -l
+
+#----
+
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -B"
+cts_assert "Could not perform bump on $test_node_2 via $test_node_1"
+
+do_cmd echo "verify the CIB contents on all nodes"
+compare_cibs bump_remote_1 $test_node_1 -l
+compare_cibs bump_remote_2 $test_node_2 -l
+compare_cibs bump_remote_3 $test_node_3 -l
+
+#----
+
+do_cmd echo "make all slaves"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -r"
+cts_assert "Could not set CIB services to slaves"
+
+do_cmd echo "make node1 master"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -w"
+cts_assert "Could not set CIB service on $test_node_1 to master"
+
+#----
+
+do_cmd echo "sync from node2"
+do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -S -h $test_node_2"
+cts_assert "Could not sync from $test_node_2"
+
+#----
+
+do_cmd echo "verify the CIB contents on all nodes"
+compare_cibs sync_from_1 $test_node_1 -l
+compare_cibs sync_from_2 $test_node_2 -l
+compare_cibs sync_from_3 $test_node_3 -l
+
+echo "test: PASSED"
+
diff --git a/crm/test/helper.sh.in b/crm/test/helper.sh.in
index ad7f9a3ba4..6b3b040cbf 100644
--- a/crm/test/helper.sh.in
+++ b/crm/test/helper.sh.in
@@ -1,468 +1,518 @@
#!/bin/bash
#
# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
[[ $0 != *helper.sh ]] || {
echo >&2 "helper.sh expects to be sourced!"
exit 1
}
# no more exit below here, use return instead; gives more flexibility ...
test_type=`basename $0`
test_nodes=1
dump_dir=~/BUILD/tests
HALIB_DIR=@libdir@/heartbeat
HAVAR_DIR=@HA_VARLIBDIR@/heartbeat
HAINIT_DIR=@INITDIR@
INIT_USER=root
CRMD_USER=root
ADMIN_USER=root
test_node_1=fish
test_node_2=chips
test_node_3=mayo
CRM_OPTS="-VV"
CRM_ERR_SHUTDOWN=0
OUTPUT_STDERR=0
OUTPUT_STDOUT=0
OUTPUT_CMDS=0
OUTPUT_ECHO=1
repeats=1
iteration=0
done=0
ip_rsc_1=192.168.9.161
ip_rsc_2=192.168.9.162
logfile=/var/log/messages
if [ ! -z "$CRM_LOCAL_OPTS" -a -f "$CRM_LOCAL_OPTS" ]; then
. "$CRM_LOCAL_OPTS"
fi
while test "$done" = "0"; do
case "$1" in
--debug | -x) set -x; shift;;
--silent | -s) OUTPUT_ECHO=0; shift;;
--stdout | -o) OUTPUT_STDOUT=1; shift;;
--stderr | -e) OUTPUT_STDERR=1; shift;;
--verbose | -v) OUTPUT_CMDS=1; shift;;
--logfile | -l) logfile=${2?no logfile??} shift 2;;
--dumpdir | -d) dump_dir=${2?no dumpdir??} shift 2;;
--repeat | -r) repeats=${2?no max repetitions??} || return 1; shift 2;;
--init | -i) INIT_USER=${2?no init user??} || return 1; shift 2;;
--admin | -a) ADMIN_USER=${2?no admin??} || return 1; shift 2;;
--crmd | -c) CRMD_USER=${2?no crmd user??} || return 1; shift 2;;
--node1 | -1) test_node_1=${2?no node1??} || return 1; shift 2;;
--node2 | -2) test_node_2=${2?no node2??} || return 1; shift 2;;
--node3 | -3) test_node_3=${2?no node2??} || return 1; shift 2;;
--ip1) ip_rsc_1=${2?no ip1??} || return 1; shift 2;;
--ip2) ip_rsc_2=${2?no ip2??} || return 1; shift 2;;
""|--) done=1;;
*) echo "unknown option: $1"; return 1;;
esac
done
function test_init() {
export date_str=`date "+%Y-%b-%d_%H:%M"`
export test_dump_dir="$dump_dir/$test_type/$date_str"
mkdir -p "$test_dump_dir"
echo Test dir: $test_dump_dir
}
function remote_cmd() {
user=$1
host=$2
shift
shift
`which ssh` ${user}@${host} "$*"
return $?;
}
function is_state() {
ret=1
host=$1
state=$2
output=`remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host | grep $state`
if [ ! -z "$output" ]; then
echo "$host is in $state"
return 0
fi
echo "$host is NOT in $state"
return $ret
}
function is_running() {
rsc=$1
askhost=$2
host=$3
if [ -z $host ]; then
output=`remote_cmd $ADMIN_USER $askhost $HALIB_DIR/crmadmin -W $rsc | grep -v $HALIB_DIR/crmadmin`
output=`echo $output | grep -v NOT`
else
output=`remote_cmd $ADMIN_USER $askhost $HALIB_DIR/crmadmin -W $rsc | grep -v $HALIB_DIR/crmadmin`
output=`echo $output | grep $host`
fi
if [ -z "$output" ]; then
if [ -z $host ]; then
echo "$rsc is NOT running"
else
echo "$rsc is NOT running on host $host"
fi
return 1
fi
if [ -z $host ]; then
echo "$rsc is running"
else
echo "$rsc is running on host $host"
fi
return 0
}
function is_dc() {
host=$1
output=`remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host | grep -v S_PENDING | grep -v S_NOT_DC | grep -v S_ELECTION | grep -v S_RECOVERY`
if [ ! -z "$output" ]; then
echo "$host is DC"
return 0
fi
echo "$host is NOT DC"
remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host
return 1
}
function wait_for_state() {
state=$1
max=$2
host=$3
target=$4
sleep 1
if [ "$host" = "DC" ]; then
host=`remote_cmd $ADMIN_USER $target $HALIB_DIR/crmadmin -D`
host=${host#*: }
fi
count=1
until is_state $host $state; do
if [ $count -gt $max ]; then
echo "Attempt to reach $state on $host failed"
return 1
fi
count=`expr $count + 1`
sleep 1
done
echo "$state reached on $host"
return 0
}
function make_node()
{
host=$1
uname=$2
type=$3
uuid=`uuidgen`
if [ -z $type ]; then
type="member"
fi
node_xml="'<node id=\"${uuid}\" uname=\"${uname}\" description=\"test node: ${uname}\" type=\"${type}\"/>'";
remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -C -o node -X $node_xml"
}
+function make_node_local()
+{
+ host=$1
+ uname=$2
+ type=$3
+ uuid=`uuidgen`
+
+ if [ -z $type ]; then
+ type="member"
+ fi
+
+ node_xml="'<node id=\"${uuid}\" uname=\"${uname}\" description=\"test node: ${uname}\" type=\"${type}\"/>'";
+
+ remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -l -C -o node -X $node_xml"
+}
+
function make_resource()
{
host=$1
id=$2
class=$3
type=$4
priority=$5
version=$6
stopfail=$7
shift 7
args=$*
uuid=`uuidgen`
node_xml="'<resource id=\"${id}\" class=\"${class}\" type=\"${type}\""
if [ ! -z $version -a "x$version" != "x-" ]; then
node_xml="$node_xml version=\"${version}\""
fi
if [ ! -z $priority -a "x$priority" != "x-" ]; then
node_xml="$node_xml priority=\"${priority}\""
fi
if [ ! -z $stopfail -a "x$stopfail" != "x-" ]; then
node_xml="$node_xml on_stopfail=\"${stopfail}\""
fi
node_xml="$node_xml >";
if [ ! -z "$args" -a "x$args" != "x-" ]; then
node_xml="$node_xml <instance_attributes><rsc_parameters>${args}</rsc_parameters></instance_attributes>"
fi
node_xml="$node_xml </resource>'";
remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -C -o resource -X $node_xml" 2>&1 > /dev/null
}
function make_constraint_adv()
{
host=$1
shift
node_xml=$*
node_xml=`echo $node_xml | tr '\n' ' '`
do_cmd $QUIET remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -C -o constraint -X $node_xml" 2>&1 > /dev/null
}
function make_constraint()
{
host=$1
rsc=$2
result=$3
uuid1=`uuidgen`
uuid2=`uuidgen`
node_xml="'<rsc_location id=\"${uuid1}\" rsc=\"${rsc}\"><rule id=\"${uuid2}\" result=\"${result}\"/></rsc_location>'"
make_constraint_adv $host $node_xml
}
function cts_assert() {
do_cts_assert $? 0 "Assert failed - " $*
}
function cts_assert_false() {
do_cts_assert $? 1 "AssertFalse failed - " $*
}
function do_cts_assert() {
result=$1
expected=$2
shift
shift
if [ $result -ne $expected ]; then
+ do_cmd echo "Expected rc ($expected) != $result"
err $*
fi
}
function err() {
if [ ! -z $HOST ]; then
if [ $CRM_ERR_SHUTDOWN -eq 1 ]; then
remote_cmd $ADMIN_USER $test_node_1 killall -9 crmd
remote_cmd $ADMIN_USER $test_node_2 killall -9 crmd
fi
fi
do_cmd echo Dumping test nodes to $dump_dir
dump_node 1 $test_node_1
dump_node 2 $test_node_2
dump_node 3 $test_node_3
do_cmd echo "ERROR: $*"
echo "test $test_type: FAILED"
echo "ERROR: $*" > $test_dump_dir/error.txt
exit 1
}
function dump_node() {
a_node_index=$1
a_node=$2
- dump_file="${a_node_index}-${a_node}.tgz"
+ dump_file="/tmp/${a_node_index}-${a_node}.tgz"
`which ssh` ${ADMIN_USER}@${a_node} "ps axf > /tmp/crm/processes"
- `which ssh` ${ADMIN_USER}@${a_node} \
- "tar zcvf /tmp/${dump_file} /tmp/crm ~${CRMD_USER}/core"
-
- scp -r ${ADMIN_USER}@${a_node}:/tmp/${dump_file} $test_dump_dir
+ `which ssh` ${ADMIN_USER}@${a_node} "tar zcvf ${dump_file} /tmp/crm"
+ scp -r ${ADMIN_USER}@${a_node}:${dump_file} $test_dump_dir
+ `which ssh` ${ADMIN_USER}@${a_node} "rm ${dump_file}"
}
function do_cmd() {
cmdline=$*
old_OUTPUT_STDOUT=$OUTPUT_STDOUT
cmd=$1
if [ $OUTPUT_STDERR -eq 1 -o $OUTPUT_STDOUT -eq 1 -o $OUTPUT_CMDS -eq 1 ]; then
if [ $cmd = "remote_cmd" ]; then
user=$2
host=$3
shift 3
echo `date` ": Running '$*' as ${user}@${host}" | tee -a $test_dump_dir/test.txt;
elif [ $cmd = "wait_for_state" ]; then
host=$4
if [ -z $host ]; then
host="the DC"
fi
echo `date` ": Waiting for state $2 on $host (max $3 attempts)" | tee -a $test_dump_dir/test.txt;
elif [ $cmd = "${HALIB_DIR}/crmtest/testutils.pl" ]; then
echo `date` ": Searching '$cmdline'" | tee -a $test_dump_dir/test.txt;
elif [ $cmd = "echo" ]; then
echo -n ""
# if [ $OUTPUT_CMDS -eq 1 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then
# echo -n `date` ": "
# $cmdline
# elif [ $OUTPUT_ECHO -eq 1 -a $OUTPUT_CMDS -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then
# $cmdline
# fi
else
echo `date` ": Running '$cmdline'" | tee -a $test_dump_dir/test.txt;
fi
fi
if [ $cmd = "echo" ]; then
if [ $OUTPUT_CMDS -eq 1 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then
echo -n `date` ": " | tee -a $test_dump_dir/test.txt
echo -n -e "\033[01;33m"
$cmdline | tee -a $test_dump_dir/test.txt
echo -n -e "\033[00m"
elif [ $OUTPUT_ECHO -eq 1 -a $OUTPUT_CMDS -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then
echo -n `date` ": " | tee -a $test_dump_dir/test.txt
echo -n -e "\033[01;33m"
$cmdline | tee -a $test_dump_dir/test.txt
echo -n -e "\033[00m"
fi
elif [ $OUTPUT_STDERR -eq 1 -a $OUTPUT_STDOUT -eq 1 ]; then
$cmdline 2>&1 >> $test_dump_dir/test.txt
rc=$?
elif [ $OUTPUT_STDERR -eq 0 -a $OUTPUT_STDOUT -eq 0 ]; then
$cmdline 2>&1 >/dev/null
rc=$?
elif [ $OUTPUT_STDOUT -eq 0 ]; then
$cmdline >/dev/null 2>> $test_dump_dir/test.txt
rc=$?
elif [ $OUTPUT_STDERR -eq 0 ]; then
$cmdline 2>/dev/null >> $test_dump_dir/test.txt
rc=$?
else
$cmdline 2>&1 >> $test_dump_dir/test.txt
rc=$?
fi
if [ $cmd = "${HALIB_DIR}/crmtest/testutils.pl" ]; then
# for the logs...
if [ $OUTPUT_STDERR -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_CMDS -eq 0 ]; then
echo `date` ": Searching '$cmdline'" >> $test_dump_dir/test.txt;
fi
$cmdline -v 2>&1 >> $test_dump_dir/test.txt
if [ $rc -lt 0 ]; then
echo `date` ": Searching '$cmdline'"
$cmdline
fi
fi
OUTPUT_STDOUT=$old_OUTPUT_STDOUT
# echo Result: $rc
return $rc
}
function crm-cleanup() {
if [ -z $1 ]; then
test_init
fi
do_cmd echo Cleaning up on $test_node_1, $test_node_2 and $test_node_3
crm-cleanup-node $test_node_1 $1
crm-cleanup-node $test_node_2 $1
crm-cleanup-node $test_node_3 $1
}
function crm-cleanup-node() {
ha_host=$1
do_cmd echo Cleaning up on $ha_host
# stop all running HAs
do_cmd remote_cmd $INIT_USER $ha_host $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null"
# be *very* sure everything has stopped
do_cmd remote_cmd $INIT_USER $ha_host "killall -q9 heartbeat ccm lrmd crmd"
# resources too
do_cmd remote_cmd $INIT_USER $ha_host "/etc/ha.d/resource.d/IPaddr $ip_rsc_1 stop" 2>&1 > /dev/null
do_cmd remote_cmd $INIT_USER $ha_host "/etc/ha.d/resource.d/IPaddr $ip_rsc_2 stop" 2>&1 > /dev/null
if [ -z $2 ]; then
# make *sure* theres nothing left over from last time
do_cmd remote_cmd $INIT_USER $ha_host "rm -f $HAVAR_DIR/crm/cib*.xml"
- remote_cmd $INIT_USER $ha_host "rm -f /tmp/crm/*" 2>&1 > /dev/null
- remote_cmd $CRMD_USER $ha_host "touch /tmp/crm/cluster.log"
+ `which ssh` ${INIT_USER}@${ha_host} "rm -f /tmp/crm/*" 2>/dev/null
remote_cmd $CRMD_USER $ha_host "rm -f /tmp/crm/*/core" 2>&1 > /dev/null
do_cmd remote_cmd $INIT_USER $ha_host "/etc/init.d/syslog restart"
+ do_cmd remote_cmd $CRMD_USER $ha_host "logger -i -p local7.info __crmtest_${test_type}_log_mark__"
+ do_cmd sleep 2
# zero out logs, maybe
if [ ${test_node_1} = "fish" ]; then
do_cmd remote_cmd $INIT_USER $ha_host "cat /dev/null > /var/log/messages"
fi
fi
}
function mark_log()
{
export crm_log_pos=$(stat -L -c %s $logfile)
}
+function gres()
+{
+ if [ $# -lt "3" ]
+ then
+ echo Usage: gres pattern replacement files
+ return 1
+ fi
+ pattern=$1
+ replacement=$2
+ shift; shift;
+ filelist=$*
+ A="`echo | tr '\012' '\001' `"
+
+ for file in $filelist; do
+
+ fname=`basename $file`
+ dname=`dirname $file`
+
+ is_backup=${fname##.gres.}
+ if [ "$is_backup" = "$fname" ]; then
+ grep -q "$pattern" "$file"
+ isthere=$?
+
+ if [ $isthere = 0 ]; then
+# echo Processing $file...
+ cp "$file" "$dname/.gres.$fname" && sed -e "s$A$pattern$A$replacement$A" "$dname/.gres.$fname" > "$file"
+ fi
+# else
+# echo "Ignoring backup of $is_backup in $dname"
+ fi
+ done
+}
+
# successfully sourced
true