diff --git a/crm/test/2node-fail.sh.in b/crm/test/2node-fail.sh.in index 8abe8be6d4..fc3e53fe3b 100644 --- a/crm/test/2node-fail.sh.in +++ b/crm/test/2node-fail.sh.in @@ -1,342 +1,337 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # required_nodes=2 testdir=@libdir@/heartbeat/crmtest . ${testdir}/helper.sh || exit 1 CRM_ERR_SHUTDOWN=0 test_nodes=2 function 2node_fail_test() { test_type=$1 fail_pieces=$2 fail_node=$3 good_node=$4 do_failback=$5 accelerated=$6 if [ $fail_node = $test_node_1 ]; then moved_rsc=rsc1 else moved_rsc=rsc2 fi #---- echo -ne "\033]0;$test_type: Iteration $iteration of $repeats\007" crm-cleanup do_cmd echo "#############################" do_cmd echo "$test_type: Iteration $iteration of $repeats" # make *sure* theres nothing left over from last time #---- do_cmd echo "wait for HA to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." #---- do_cmd echo "wait for CRMd to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd wait_for_state S_IDLE 3 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup)!" #---- do_cmd echo Create the first constraint and wait for S_IDLE rsc=rsc1 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml=" " crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert Adding constraint1 did not pass #---- do_cmd echo Create the second constraint and wait for S_IDLE rsc=rsc2 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml=" " crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert Adding constraint2 did not pass #---- do_cmd echo Create the first resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - ignore $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" \ - -s "crmd(.*) Resource state: rsc1(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc1" cts_assert Adding rsc1 did not pass #---- do_cmd echo Create the second resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - ignore $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" \ - -s "crmd(.*) Resource state: rsc2(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc2" cts_assert Adding rsc2 did not pass #---- do_cmd echo Various sanity checks - stage 1 do_cmd wait_for_state S_IDLE 3 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_dc $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd is_running rsc1 $test_node_1 x$test_node_1 cts_assert_false "rsc1 IS running on x$test_node_1" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_1 cts_assert "rsc2 NOT running on $test_node_1" #---- do_cmd echo "wait for HA to start on $test_node_2" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" \ -s "${test_node_2} heartbeat(.*) info: Starting (.*)lrmd" \ -e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." #---- do_cmd echo "wait for CRMd to start on $test_node_2" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" \ -s "${test_node_2} crmd(.*) State transition S_PENDING \-> S_NOT_DC" cts_assert "CRMd startup on ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 8000 \ -s "${test_node_1} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" \ - -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}" + -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" cts_assert "rsc2 was not transferred to ${test_node_2} on startup." #---- do_cmd echo Various sanity checks - stage 2 do_cmd wait_for_state S_NOT_DC 30 $test_node_2 cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- is_dc $fail_node 2>&1 > /dev/null test_for_election=$? do_cmd echo Killing $fail_pieces on $fail_node crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $ADMIN_USER $fail_node "killall -9 $fail_pieces" & if [ $test_for_election = 0 ]; then do_cmd echo Killed the DC... checking for DC Failover do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90000 \ -s "${good_node} crmd(.*) State transition" \ -s "${good_node} crmd(.*) State transition (.*) \-> S_ELECTION" \ -s "${good_node} crmd(.*) State transition (.*) \-> S_IDLE" cts_assert "Transition of the DC from ${fail_node} to ${good_node} failed." else do_cmd echo Killed slave node... checking the DC noticed do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90000 \ -s "${good_node} crmd(.*) State transition" \ -s "${good_node} crmd(.*) State transition (.*) \-> S_POLICY_ENGINE" \ -s "${good_node} crmd(.*) State transition (.*) \-> S_IDLE" cts_assert "Failure of slave node ${fail_node} was not noticed on the DC (${good_node})." fi do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 80000 \ - -s "${god_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" \ - -s "crmd(.*) Resource state: ${moved_rsc}(.*) after start] on ${good_node}" + -s "${god_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" cts_assert "Move of ${moved_rsc} to ${good_node} failed." #---- do_cmd echo Various sanity checks - stage 3 do_cmd wait_for_state S_IDLE 60 $good_node cts_assert "S_IDLE not reached on $good_node after kill!" do_cmd is_running rsc1 $good_node cts_assert "rsc1 NOT running on $good_node" do_cmd is_running rsc2 $good_node cts_assert "rsc2 NOT running on $good_node" do_cmd is_running rsc1 $good_node $fail_node cts_assert_false "rsc1 IS running on $fail_node" do_cmd is_running rsc2 $good_node $good_node cts_assert "rsc2 NOT running on $good_node" #---- if [ $do_failback = 1 ]; then do_cmd echo "Re-Starting on failed node $fail_node" if [ "$fail_pieces" = "crmd" ]; then do_cmd echo "HA still running, skipping restart" else do_cmd echo "wait for HA to start on $fail_node" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $fail_node $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${fail_node} ccm(.*) Hostname: ${fail_node}" \ -s "${fail_node} heartbeat(.*) info: Starting (.*)lrmd" \ -e "${fail_node} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${fail_node} failed." fi #---- do_cmd echo "wait for CRMd to start on $fail_node" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $fail_node $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 20000 \ -s "${fail_node} crmd(.*)FSA Hostname: ${fail_node}" \ -s "${fail_node} crmd(.*) State transition S_PENDING \-> S_NOT_DC" cts_assert "CRMd startup on ${fail_node} failed." do_cmd echo "Checking ${moved_rsc} was moved back" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 20000 \ -s "${good_node} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${fail_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" \ - -s "crmd(.*) Resource state: ${moved_rsc}(.*) after start] on ${fail_node}" + -s "${fail_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" cts_assert "$moved_rsc was not transferred to ${fail_node} on startup." do_cmd echo Various sanity checks - stage 4 do_cmd wait_for_state S_NOT_DC 30 $fail_node cts_assert "S_NOT_DC not reached on $fail_node (restart)!" do_cmd wait_for_state S_IDLE 30 $good_node cts_assert "S_IDLE not reached on $good_node (restart)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" fi #---- do_cmd echo "test ${test_type}: PASSED" } while [ $iteration -lt $repeats ]; do iteration=`expr $iteration + 1` echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" echo "########### $test_type : Begining iteration $iteration of $repeats ###########" ( 2node_fail_test 2node__failback_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 1 0 2node_fail_test 2node__fail_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 0 0 2node_fail_test 2node__fail_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 0 0 2node_fail_test 2node__failback_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 1 0 2node_fail_test 2node__failback_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 1 0 2node_fail_test 2node__fail_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 0 0 2node_fail_test 2node__fail_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 0 0 2node_fail_test 2node__failback_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 1 0 2node_fail_test 2node__failback_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 1 0 #2node_fail_test 2node__fail_slave_LRM "lrmd" ${test_node_2} ${test_node_1} #2node_fail_test 2node__fail_DC_LRM "lrmd" ${test_node_1} ${test_node_2} echo "test suite: PASSED" ) done diff --git a/crm/test/2node-fence.sh.in b/crm/test/2node-fence.sh.in index 09396762cd..5a56d64f2b 100644 --- a/crm/test/2node-fence.sh.in +++ b/crm/test/2node-fence.sh.in @@ -1,353 +1,347 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # required_nodes=2 testdir=@libdir@/heartbeat/crmtest . ${testdir}/helper.sh || exit 1 CRM_ERR_SHUTDOWN=0 test_nodes=2 function 2node_fail_test() { test_type=$1 fail_pieces=$2 fail_node=$3 good_node=$4 do_failback=$5 accelerated=$6 if [ $fail_node = $test_node_1 ]; then moved_rsc=rsc1 else moved_rsc=rsc2 fi #---- echo -ne "\033]0;$test_type: Iteration $iteration of $repeats\007" crm-cleanup do_cmd echo "#############################" do_cmd echo "$test_type: Iteration $iteration of $repeats" # make *sure* theres nothing left over from last time #---- do_cmd echo "wait for HA to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." #---- do_cmd echo "wait for CRMd to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd wait_for_state S_IDLE 3 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup)!" #---- do_cmd echo Create the first constraint and wait for S_IDLE rsc=rsc1 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert Adding constraint1 did not pass #---- do_cmd echo Create the second constraint and wait for S_IDLE rsc=rsc2 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert Adding constraint2 did not pass #---- do_cmd echo Create the first resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - stonith $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" \ - -s "crmd(.*) Resource state: rsc1(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc1" cts_assert Adding rsc1 did not pass #---- do_cmd echo Create the second resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - stonith $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" \ - -s "crmd(.*) Resource state: rsc2(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc2" cts_assert Adding rsc2 did not pass #---- do_cmd echo Create the fencing resource and wait for S_IDLE after start crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_incarnation $test_node_1 fence1 stonith null 2 - - ignore do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on fence1" \ - -s "crmd(.*) Resource state: fence1(.*) after start" + -s "crmd(.*) Performing op start(.*) on fence1" cts_assert Adding fence1 did not pass #---- do_cmd echo Various sanity checks - stage 1 do_cmd wait_for_state S_IDLE 3 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_dc $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd is_running rsc1 $test_node_1 x$test_node_1 cts_assert_false "rsc1 IS running on x$test_node_1" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_1 cts_assert "rsc2 NOT running on $test_node_1" #---- do_cmd echo "wait for HA to start on $test_node_2" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" \ -s "${test_node_2} heartbeat(.*) info: Starting (.*)lrmd" \ -e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." #---- do_cmd echo "wait for CRMd to start on $test_node_2" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" \ -s "${test_node_2} crmd(.*) State transition S_PENDING \-> S_NOT_DC" cts_assert "CRMd startup on ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 8000 \ -s "${test_node_1} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" \ - -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}" + -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" cts_assert "rsc2 was not transferred to ${test_node_2} on startup." #---- do_cmd echo Various sanity checks - stage 2 do_cmd wait_for_state S_NOT_DC 30 $test_node_2 cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- is_dc $fail_node 2>&1 > /dev/null test_for_election=$? do_cmd echo Killing $fail_pieces on $fail_node crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $ADMIN_USER $fail_node "killall -9 $fail_pieces" & if [ $test_for_election = 0 ]; then do_cmd echo Killed the DC... checking for DC Failover do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90000 \ -s "${good_node} crmd(.*) State transition" \ -s "${good_node} crmd(.*) State transition (.*) \-> S_ELECTION" \ -s "${good_node} crmd(.*) State transition (.*) \-> S_IDLE" cts_assert "Transition of the DC from ${fail_node} to ${good_node} failed." else do_cmd echo Killed slave node... checking the DC noticed do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90000 \ -s "${good_node} crmd(.*) State transition" \ -s "${good_node} crmd(.*) State transition (.*) \-> S_POLICY_ENGINE" \ -s "${good_node} crmd(.*) State transition (.*) \-> S_IDLE" cts_assert "Failure of slave node ${fail_node} was not noticed on the DC (${good_node})." fi do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 80000 \ - -s "${god_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" \ - -s "crmd(.*) Resource state: ${moved_rsc}(.*) after start] on ${good_node}" + -s "${god_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" cts_assert "Move of ${moved_rsc} to ${good_node} failed." #---- do_cmd echo Various sanity checks - stage 3 do_cmd wait_for_state S_IDLE 60 $good_node cts_assert "S_IDLE not reached on $good_node after kill!" do_cmd is_running rsc1 $good_node cts_assert "rsc1 NOT running on $good_node" do_cmd is_running rsc2 $good_node cts_assert "rsc2 NOT running on $good_node" do_cmd is_running rsc1 $good_node $fail_node cts_assert_false "rsc1 IS running on $fail_node" do_cmd is_running rsc2 $good_node $good_node cts_assert "rsc2 NOT running on $good_node" #---- if [ $do_failback = 1 ]; then do_cmd echo "Re-Starting on failed node $fail_node" if [ "$fail_pieces" = "crmd" ]; then do_cmd echo "HA still running, skipping restart" else do_cmd echo "wait for HA to start on $fail_node" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $fail_node $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${fail_node} ccm(.*) Hostname: ${fail_node}" \ -s "${fail_node} heartbeat(.*) info: Starting (.*)lrmd" \ -e "${fail_node} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${fail_node} failed." fi #---- do_cmd echo "wait for CRMd to start on $fail_node" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $fail_node $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 20000 \ -s "${fail_node} crmd(.*)FSA Hostname: ${fail_node}" \ -s "${fail_node} crmd(.*) State transition S_PENDING \-> S_NOT_DC" cts_assert "CRMd startup on ${fail_node} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 20000 \ -s "${good_node} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${fail_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" \ - -s "crmd(.*) Resource state: ${moved_rsc}(.*) after start] on ${fail_node}" + -s "${fail_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" cts_assert "$moved_rsc was not transferred to ${fail_node} on startup." do_cmd echo Various sanity checks - stage 4 do_cmd wait_for_state S_NOT_DC 30 $fail_node cts_assert "S_NOT_DC not reached on $fail_node (restart)!" do_cmd wait_for_state S_IDLE 30 $good_node cts_assert "S_IDLE not reached on $good_node (restart)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" fi #---- do_cmd echo "test ${test_type}: PASSED" } while [ $iteration -lt $repeats ]; do iteration=`expr $iteration + 1` echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" echo "########### $test_type : Begining iteration $iteration of $repeats ###########" ( 2node_fail_test 2node__fail_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 0 0 2node_fail_test 2node__fail_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 0 0 2node_fail_test 2node__failback_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 1 0 2node_fail_test 2node__failback_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 1 0 2node_fail_test 2node__fail_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 0 0 2node_fail_test 2node__fail_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 0 0 2node_fail_test 2node__failback_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 1 0 2node_fail_test 2node__failback_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 1 0 #2node_fail_test 2node__fail_slave_LRM "lrmd" ${test_node_2} ${test_node_1} #2node_fail_test 2node__fail_DC_LRM "lrmd" ${test_node_1} ${test_node_2} echo "test suite: PASSED" ) done diff --git a/crm/test/2node.sh.in b/crm/test/2node.sh.in index 1253d2e15d..b7ffe26961 100644 --- a/crm/test/2node.sh.in +++ b/crm/test/2node.sh.in @@ -1,225 +1,217 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # required_nodes=2 testdir=@libdir@/heartbeat/crmtest . ${testdir}/helper.sh || exit 1 CRM_ERR_SHUTDOWN=0 test_nodes=2 while [ $iteration -lt $repeats ]; do iteration=`expr $iteration + 1` echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" echo "########### Begining iteration $iteration of $repeats ###########" ( # make *sure* theres nothing left over from last time crm-cleanup #---- start_node $test_node_1 #---- do_cmd echo "Analyse CRMd startup" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90 \ -s "crmd(.*) State transition (.*) \-> S_ELECTION" \ -s "crmd(.*) State transition (.*) \-> S_INTEGRATION" cts_assert "DC election on ${test_node_1} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "Updating node state to member for ${test_node_1}" \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert "IDLE and CRMd join on ${test_node_1} failed." do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup)!" #---- do_cmd echo Create the first constraint and wait for S_IDLE rsc=rsc1 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml=" " crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 100 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" #cts_assert Adding constraint1 did not pass #---- do_cmd echo Create the second constraint and wait for S_IDLE rsc=rsc2 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml=" " crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 100 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" #cts_assert Adding constraint2 did not pass #---- do_cmd echo Create the first resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 200 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" \ - -s "crmd(.*) Resource state: rsc1(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc1" cts_assert Adding rsc1 did not pass #---- do_cmd echo Create the second resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 200 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" \ - -s "crmd(.*) Resource state: rsc2(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc2" cts_assert Adding rsc2 did not pass #---- do_cmd echo Various sanity checks - stage 1 do_cmd wait_for_state S_IDLE 3 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_dc $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd is_running rsc1 $test_node_1 x$test_node_1 cts_assert_false "rsc1 IS running on x$test_node_1" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_1 cts_assert "rsc2 NOT running on $test_node_1" #---- start_node $test_node_2 #---- do_cmd echo "Analyse CRMd startup" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 400 \ -s "${test_node_2} crmd(.*) State transition S_PENDING \-> S_NOT_DC" cts_assert "CRMd startup on ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2000 \ -s "${test_node_1} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" \ - -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}" + -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" cts_assert "rsc2 was not transferred to ${test_node_2} on startup." #---- do_cmd echo Various sanity checks - stage 2 do_cmd wait_for_state S_NOT_DC 30 $test_node_2 cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- stop_node $test_node_1 crmd do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2000 \ -s "${test_node_2} crmd(.*) State transition (.*) \-> S_ELECTION" \ -s "${test_node_1} crmd(.*) State transition (.*) \-> S_NOT_DC" cts_assert "Transition of the DC from ${test_node_1} to ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 4000 \ - -s "crmd(.*) Performing op start(.*) on rsc1" \ - -s "crmd(.*) Resource state: rsc1(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc1" cts_assert "rsc1 was not transferred to ${test_node_2} on shutdown." #---- do_cmd echo Various sanity checks - stage 3 do_cmd wait_for_state S_IDLE 60 $test_node_2 cts_assert "S_IDLE not reached on $test_node_2 after shutdown!" do_cmd is_running rsc1 $test_node_2 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_2 cts_assert "rsc2 NOT running" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 12000 \ - -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_2}" do_cmd is_running rsc1 $test_node_2 $test_node_1 cts_assert_false "rsc1 IS running on $test_node_1" do_cmd is_running rsc2 $test_node_2 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- stop_node $test_node_2 crmd #---- do_cmd remote_cmd $CRMD_USER $test_node_2 killall -9 crmd echo "test: PASSED" ) done diff --git a/crm/test/3node-fail.sh.in b/crm/test/3node-fail.sh.in index 90d03a2d54..c2e504bd6c 100644 --- a/crm/test/3node-fail.sh.in +++ b/crm/test/3node-fail.sh.in @@ -1,269 +1,267 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # required_nodes=3 testdir=@libdir@/heartbeat/crmtest . ${testdir}/helper.sh || exit 1 CRM_ERR_SHUTDOWN=0 test_nodes=3 while [ $iteration -lt $repeats ]; do iteration=`expr $iteration + 1` echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" echo "########### Begining iteration $iteration of $repeats ###########" ( # make *sure* theres nothing left over from last time crm-cleanup #---- do_cmd echo "wait for HA to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." #---- do_cmd echo "wait for HA to start on ${test_node_2}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_2} ccm(.*) info: Hostname: ${test_node_2}" \ -s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." #---- do_cmd echo "wait for HA to start on ${test_node_3}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_3} ccm(.*) info: Hostname: ${test_node_3}" \ -s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_3} failed." #---- do_cmd echo "wait for CRMd to start on ${test_node_1}, ${test_node_2} and ${test_node_3}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_2} crmd(.*) info:(.*)FSA Hostname: ${test_node_2}" cts_assert "CRMd startup on ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_3} crmd(.*) info:(.*)FSA Hostname: ${test_node_3}" cts_assert "CRMd startup on ${test_node_3} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 25000 \ -s "${test_node_1} crmd(.*) State transition (.*) \-> S_IDLE" \ -s "${test_node_2} crmd(.*) State transition (.*) \-> S_NOT_DC" \ -s "${test_node_3} crmd(.*) State transition (.*) \-> S_NOT_DC" cts_assert "CRMd did not start up into correct modes." #---- do_cmd echo Various sanity checks - stage 1 do_cmd is_dc $test_node_1 $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 DC $test_node_1 cts_assert "S_IDLE not reached on the DC!" #---- do_cmd echo Create the first constraint and wait for S_IDLE rsc=rsc1 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` node_xml=" " crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert Adding constraint1 did not pass #---- do_cmd echo Create the second constraint and wait for S_IDLE rsc=rsc2 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` node_xml=" " crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert Adding constraint2 did not pass #---- do_cmd echo Create the first resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - ignore $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" \ - -s "crmd(.*) Resource state: rsc1(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc1" cts_assert Adding rsc1 did not pass #---- do_cmd echo Create the second resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - ignore $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" \ - -s "crmd(.*) Resource state: rsc2(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc2" cts_assert Adding rsc2 did not pass #---- do_cmd echo Various sanity checks - stage 2 do_cmd is_dc $test_node_1 $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 DC $test_node_1 cts_assert "S_IDLE not reached on the DC!" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Kill CRM on $test_node_1 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $ADMIN_USER $test_node_1 "killall -9 crmd" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*) State transition (.*) \-> S_ELECTION" cts_assert "Transition of the DC away from ${test_node_1} failed." # -s "crmd(.*) State transition (.*) \->" \ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" cts_assert "No DC Declared" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" \ -s "crmd(.*)State transition(.*) \-> S_IDLE" cts_assert "DC not idle" #---- do_cmd echo Various sanity checks - stage 3 do_cmd is_dc $test_node_2 $test_node_2 cts_assert "$test_node_2 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 $test_node_2 cts_assert "S_IDLE not reached on $test_node_2!" do_cmd is_running rsc1 $test_node_2 cts_assert "rsc1 NOT running on $test_node_2" do_cmd is_running rsc2 $test_node_2 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Kill the CRM, CCM and Heartbeat on $test_node_2 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $ADMIN_USER $test_node_2 "killall -9 heartbeat crmd lrmd ccm" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "crmd(.*) State transition S_NOT_DC \-> S_ELECTION" cts_assert "Transition of the DC away from ${test_node_2} failed." # -s "crmd(.*) State transition(.*) \->" \ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" \ -s "crmd(.*)State transition(.*) \-> S_IDLE" cts_assert "No DC Declared" #---- do_cmd echo Various sanity checks - stage 4 do_cmd is_dc $test_node_3 $test_node_3 cts_assert "$test_node_3 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 $test_node_3 cts_assert "S_IDLE not reached on $test_node_3!" do_cmd is_running rsc1 $test_node_3 cts_assert "rsc1 NOT running on $test_node_3" do_cmd is_running rsc2 $test_node_3 $test_node_3 cts_assert "rsc2 NOT running on $test_node_3" #---- do_cmd echo Shutdown $test_node_3 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_3} crmd(.*)State transition (.*) \-> S_PENDING" \ -s "${test_node_3} crmd(.*)State transition S_PENDING \-> S_STOPPING" \ -s "${test_node_3} crmd(.*)\[crmd\] stopped" cts_assert "Shutdown of ${test_node_3} failed." #---- do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd echo "test: PASSED" ) done diff --git a/crm/test/3node-rapid.sh.in b/crm/test/3node-rapid.sh.in index 297b8d4385..b33e55516b 100644 --- a/crm/test/3node-rapid.sh.in +++ b/crm/test/3node-rapid.sh.in @@ -1,290 +1,282 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # required_nodes=3 testdir=@libdir@/heartbeat/crmtest . ${testdir}/helper.sh || exit 1 CRM_ERR_SHUTDOWN=0 test_nodes=3 while [ $iteration -lt $repeats ]; do iteration=`expr $iteration + 1` echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" echo "########### Begining iteration $iteration of $repeats ###########" ( # make *sure* theres nothing left over from last time crm-cleanup #---- do_cmd echo "wait for HA to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." #---- do_cmd echo "wait for HA to start on ${test_node_2}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_2} ccm(.*) info: Hostname: ${test_node_2}" \ -s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." #---- do_cmd echo "wait for HA to start on ${test_node_3}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_3} ccm(.*) info: Hostname: ${test_node_3}" \ -s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_3} failed." #---- do_cmd echo "wait for CRMd to start on ${test_node_1}, ${test_node_2} and ${test_node_3}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_2} crmd(.*) info:(.*)FSA Hostname: ${test_node_2}" cts_assert "CRMd startup on ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_3} crmd(.*) info:(.*)FSA Hostname: ${test_node_3}" cts_assert "CRMd startup on ${test_node_3} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 25000 \ -s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ -s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" \ -s "${test_node_3} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" cts_assert "CRMd did not start up into correct modes." #---- do_cmd echo Various sanity checks - stage 1 do_cmd is_dc $test_node_1 $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 DC $test_node_1 cts_assert "S_IDLE not reached on the DC!" #---- do_cmd echo Create the first constraint and wait for S_IDLE rsc=rsc1 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert Adding constraint1 did not pass #---- do_cmd echo Create the second constraint and wait for S_IDLE rsc=rsc2 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert Adding constraint2 did not pass #---- do_cmd echo Create the first resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ - -s "crmd(.*) Performing op start(.*) on rsc1" \ - -s "crmd(.*) Resource state: rsc1(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc1" cts_assert Adding rsc1 did not pass #---- do_cmd echo Create the second resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ - -s "crmd(.*) Performing op start(.*) on rsc2" \ - -s "crmd(.*) Resource state: rsc2(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc2" cts_assert Adding rsc2 did not pass #---- do_cmd echo Various sanity checks - stage 2 do_cmd is_dc $test_node_1 $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 DC $test_node_1 cts_assert "S_IDLE not reached on the DC!" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Shutdown $test_node_1 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \ -s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" cts_assert "Transition of the DC away from ${test_node_1} failed." # -s "crmd(.*) State transition (.*) \->" \ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "${test_node_1} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \ -s "${test_node_1} crmd(.*)\[crmd\] stopped" \ -e "${test_node_1} crmd(.*)Timer I_TERMINATE just popped" cts_assert "Shutdown of ${test_node_1} failed" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" cts_assert "No DC Declared" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \ -s "crmd(.*)State transition(.*) \-> \"S_IDLE\"" cts_assert "DC not idle" #---- do_cmd echo Various sanity checks - stage 3 do_cmd is_dc $test_node_2 $test_node_2 cts_assert "$test_node_2 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 $test_node_2 cts_assert "S_IDLE not reached on $test_node_2!" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_2}" do_cmd is_running rsc1 $test_node_2 $test_node_2 cts_assert "rsc1 NOT running on $test_node_2" do_cmd is_running rsc2 $test_node_2 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Shutdown $test_node_2 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \ -s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" cts_assert "Transition of the DC away from ${test_node_2} failed." # -s "crmd(.*) State transition(.*) \->" \ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "${test_node_2} crmd(.*)State transition" \ -s "${test_node_2} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \ -s "${test_node_2} crmd(.*)\[crmd\] stopped" \ -e "${test_node_2} crmd(.*)Timer I_TERMINATE just popped" cts_assert "Shutdown of ${test_node_2} failed" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 11000 \ -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \ -s "crmd(.*)State transition(.*) \-> \"S_IDLE\"" cts_assert "No DC Declared" #---- do_cmd echo Various sanity checks - stage 4 do_cmd is_dc $test_node_3 $test_node_3 cts_assert "$test_node_3 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 $test_node_3 cts_assert "S_IDLE not reached on $test_node_3!" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_3}" do_cmd is_running rsc1 $test_node_3 $test_node_3 cts_assert "rsc1 NOT running on $test_node_3" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_3}" do_cmd is_running rsc2 $test_node_3 $test_node_3 cts_assert "rsc2 NOT running on $test_node_3" #---- do_cmd echo Shutdown $test_node_3 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_3} crmd(.*)State transition (.*) \-> \"S_PENDING\"" \ -s "${test_node_3} crmd(.*)State transition \"S_PENDING\" \-> \"S_STOPPING\"" \ -s "${test_node_3} crmd(.*)\[crmd\] stopped" cts_assert "Shutdown of ${test_node_3} failed." #---- do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd echo "test: PASSED" ) done diff --git a/crm/test/3node.sh.in b/crm/test/3node.sh.in index bb2e16d66a..6f94238c62 100644 --- a/crm/test/3node.sh.in +++ b/crm/test/3node.sh.in @@ -1,283 +1,275 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # required_nodes=3 testdir=@libdir@/heartbeat/crmtest . ${testdir}/helper.sh || exit 1 CRM_ERR_SHUTDOWN=0 test_nodes=3 while [ $iteration -lt $repeats ]; do iteration=`expr $iteration + 1` echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" echo "########### Begining iteration $iteration of $repeats ###########" ( # make *sure* theres nothing left over from last time crm-cleanup #---- do_cmd echo "wait for HA to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." #---- do_cmd echo "wait for HA to start on ${test_node_2}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_2} ccm(.*) info: Hostname: ${test_node_2}" \ -s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." #---- do_cmd echo "wait for HA to start on ${test_node_3}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_3} ccm(.*) info: Hostname: ${test_node_3}" \ -s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_3} failed." #---- do_cmd echo "wait for CRMd to start on ${test_node_1}, ${test_node_2} and ${test_node_3}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_2} crmd(.*) info:(.*)FSA Hostname: ${test_node_2}" cts_assert "CRMd startup on ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_3} crmd(.*) info:(.*)FSA Hostname: ${test_node_3}" cts_assert "CRMd startup on ${test_node_3} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 25000 \ -s "${test_node_1} crmd(.*) State transition (.*) \-> S_IDLE" \ -s "${test_node_2} crmd(.*) State transition (.*) \-> S_NOT_DC" \ -s "${test_node_3} crmd(.*) State transition (.*) \-> S_NOT_DC" cts_assert "CRMd did not start up into correct modes." #---- do_cmd echo Various sanity checks - stage 1 do_cmd is_dc $test_node_1 $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 DC $test_node_1 cts_assert "S_IDLE not reached on the DC!" #---- do_cmd echo Create the first constraint and wait for S_IDLE rsc=rsc1 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` node_xml=" " crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert Adding constraint1 did not pass #---- do_cmd echo Create the second constraint and wait for S_IDLE rsc=rsc2 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` node_xml="' " crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" cts_assert Adding constraint2 did not pass #---- do_cmd echo Create the first resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" \ - -s "crmd(.*) Resource state: rsc1(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc1" cts_assert Adding rsc1 did not pass #---- do_cmd echo Create the second resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" \ - -s "crmd(.*) Resource state: rsc2(.*) after start" + -s "crmd(.*) Performing op start(.*) on rsc2" cts_assert Adding rsc2 did not pass #---- do_cmd echo Various sanity checks - stage 2 do_cmd is_dc $test_node_1 $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 DC $test_node_1 cts_assert "S_IDLE not reached on the DC!" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Shutdown $test_node_1 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*) State transition (.*) \-> S_ELECTION" \ -s "${test_node_1} crmd(.*) State transition (.*) \-> S_NOT_DC" cts_assert "Transition of the DC away from ${test_node_1} failed." # -s "crmd(.*) State transition (.*) \->" \ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "${test_node_1} crmd(.*)State transition S_NOT_DC \-> S_STOPPING" \ -s "${test_node_1} crmd(.*)\[crmd\] stopped" \ -e "${test_node_1} crmd(.*)Timer I_TERMINATE just popped" cts_assert "Shutdown of ${test_node_1} failed" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" cts_assert "No DC Declared" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" \ -s "crmd(.*)State transition(.*) \-> S_IDLE" cts_assert "DC not idle" #---- do_cmd echo Various sanity checks - stage 3 do_cmd is_dc $test_node_2 $test_node_2 cts_assert "$test_node_2 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 $test_node_2 cts_assert "S_IDLE not reached on $test_node_2!" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_2}" do_cmd is_running rsc1 $test_node_2 $test_node_2 cts_assert "rsc1 NOT running on $test_node_2" do_cmd is_running rsc2 $test_node_2 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Shutdown $test_node_2 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*) State transition (.*) \-> S_ELECTION" \ -s "${test_node_2} crmd(.*) State transition (.*) \-> S_NOT_DC" cts_assert "Transition of the DC away from ${test_node_2} failed." # -s "crmd(.*) State transition(.*) \->" \ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "${test_node_2} crmd(.*)State transition" \ -s "${test_node_2} crmd(.*)State transition S_NOT_DC \-> S_STOPPING" \ -s "${test_node_2} crmd(.*)\[crmd\] stopped" \ -e "${test_node_2} crmd(.*)Timer I_TERMINATE just popped" cts_assert "Shutdown of ${test_node_2} failed" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 11000 \ -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" \ -s "crmd(.*)State transition(.*) \-> S_IDLE" cts_assert "No DC Declared" #---- do_cmd echo Various sanity checks - stage 4 do_cmd is_dc $test_node_3 $test_node_3 cts_assert "$test_node_3 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 $test_node_3 cts_assert "S_IDLE not reached on $test_node_3!" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_3}" do_cmd is_running rsc1 $test_node_3 $test_node_3 cts_assert "rsc1 NOT running on $test_node_3" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_3}" do_cmd is_running rsc2 $test_node_3 $test_node_3 cts_assert "rsc2 NOT running on $test_node_3" #---- stop_node $test_node_3 crmd #---- do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd echo "test: PASSED" ) done