diff --git a/crm/test/1node.sh b/crm/test/1node.sh index dffbc78abc..c88254f478 100644 --- a/crm/test/1node.sh +++ b/crm/test/1node.sh @@ -1,97 +1,87 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # . helper.sh #. @libdir@/heartbeat/crmtest/helper.sh CRM_ERR_SHUTDOWN=1 # make *sure* theres nothing left over from last time -# stop all running HAs -do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" -do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" - -# be *very* sure everything has stopped -do_cmd remote_cmd $INIT_USER $test_node_1 "killall -q9 heartbeat ccm lrmd crmd" -do_cmd remote_cmd $INIT_USER $test_node_2 "killall -q9 heartbeat ccm lrmd crmd" - -# make *sure* theres nothing left over from last time -do_cmd remote_cmd $INIT_USER $test_node_1 "rm -f $HAVAR_DIR/crm/cib*.xml" -do_cmd remote_cmd $INIT_USER $test_node_2 "rm -f $HAVAR_DIR/crm/cib*.xml" +crm-cleanup do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & # start HA anew do_cmd echo "wait for HA to start" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd echo "wait for CRMd to start" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup)!" # Create the CIB for this test and wait for all transitions to complete #do_cmd make_node $test_node_1 $test_node_1 args="" do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - $args args="" do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - $args do_cmd make_constraint $test_node_1 rsc1 can do_cmd make_constraint $test_node_1 rsc2 can do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 after CIB create" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not maintained on $test_node_1!" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_dc $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd is_running rsc1 $test_node_1 x$test_node_1 cts_assert_false "rsc1 IS running on x$test_node_1" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_1 cts_assert "rsc2 NOT running on $test_node_1" # shutdown do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 do_cmd wait_for_state S_PENDING 30 $test_node_1 cts_assert "S_PENDING not reached on $test_node_1!" # escalate the shutdown do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 # just in case do_cmd remote_cmd $CRMD_USER $test_node_1 killall -9q crmd echo "test: PASSED" diff --git a/crm/test/2node.sh b/crm/test/2node.sh index 89d88da51c..03161fbc81 100644 --- a/crm/test/2node.sh +++ b/crm/test/2node.sh @@ -1,185 +1,176 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # . helper.sh CRM_ERR_SHUTDOWN=0 - -# stop all running HAs -do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" -do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" - -# be *very* sure everything has stopped -do_cmd remote_cmd $INIT_USER $test_node_1 "killall -q9 heartbeat ccm lrmd crmd" -do_cmd remote_cmd $INIT_USER $test_node_2 "killall -q9 heartbeat ccm lrmd crmd" - # make *sure* theres nothing left over from last time -do_cmd remote_cmd $INIT_USER $test_node_1 "rm -f $HAVAR_DIR/crm/cib*.xml" -do_cmd remote_cmd $INIT_USER $test_node_2 "rm -f $HAVAR_DIR/crm/cib*.xml" +crm-cleanup + do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd echo "wait for HA to start" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd echo "wait for CRMd to start" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup)!" # Erase the contents of the CIB and wait for things to settle down #do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/cibadmin -E #do_cmd wait_for_state S_IDLE 10 $test_node_1 #cts_assert "S_IDLE not reached on $test_node_1 after CIB erase" # Create the CIB for this test and wait for all transitions to complete #do_cmd make_node $test_node_1 $test_node_1 #do_cmd make_node $test_node_1 $test_node_2 args="" do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - $args args="" do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - $args #do_cmd make_constraint $test_node_1 rsc1 can uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` rsc=rsc1 node_xml="' '" do_cmd make_constraint_adv $test_node_1 $node_xml #do_cmd make_constraint $test_node_1 rsc2 can uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` rsc=rsc2 node_xml="' '" do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_dc $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd is_running rsc1 $test_node_1 x$test_node_1 cts_assert_false "rsc1 IS running on x$test_node_1" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_1 cts_assert "rsc2 NOT running on $test_node_1" do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd echo "wait for HA to start on $test_node_2" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" -s "${test_node_2} heartbeat(.*): info: Starting (.*)lrmd" -e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "-VVVV" "2>&1 >/dev/null" & #do_cmd wait_for_state S_INTEGRATION 30 $test_node_1 #cts_assert "S_INTEGRATION not reached on $test_node_1 (new node)!" do_cmd echo "wait for CRMd to start on $test_node_2" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" cts_assert "CRMd startup on ${test_node_2} failed." #do_cmd echo "wait for transition to complete" #do_cmd ./testutils.pl -l ${logfile} --search $logfile '"${test_node_2} crmd(.*)erforming op start(.*) on rsc2"' 1 1500 #cts_assert "Transition of rsc2 to ${test_node_2} failed." do_cmd wait_for_state S_NOT_DC 30 $test_node_2 cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 & do_cmd echo "Looking for transition messages" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_2} crmd(.*) State transition (.*) -> \"S_ELECTION\"" -s "${test_node_1} crmd(.*) State transition (.*) -> \"S_NOT_DC\"" -s "${test_node_1} crmd(.*)State transition (.*) -> \"S_STOPPING\"" cts_assert "Shutdown of ${test_node_1} followed by stability on ${test_node_2} failed." do_cmd wait_for_state S_IDLE 60 $test_node_2 cts_assert "S_IDLE not reached on $test_node_2!" do_cmd is_running rsc1 $test_node_2 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_2 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_2 $test_node_1 cts_assert_false "rsc1 IS running on $test_node_1" do_cmd is_running rsc2 $test_node_2 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 do_cmd wait_for_state S_PENDING 30 $test_node_2 cts_assert "S_PENDING not reached on $test_node_2!" # escalate the shutdown do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 & do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_2} crmd(.*)State transition \"S_PENDING\" -> \"S_STOPPING\"" cts_assert "Shutdown of ${test_node_2} failed." # just in case do_cmd remote_cmd $CRMD_USER $test_node_2 killall -9 crmd echo "test: PASSED" diff --git a/crm/test/3node-rapid.sh b/crm/test/3node-rapid.sh index 2fe868a8a4..079a1ae13e 100644 --- a/crm/test/3node-rapid.sh +++ b/crm/test/3node-rapid.sh @@ -1,173 +1,171 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # . helper.sh CRM_ERR_SHUTDOWN=0 -# stop all running HAs -do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" -do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" - -# be *very* sure everything has stopped -do_cmd remote_cmd $INIT_USER $test_node_1 "killall -q9 heartbeat ccm lrmd crmd" -do_cmd remote_cmd $INIT_USER $test_node_2 "killall -q9 heartbeat ccm lrmd crmd" - # make *sure* theres nothing left over from last time -do_cmd remote_cmd $INIT_USER $test_node_1 "rm -f $HAVAR_DIR/crm/cib*.xml" -do_cmd remote_cmd $INIT_USER $test_node_2 "rm -f $HAVAR_DIR/crm/cib*.xml" - -do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & +crm-cleanup do_cmd echo "wait for HA to start" +do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." do_cmd echo "wait for HA to start" +do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_2} ccm(.*): info: Hostname: ${test_node_2}" -s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." do_cmd echo "wait for HA to start" +do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_3} ccm(.*): info: Hostname: ${test_node_3}" -s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_3} failed." do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd echo "wait for CRMd to start on each node" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}" -s "${test_node_2} crmd(.*): info:(.*)FSA Hostname: ${test_node_2}" -s "${test_node_3} crmd(.*): info:(.*)FSA Hostname: ${test_node_3}" cts_assert "CRMd startup on failed." do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup)!" +do_cmd wait_for_state S_NOT_DC 30 $test_node_2 +cts_assert "S_NOT_DC not reached on $test_node_2 (startup)!" + +do_cmd wait_for_state S_NOT_DC 30 $test_node_3 +cts_assert "S_NOT_DC not reached on $test_node_3 (startup)!" + # Erase the contents of the CIB and wait for things to settle down #do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/cibadmin -E #do_cmd wait_for_state S_IDLE 10 $test_node_1 #cts_assert "S_IDLE not reached on $test_node_1 after CIB erase" # Create the CIB for this test and wait for all transitions to complete #do_cmd make_node $test_node_1 $test_node_1 #do_cmd make_node $test_node_1 $test_node_2 args="" do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - $args args="" do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - $args #do_cmd make_constraint $test_node_1 rsc1 can uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` rsc=rsc1 node_xml="' '" do_cmd make_constraint_adv $test_node_1 $node_xml #do_cmd make_constraint $test_node_1 rsc2 can uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` rsc=rsc2 node_xml="' '" do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd is_dc $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 & do_cmd echo "Looking for transition messages" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_2} crmd(.*) State transition (.*) -> \"S_ELECTION\"" -s "${test_node_1} crmd(.*) State transition (.*) -> \"S_NOT_DC\"" -s "${test_node_1} crmd(.*)State transition (.*) -> \"S_STOPPING\"" cts_assert "Shutdown of ${test_node_1} followed by stability on ${test_node_2} failed." do_cmd is_dc $test_node_2 $test_node_2 cts_assert "$test_node_2 is supposed to be the DC" do_cmd wait_for_state S_IDLE 60 $test_node_2 cts_assert "S_IDLE not reached on $test_node_2!" do_cmd is_running rsc1 $test_node_2 cts_assert "rsc1 NOT running on $test_node_2" do_cmd is_running rsc2 $test_node_2 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 & do_cmd echo "Looking for transition messages" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_3} crmd(.*) State transition (.*) -> \"S_ELECTION\"" -s "${test_node_2} crmd(.*) State transition (.*) -> \"S_NOT_DC\"" -s "${test_node_2} crmd(.*)State transition (.*) -> \"S_STOPPING\"" cts_assert "Shutdown of ${test_node_2} followed by stability on ${test_node_3} failed." do_cmd is_dc $test_node_3 cts_assert "$test_node_3 is supposed to be the DC" do_cmd wait_for_state S_IDLE 60 $test_node_3 cts_assert "S_IDLE not reached on $test_node_3!" do_cmd is_running rsc1 $test_node_3 $test_node_3 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_3 $test_node_3 cts_assert "rsc2 NOT running" do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 do_cmd wait_for_state S_PENDING 30 $test_node_3 cts_assert "S_PENDING not reached on $test_node_3!" # escalate the shutdown do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 & do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_3} crmd(.*)State transition \"S_PENDING\" -> \"S_STOPPING\"" cts_assert "Shutdown of ${test_node_3} failed." # just in case do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd echo "test: PASSED" diff --git a/crm/test/3node.sh b/crm/test/3node.sh index 6697f2b6d3..47bd986b39 100644 --- a/crm/test/3node.sh +++ b/crm/test/3node.sh @@ -1,223 +1,213 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # . helper.sh CRM_ERR_SHUTDOWN=0 - - -# stop all running HAs -do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" -do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" - -# be *very* sure everything has stopped -do_cmd remote_cmd $INIT_USER $test_node_1 "killall -q9 heartbeat ccm lrmd crmd" -do_cmd remote_cmd $INIT_USER $test_node_2 "killall -q9 heartbeat ccm lrmd crmd" - # make *sure* theres nothing left over from last time -do_cmd remote_cmd $INIT_USER $test_node_1 "rm -f $HAVAR_DIR/crm/cib*.xml" -do_cmd remote_cmd $INIT_USER $test_node_2 "rm -f $HAVAR_DIR/crm/cib*.xml" +crm-cleanup + do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd echo "wait for HA to start" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd echo "wait for CRMd to start" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup)!" # Erase the contents of the CIB and wait for things to settle down #do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/cibadmin -E #do_cmd wait_for_state S_IDLE 10 $test_node_1 #cts_assert "S_IDLE not reached on $test_node_1 after CIB erase" # Create the CIB for this test and wait for all transitions to complete #do_cmd make_node $test_node_1 $test_node_1 #do_cmd make_node $test_node_1 $test_node_2 args="" do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - $args args="" do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - $args #do_cmd make_constraint $test_node_1 rsc1 can uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` rsc=rsc1 node_xml="' '" do_cmd make_constraint_adv $test_node_1 $node_xml #do_cmd make_constraint $test_node_1 rsc2 can uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` rsc=rsc2 node_xml="' '" do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_dc $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd is_running rsc1 $test_node_1 x$test_node_1 cts_assert_false "rsc1 IS running on x$test_node_1" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_1 cts_assert "rsc2 NOT running on $test_node_1" do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd echo "wait for HA to start on $test_node_2" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" -s "${test_node_2} heartbeat(.*): info: Starting (.*)lrmd" -e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "-VVVV" "2>&1 >/dev/null" & #do_cmd wait_for_state S_INTEGRATION 30 $test_node_1 #cts_assert "S_INTEGRATION not reached on $test_node_1 (new node)!" do_cmd echo "wait for CRMd to start on $test_node_2" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" cts_assert "CRMd startup on ${test_node_2} failed." #do_cmd echo "wait for transition to complete" #do_cmd ./testutils.pl -l ${logfile} --search $logfile '"${test_node_2} crmd(.*)erforming op start(.*) on rsc2"' 1 1500 #cts_assert "Transition of rsc2 to ${test_node_2} failed." do_cmd wait_for_state S_NOT_DC 30 $test_node_2 cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd echo "wait for HA to start on $test_node_3" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_3} ccm(.*) Hostname: ${test_node_3}" -s "${test_node_3} heartbeat(.*): info: Starting (.*)lrmd" -e "${test_node_3} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_3} failed." do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd "-VVVV" "2>&1 >/dev/null" & #do_cmd wait_for_state S_INTEGRATION 30 $test_node_1 #cts_assert "S_INTEGRATION not reached on $test_node_1 (new node)!" do_cmd echo "wait for CRMd to start on $test_node_3" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_3} crmd(.*)FSA Hostname: ${test_node_3}" cts_assert "CRMd startup on ${test_node_3} failed." do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 & do_cmd wait_for_state S_IDLE 60 $test_node_2 do_cmd echo "Looking for transition messages" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_2} crmd(.*) State transition (.*) -> \"S_ELECTION\"" -s "${test_node_1} crmd(.*) State transition (.*) -> \"S_NOT_DC\"" -s "${test_node_1} crmd(.*)State transition (.*) -> \"S_STOPPING\"" cts_assert "Shutdown of ${test_node_1} followed by stability on ${test_node_2} failed." do_cmd wait_for_state S_IDLE 60 $test_node_2 cts_assert "S_IDLE not reached on $test_node_2!" do_cmd is_running rsc1 $test_node_2 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_2 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_2 $test_node_1 cts_assert_false "rsc1 IS running on $test_node_1" do_cmd is_running rsc1 $test_node_2 $test_node_2 cts_assert "rsc1 NOT running on $test_node_2" do_cmd is_running rsc2 $test_node_2 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 do_cmd echo "Looking for transition messages" do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_3} crmd(.*) State transition (.*) -> \"S_ELECTION\"" -s "${test_node_2} crmd(.*) State transition (.*) -> \"S_NOT_DC\"" -s "${test_node_2} crmd(.*)State transition (.*) -> \"S_STOPPING\"" cts_assert "Shutdown of ${test_node_2} followed by stability on ${test_node_3} failed." do_cmd is_running rsc1 $test_node_3 $test_node_3 cts_assert "rsc1 NOT running on $test_node_3" do_cmd is_running rsc2 $test_node_3 $test_node_3 cts_assert "rsc2 NOT running on $test_node_3" do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 do_cmd wait_for_state S_PENDING 30 $test_node_3 cts_assert "S_PENDING not reached on $test_node_3!" # escalate the shutdown do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 & do_cmd ./testutils.pl -l ${logfile} --search -a -m 1500 -s "${test_node_3} crmd(.*)State transition \"S_PENDING\" -> \"S_STOPPING\"" cts_assert "Shutdown of ${test_node_3} failed." # just in case do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd echo "test: PASSED" diff --git a/crm/test/Makefile.am b/crm/test/Makefile.am index c5851db9b3..582c8c3779 100644 --- a/crm/test/Makefile.am +++ b/crm/test/Makefile.am @@ -1,30 +1,32 @@ # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # MAINTAINERCLEANFILES = Makefile.in testdir = $(libdir)/heartbeat/crmtest -test_SCRIPTS = helper.sh 1node.sh 2node.sh testutils.pl +test_SCRIPTS = helper.sh testutils.pl \ + 1node.sh 2node.sh 3node.sh 3node-rapid.sh stop.sh -EXTRA_DIST = 1node.sh 2node.sh testutils.pl +EXTRA_DIST = helper.sh testutils.pl \ + 1node.sh 2node.sh 3node.sh 3node-rapid.sh stop.sh install-exec-local: $(mkinstalldirs) $(DESTDIR)/$(testdir) -chown $(HA_CCMUSER) $(DESTDIR)/$(testdir) -chgrp $(HA_APIGROUP) $(DESTDIR)/$(testdir) -chmod g+w $(DESTDIR)/$(testdir) diff --git a/crm/test/helper.sh.in b/crm/test/helper.sh.in index 44a219340d..086c30908b 100644 --- a/crm/test/helper.sh.in +++ b/crm/test/helper.sh.in @@ -1,325 +1,343 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # HALIB_DIR=@libdir@/heartbeat HAVAR_DIR=@HA_VARLIBDIR@/heartbeat HAINIT_DIR=@INITDIR@ INIT_USER=root CRMD_USER=root ADMIN_USER=root test_node_1=fish test_node_2=chips test_node_3=mayo CRM_ERR_SHUTDOWN=0 OUTPUT_STDERR=0 OUTPUT_STDOUT=0 OUTPUT_CMDS=0 done=0 ip_rsc_1=192.168.9.161 ip_rsc_2=192.168.9.162 logfile=/var/log/messages while test "$done" = "0"; do case "$1" in --debug | -x) set -x; shift;; --stdout | -o) OUTPUT_STDOUT=1; shift;; --stderr | -e) OUTPUT_STDERR=1; shift;; --verbose | -v) OUTPUT_CMDS=1; shift;; --logfile | -l) logfile=$2; shift 2;; --init | -i) INIT_USER=$2; shift 2;; --admin | -a) ADMIN_USER=$2; shift 2;; --crmd | -c) CRMD_USER=$2; shift 2;; --node1 | -1) test_node_1=$2; shift 2;; --node2 | -2) test_node_2=$2; shift 2;; --ip1) ip_rsc_1=$2; shift 2;; --ip2) ip_rsc_2=$2; shift 2;; "") done=1;; *) echo "unknown option: $1"; exit 1;; esac done function remote_cmd() { user=$1 host=$2 shift shift `which ssh` ${user}@${host} "$*" return $?; } function is_state() { ret=1 host=$1 state=$2 output=`remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host | grep $state` if [ ! -z "$output" ]; then echo "$host is in $state" return 0 fi echo "$host is NOT in $state" return $ret } function is_running() { rsc=$1 askhost=$2 host=$3 if [ -z $host ]; then output=`remote_cmd $ADMIN_USER $askhost $HALIB_DIR/crmadmin -W $rsc | grep -v $HALIB_DIR/crmadmin | grep -v NOT` else output=`remote_cmd $ADMIN_USER $askhost $HALIB_DIR/crmadmin -W $rsc | grep -v $HALIB_DIR/crmadmin | grep $host` fi if [ -z "$output" ]; then if [ -z $host ]; then echo "$rsc is NOT running" else echo "$rsc is NOT running on host $host" fi return 1 fi if [ -z $host ]; then echo "$rsc is running" else echo "$rsc is running on host $host" fi return 0 } function is_dc() { host=$1 output=`remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host | grep -v S_PENDING | grep -v S_NOT_DC | grep -v S_ELECTION | grep -v S_RECOVERY` if [ ! -z "$output" ]; then echo "$host is DC" return 0 fi echo "$host is NOT DC" return 1 } function wait_for_state() { state=$1 max=$2 host=$3 if [ -z "$host" ]; then host=`remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -D > /dev/null` fi count=1 until is_state $host $state; do if [ $count -gt $max ]; then echo "Attempt to reach $state on $host failed" return 1 fi count=`expr $count + 1` sleep 1 done echo "$state reached on $host" return 0 } function make_node() { host=$1 uname=$2 type=$3 uuid=`uuidgen` if [ -z $type ]; then type="member" fi node_xml="''"; remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -C -o node -X $node_xml" } function make_resource() { host=$1 id=$2 class=$3 type=$4 priority=$5 version=$6 shift 6 args=$* uuid=`uuidgen` node_xml="''" make_constraint_adv $host $node_xml } function cts_assert() { do_cts_assert $? 0 "Assert failed - " $* } function cts_assert_false() { do_cts_assert $? 1 "AssertFalse failed - " $* } function do_cts_assert() { result=$1 expected=$2 shift shift if [ $result -ne $expected ]; then err $* fi } function err() { echo "ERROR: $*" if [ ! -z $HOST ]; then if [ $CRM_ERR_SHUTDOWN -eq 1 ]; then remote_cmd hacluster $test_node_1 killall -9 crmd remote_cmd hacluster $test_node_2 killall -9 crmd fi fi echo "test: FAILED" exit 1 } function do_cmd() { cmdline=$* old_OUTPUT_STDOUT=$OUTPUT_STDOUT if [ $OUTPUT_STDERR -eq 1 -o $OUTPUT_STDOUT -eq 1 -o $OUTPUT_CMDS -eq 1 ]; then cmd=$1 if [ $cmd = "remote_cmd" ]; then user=$2 host=$3 shift 3 echo `date` ": Running '$*' as ${user}@${host}"; elif [ $cmd = "wait_for_state" ]; then host=$4 if [ -z $host ]; then host="the DC" fi echo `date` ": Waiting for state $2 on $host (max $3 attempts)"; elif [ $cmd = "./testutils.pl" ]; then OUTPUT_STDOUT=1 echo `date` ": Running '$cmdline'"; elif [ $cmd = "echo" ]; then if [ $OUTPUT_CMDS -eq 1 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then echo -n `date` ": " $cmdline fi else echo `date` ": Running '$cmdline'"; fi fi if [ $OUTPUT_STDERR -eq 1 -a $OUTPUT_STDOUT -eq 1 ]; then $cmdline rc=$? elif [ $OUTPUT_STDERR -eq 0 -a $OUTPUT_STDOUT -eq 0 ]; then $cmdline 2>&1 >/dev/null rc=$? elif [ $OUTPUT_STDOUT -eq 0 ]; then $cmdline >/dev/null rc=$? elif [ $OUTPUT_STDERR -eq 0 ]; then $cmdline 2>/dev/null rc=$? else $cmdline rc=$? fi OUTPUT_STDOUT=$old_OUTPUT_STDOUT # echo Result: $rc return $rc } +function crm-cleanup() { +# stop all running HAs + do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" + do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" + do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" + +# be *very* sure everything has stopped + do_cmd remote_cmd $INIT_USER $test_node_1 "killall -q9 heartbeat ccm lrmd crmd" + do_cmd remote_cmd $INIT_USER $test_node_2 "killall -q9 heartbeat ccm lrmd crmd" + do_cmd remote_cmd $INIT_USER $test_node_3 "killall -q9 heartbeat ccm lrmd crmd" + + if [ -z $1 ]; then +# make *sure* theres nothing left over from last time + do_cmd remote_cmd $INIT_USER $test_node_1 "rm -f $HAVAR_DIR/crm/cib*.xml /tmp/crm/*" + do_cmd remote_cmd $INIT_USER $test_node_2 "rm -f $HAVAR_DIR/crm/cib*.xml /tmp/crm/*" + do_cmd remote_cmd $INIT_USER $test_node_3 "rm -f $HAVAR_DIR/crm/cib*.xml /tmp/crm/*" + fi +} diff --git a/crm/test/Makefile.am b/crm/test/stop.sh similarity index 67% copy from crm/test/Makefile.am copy to crm/test/stop.sh index c5851db9b3..91ec2d33ee 100644 --- a/crm/test/Makefile.am +++ b/crm/test/stop.sh @@ -1,30 +1,23 @@ +#!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -MAINTAINERCLEANFILES = Makefile.in -testdir = $(libdir)/heartbeat/crmtest +. helper.sh -test_SCRIPTS = helper.sh 1node.sh 2node.sh testutils.pl - -EXTRA_DIST = 1node.sh 2node.sh testutils.pl - -install-exec-local: - $(mkinstalldirs) $(DESTDIR)/$(testdir) - -chown $(HA_CCMUSER) $(DESTDIR)/$(testdir) - -chgrp $(HA_APIGROUP) $(DESTDIR)/$(testdir) - -chmod g+w $(DESTDIR)/$(testdir) +# make *sure* theres nothing left over from last time +crm-cleanup 0