diff --git a/crm/test/2node-fail.sh.in b/crm/test/2node-fail.sh.in index 263f0ed7ed..6253cd869e 100644 --- a/crm/test/2node-fail.sh.in +++ b/crm/test/2node-fail.sh.in @@ -1,339 +1,339 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # testdir=@libdir@/heartbeat/crmtest . ${testdir}/helper.sh || exit 1 CRM_ERR_SHUTDOWN=0 test_nodes=2 function 2node_fail_test() { test_type=$1 fail_pieces=$2 fail_node=$3 good_node=$4 do_failback=$5 accelerated=$6 if [ $fail_node = $test_node_1 ]; then moved_rsc=rsc1 else moved_rsc=rsc2 fi #---- echo -ne "\033]0;$test_type: Iteration $iteration of $repeats\007" crm-cleanup do_cmd echo "#############################" do_cmd echo "$test_type: Iteration $iteration of $repeats" # make *sure* theres nothing left over from last time #---- do_cmd echo "wait for HA to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" \ -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." #---- do_cmd echo "wait for CRMd to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}" \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd wait_for_state S_IDLE 3 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup)!" #---- do_cmd echo Create the first constraint and wait for S_IDLE rsc=rsc1 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert Adding constraint1 did not pass #---- do_cmd echo Create the second constraint and wait for S_IDLE rsc=rsc2 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert Adding constraint2 did not pass #---- do_cmd echo Create the first resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - ignore $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ -s "crmd(.*) Performing op start(.*) on rsc1" \ -s "crmd(.*) Resource state: rsc1(.*) after start" cts_assert Adding rsc1 did not pass #---- do_cmd echo Create the second resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - ignore $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ -s "crmd(.*) Performing op start(.*) on rsc2" \ -s "crmd(.*) Resource state: rsc2(.*) after start" cts_assert Adding rsc2 did not pass #---- do_cmd echo Various sanity checks - stage 1 do_cmd wait_for_state S_IDLE 3 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_dc $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd is_running rsc1 $test_node_1 x$test_node_1 cts_assert_false "rsc1 IS running on x$test_node_1" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_1 cts_assert "rsc2 NOT running on $test_node_1" #---- do_cmd echo "wait for HA to start on $test_node_2" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" \ -s "${test_node_2} heartbeat(.*): info: Starting (.*)lrmd" \ -e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." #---- do_cmd echo "wait for CRMd to start on $test_node_2" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" \ -s "${test_node_2} crmd(.*) State transition \"S_PENDING\" \-> \"S_NOT_DC\"" cts_assert "CRMd startup on ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3500 \ -s "${test_node_1} crmd(.*) State transition(.*) \-> \"S_IDLE\"" \ -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" \ -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}" cts_assert "rsc2 was not transferred to ${test_node_2} on startup." #---- do_cmd echo Various sanity checks - stage 2 do_cmd wait_for_state S_NOT_DC 30 $test_node_2 cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- is_dc $fail_node 2>&1 > /dev/null test_for_election=$? do_cmd echo Killing $fail_pieces on $fail_node crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $ADMIN_USER $fail_node "killall -9 $fail_pieces" & if [ $test_for_election = 0 ]; then do_cmd echo Killed the DC... checking for DC Failover - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 50000 \ + do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 60000 \ -s "${good_node} crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \ -s "${good_node} crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert "Transition of the DC from ${fail_node} to ${good_node} failed." else do_cmd echo Killed slave node... checking the DC noticed - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 50000 \ + do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 60000 \ -s "${good_node} crmd(.*) State transition (.*) \-> \"S_POLICY_ENGINE\"" \ -s "${good_node} crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert "Failure of slave node ${fail_node} was not noticed on the DC (${good_node})." fi - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 50000 \ + do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 60000 \ -s "${god_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" \ -s "crmd(.*) Resource state: ${moved_rsc}(.*) after start] on ${good_node}" cts_assert "Move of ${moved_rsc} to ${good_node} failed." #---- do_cmd echo Various sanity checks - stage 3 do_cmd wait_for_state S_IDLE 60 $good_node cts_assert "S_IDLE not reached on $good_node after kill!" do_cmd is_running rsc1 $good_node cts_assert "rsc1 NOT running on $good_node" do_cmd is_running rsc2 $good_node cts_assert "rsc2 NOT running on $good_node" do_cmd is_running rsc1 $good_node $fail_node cts_assert_false "rsc1 IS running on $fail_node" do_cmd is_running rsc2 $good_node $good_node cts_assert "rsc2 NOT running on $good_node" #---- if [ $do_failback = 1 ]; then do_cmd echo "Re-Starting on failed node $fail_node" if [ "$fail_pieces" = "crmd" ]; then do_cmd echo "HA still running, skipping restart" else do_cmd echo "wait for HA to start on $fail_node" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $fail_node $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ + do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2000 \ -s "${fail_node} ccm(.*) Hostname: ${fail_node}" \ -s "${fail_node} heartbeat(.*): info: Starting (.*)lrmd" \ -e "${fail_node} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${fail_node} failed." fi #---- do_cmd echo "wait for CRMd to start on $fail_node" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $fail_node $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "${fail_node} crmd(.*)FSA Hostname: ${fail_node}" \ -s "${fail_node} crmd(.*) State transition \"S_PENDING\" \-> \"S_NOT_DC\"" cts_assert "CRMd startup on ${fail_node} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "${good_node} crmd(.*) State transition(.*) \-> \"S_IDLE\"" \ -s "${fail_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" \ -s "crmd(.*) Resource state: ${moved_rsc}(.*) after start] on ${fail_node}" cts_assert "$moved_rsc was not transferred to ${fail_node} on startup." do_cmd echo Various sanity checks - stage 4 do_cmd wait_for_state S_NOT_DC 30 $fail_node cts_assert "S_NOT_DC not reached on $fail_node (restart)!" do_cmd wait_for_state S_IDLE 30 $good_node cts_assert "S_IDLE not reached on $good_node (restart)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" fi #---- do_cmd echo "test ${test_type}: PASSED" } while [ $iteration -lt $repeats ]; do iteration=`expr $iteration + 1` echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" echo "########### $test_type : Begining iteration $iteration of $repeats ###########" ( 2node_fail_test 2node__fail_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 0 0 2node_fail_test 2node__fail_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 0 0 2node_fail_test 2node__failback_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 1 0 2node_fail_test 2node__failback_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 1 0 2node_fail_test 2node__fail_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 0 0 2node_fail_test 2node__fail_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 0 0 2node_fail_test 2node__failback_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 1 0 2node_fail_test 2node__failback_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 1 0 #2node_fail_test 2node__fail_slave_LRM "lrmd" ${test_node_2} ${test_node_1} #2node_fail_test 2node__fail_DC_LRM "lrmd" ${test_node_1} ${test_node_2} echo "test suite: PASSED" ) done diff --git a/crm/test/2node.sh.in b/crm/test/2node.sh.in index b068e08e02..495eb3f449 100644 --- a/crm/test/2node.sh.in +++ b/crm/test/2node.sh.in @@ -1,255 +1,259 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # testdir=@libdir@/heartbeat/crmtest . ${testdir}/helper.sh || exit 1 CRM_ERR_SHUTDOWN=0 test_nodes=2 while [ $iteration -lt $repeats ]; do iteration=`expr $iteration + 1` echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" echo "########### Begining iteration $iteration of $repeats ###########" ( # make *sure* theres nothing left over from last time crm-cleanup #---- do_cmd echo "wait for HA to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" \ -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." #---- do_cmd echo "wait for CRMd to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}" \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd wait_for_state S_IDLE 3 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup)!" #---- do_cmd echo Create the first constraint and wait for S_IDLE rsc=rsc1 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert Adding constraint1 did not pass #---- do_cmd echo Create the second constraint and wait for S_IDLE rsc=rsc2 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert Adding constraint2 did not pass #---- do_cmd echo Create the first resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ -s "crmd(.*) Performing op start(.*) on rsc1" \ -s "crmd(.*) Resource state: rsc1(.*) after start" cts_assert Adding rsc1 did not pass #---- do_cmd echo Create the second resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ -s "crmd(.*) Performing op start(.*) on rsc2" \ -s "crmd(.*) Resource state: rsc2(.*) after start" cts_assert Adding rsc2 did not pass #---- do_cmd echo Various sanity checks - stage 1 do_cmd wait_for_state S_IDLE 3 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_dc $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd is_running rsc1 $test_node_1 x$test_node_1 cts_assert_false "rsc1 IS running on x$test_node_1" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_1 cts_assert "rsc2 NOT running on $test_node_1" #---- do_cmd echo "wait for HA to start on $test_node_2" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" \ -s "${test_node_2} heartbeat(.*): info: Starting (.*)lrmd" \ -e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." #---- do_cmd echo "wait for CRMd to start on $test_node_2" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ -s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" \ -s "${test_node_2} crmd(.*) State transition \"S_PENDING\" \-> \"S_NOT_DC\"" cts_assert "CRMd startup on ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3500 \ -s "${test_node_1} crmd(.*) State transition(.*) \-> \"S_IDLE\"" \ -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" \ -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}" cts_assert "rsc2 was not transferred to ${test_node_2} on startup." #---- do_cmd echo Various sanity checks - stage 2 do_cmd wait_for_state S_NOT_DC 30 $test_node_2 cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" do_cmd wait_for_state S_IDLE 30 $test_node_1 cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" do_cmd is_running rsc1 $test_node_1 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_1 cts_assert "rsc2 NOT running" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" +do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ + -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_2}" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Shutdown $test_node_1 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \ -s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" cts_assert "Transition of the DC from ${test_node_1} to ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "${test_node_1} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \ -e "${test_node_1} crmd(.*)Timer I_TERMINATE just popped" \ -s "${test_node_1} crmd(.*)\[crmd\] stopped" cts_assert "Shutdown of ${test_node_1} failed" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*) Performing op start(.*) on rsc1" \ -s "crmd(.*) Resource state: rsc1(.*) after start" cts_assert "rsc1 was not transferred to ${test_node_2} on shutdown." #---- do_cmd echo Various sanity checks - stage 3 do_cmd wait_for_state S_IDLE 60 $test_node_2 cts_assert "S_IDLE not reached on $test_node_2 after shutdown!" do_cmd is_running rsc1 $test_node_2 cts_assert "rsc1 NOT running" do_cmd is_running rsc2 $test_node_2 cts_assert "rsc2 NOT running" +do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ + -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_2}" do_cmd is_running rsc1 $test_node_2 $test_node_1 cts_assert_false "rsc1 IS running on $test_node_1" do_cmd is_running rsc2 $test_node_2 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Shutdown $test_node_2 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 do_cmd wait_for_state S_PENDING 30 $test_node_2 cts_assert "S_PENDING not reached on $test_node_2!" #---- do_cmd echo Escalate the shutdown crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "${test_node_2} crmd(.*)State transition \"S_PENDING\" \-> \"S_STOPPING\""\ -s "${test_node_2} crmd(.*)\[crmd\] stopped" cts_assert "Shutdown of ${test_node_2} failed." #---- do_cmd remote_cmd $CRMD_USER $test_node_2 killall -9 crmd echo "test: PASSED" ) done diff --git a/crm/test/3node-rapid.sh.in b/crm/test/3node-rapid.sh.in index 1485e40c03..f365decee5 100644 --- a/crm/test/3node-rapid.sh.in +++ b/crm/test/3node-rapid.sh.in @@ -1,290 +1,296 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # testdir=@libdir@/heartbeat/crmtest . ${testdir}/helper.sh || exit 1 CRM_ERR_SHUTDOWN=0 test_nodes=3 while [ $iteration -lt $repeats ]; do iteration=`expr $iteration + 1` echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" echo "########### Begining iteration $iteration of $repeats ###########" ( # make *sure* theres nothing left over from last time crm-cleanup #---- do_cmd echo "wait for HA to start on ${test_node_1}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_1} ccm(.*): info: Hostname: ${test_node_1}" \ -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_1} failed." #---- do_cmd echo "wait for HA to start on ${test_node_2}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_2} ccm(.*): info: Hostname: ${test_node_2}" \ -s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_2} failed." #---- do_cmd echo "wait for HA to start on ${test_node_3}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ -s "${test_node_3} ccm(.*): info: Hostname: ${test_node_3}" \ -s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" \ -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast" cts_assert "Startup of Heartbeat on ${test_node_3} failed." #---- do_cmd echo "wait for CRMd to start on ${test_node_1}, ${test_node_2} and ${test_node_3}" crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_1} crmd(.*): info:(.*)FSA Hostname: ${test_node_1}" cts_assert "CRMd startup on ${test_node_1} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_2} crmd(.*): info:(.*)FSA Hostname: ${test_node_2}" cts_assert "CRMd startup on ${test_node_2} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ -s "${test_node_3} crmd(.*): info:(.*)FSA Hostname: ${test_node_3}" cts_assert "CRMd startup on ${test_node_3} failed." do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 25000 \ -s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ -s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" \ -s "${test_node_3} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" cts_assert "CRMd did not start up into correct modes." #---- do_cmd echo Various sanity checks - stage 1 do_cmd is_dc $test_node_1 $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 DC $test_node_1 cts_assert "S_IDLE not reached on the DC!" #---- do_cmd echo Create the first constraint and wait for S_IDLE rsc=rsc1 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert Adding constraint1 did not pass #---- do_cmd echo Create the second constraint and wait for S_IDLE rsc=rsc2 uuid1=`uuidgen` uuid2=`uuidgen` uuid3=`uuidgen` uuid4=`uuidgen` node_xml="' '" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_constraint_adv $test_node_1 $node_xml do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" cts_assert Adding constraint2 did not pass #---- do_cmd echo Create the first resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ -s "crmd(.*) Performing op start(.*) on rsc1" \ -s "crmd(.*) Resource state: rsc1(.*) after start" cts_assert Adding rsc1 did not pass #---- do_cmd echo Create the second resource and wait for S_IDLE after start args="" crm_log_pos=$(stat -L -c %s $logfile) do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ -s "crmd(.*) Performing op start(.*) on rsc2" \ -s "crmd(.*) Resource state: rsc2(.*) after start" cts_assert Adding rsc2 did not pass #---- do_cmd echo Various sanity checks - stage 2 do_cmd is_dc $test_node_1 $test_node_1 cts_assert "$test_node_1 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 DC $test_node_1 cts_assert "S_IDLE not reached on the DC!" do_cmd is_running rsc1 $test_node_1 $test_node_1 cts_assert "rsc1 NOT running on $test_node_1" do_cmd is_running rsc2 $test_node_1 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Shutdown $test_node_1 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \ -s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" cts_assert "Transition of the DC away from ${test_node_1} failed." # -s "crmd(.*) State transition (.*) \->" \ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "${test_node_1} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \ -s "${test_node_1} crmd(.*)\[crmd\] stopped" \ -e "${test_node_1} crmd(.*)Timer I_TERMINATE just popped" cts_assert "Shutdown of ${test_node_1} failed" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" cts_assert "No DC Declared" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \ -s "crmd(.*)State transition(.*) \-> \"S_IDLE\"" cts_assert "DC not idle" #---- do_cmd echo Various sanity checks - stage 3 do_cmd is_dc $test_node_2 $test_node_2 cts_assert "$test_node_2 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 $test_node_2 cts_assert "S_IDLE not reached on $test_node_2!" -do_cmd is_running rsc1 $test_node_2 +do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ + -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_2}" +do_cmd is_running rsc1 $test_node_2 $test_node_2 cts_assert "rsc1 NOT running on $test_node_2" do_cmd is_running rsc2 $test_node_2 $test_node_2 cts_assert "rsc2 NOT running on $test_node_2" #---- do_cmd echo Shutdown $test_node_2 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ -s "crmd(.*) State transition \"S_NOT_DC\" \-> \"S_ELECTION\"" \ -s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" cts_assert "Transition of the DC away from ${test_node_2} failed." # -s "crmd(.*) State transition(.*) \->" \ do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 6000 \ -s "${test_node_2} crmd(.*)State transition" \ -s "${test_node_2} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \ -s "${test_node_2} crmd(.*)\[crmd\] stopped" \ -e "${test_node_2} crmd(.*)Timer I_TERMINATE just popped" cts_assert "Shutdown of ${test_node_2} failed" do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \ -s "crmd(.*)State transition(.*) \-> \"S_IDLE\"" cts_assert "No DC Declared" #---- do_cmd echo Various sanity checks - stage 4 do_cmd is_dc $test_node_3 $test_node_3 cts_assert "$test_node_3 is supposed to be the DC" do_cmd wait_for_state S_IDLE 3 $test_node_3 cts_assert "S_IDLE not reached on $test_node_3!" -do_cmd is_running rsc1 $test_node_3 +do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ + -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_3}" +do_cmd is_running rsc1 $test_node_3 $test_node_3 cts_assert "rsc1 NOT running on $test_node_3" +do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ + -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_3}" do_cmd is_running rsc2 $test_node_3 $test_node_3 cts_assert "rsc2 NOT running on $test_node_3" #---- do_cmd echo Shutdown $test_node_3 crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 & do_cmd wait_for_state S_PENDING 30 $test_node_3 cts_assert "S_PENDING not reached on $test_node_3!" #---- do_cmd echo Escalate the shutdown crm_log_pos=$(stat -L -c %s $logfile) do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 & do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ -s "${test_node_3} crmd(.*)State transition \"S_PENDING\" \-> \"S_STOPPING\"" \ -s "${test_node_3} crmd(.*)\[crmd\] stopped" cts_assert "Shutdown of ${test_node_3} failed." #---- do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd echo "test: PASSED" ) done diff --git a/crm/test/Makefile.am b/crm/test/Makefile.am index 3133e013cd..ed7b331143 100644 --- a/crm/test/Makefile.am +++ b/crm/test/Makefile.am @@ -1,32 +1,34 @@ # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # MAINTAINERCLEANFILES = Makefile.in testdir = $(libdir)/heartbeat/crmtest -test_SCRIPTS = helper.sh testutils.pl stop.sh \ - 1node.sh 2node.sh 2node-fail.sh 3node.sh 3node-rapid.sh 3node-fail.sh +test_SCRIPTS = helper.sh testutils.pl stop.sh cib_suite.sh \ + 1node.sh 2node.sh 2node-fail.sh \ + 3node.sh 3node-rapid.sh 3node-fail.sh -EXTRA_DIST = helper.sh testutils.pl stop.sh \ - 1node.sh 2node.sh 2node-fail.sh 3node.sh 3node-rapid.sh 3node-fail.sh +EXTRA_DIST = helper.sh testutils.pl stop.sh cib_suite.sh \ + 1node.sh 2node.sh 2node-fail.sh \ + 3node.sh 3node-rapid.sh 3node-fail.sh install-exec-local: $(mkinstalldirs) $(DESTDIR)/$(testdir) -chown $(HA_CCMUSER) $(DESTDIR)/$(testdir) -chgrp $(HA_APIGROUP) $(DESTDIR)/$(testdir) -chmod g+w $(DESTDIR)/$(testdir) diff --git a/crm/test/cib_suite.sh.in b/crm/test/cib_suite.sh.in new file mode 100644 index 0000000000..f41afb7827 --- /dev/null +++ b/crm/test/cib_suite.sh.in @@ -0,0 +1,536 @@ +#!/bin/bash +# +# Copyright (C) 2004 Andrew Beekhof +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# + +testdir=@libdir@/heartbeat/crmtest +. ${testdir}/helper.sh || exit 1 + +erase_sync_1=" + + + + + + + + + + + +" + +erase_sync_2=" + + + + + + + + + + + +" + +erase_sync_3=" + + + + + + + + + + + +" + +create_1=" + + + + + + + + + + + + + + +" + +create_2=" + + + + + + + + + + + + + + +" + +create_3=" + + + + + + + + + + + + + + +" + +create_local_1=" + + + + + + + + + + + + + + + +" + +create_local_2=" + + + + + + + + + + + + + + +" + +create_local_3=" + + + + + + + + + + + + + + +" + +bump_1=" + + + + + + + + + + + + + + + +" + +bump_2=" + + + + + + + + + + + + + + +" + +bump_3=" + + + + + + + + + + + + + + +" + +bump_remote_1=" + + + + + + + + + + + + + + + +" + +bump_remote_2=" + + + + + + + + + + + + + + +" + +bump_remote_3=" + + + + + + + + + + + + + + +" + +sync_from_1=" + + + + + + + + + + + + + + +" + +sync_from_2=" + + + + + + + + + + + + + + +" + +sync_from_3=" + + + + + + + + + + + + + + +" + +# make *sure* theres nothing left over from last time +crm-cleanup + +function compare_cibs() +{ + cib_cmp_base=$1; shift + cib_cmp_host=$1; shift + cib_cmp_opts=$* + cib_cmp_output=${cib_cmp_base}.out + cib_cmp_expected=${cib_cmp_base}.exp + cib_cmp_failed=$test_dump_dir/test.txt + cib_cmp_diff_opts="--ignore-all-space -U 1 -u" + + do_cmd echo ssh $CRMD_USER@$cib_cmp_host "$HALIB_DIR/cibadmin -Q ${cib_cmp_opts}" + remote_cmd $CRMD_USER $cib_cmp_host "$HALIB_DIR/cibadmin -Q ${cib_cmp_opts}" > $cib_cmp_output + cts_assert "compare_cibs(): Could not perform query: opts=$cib_cmp_opts" + + gres ' timestamp=\".*\"' "" ${cib_cmp_output} + gres ' id=\"[0123456789abcdef-]*\"' "" ${cib_cmp_output} + + # this is the value stored in a variable named $cib_cmp_base + echo -e "${!cib_cmp_base}" > $cib_cmp_expected + +# if [ "$create_mode" = "true" -a ! -f $cib_cmp_expected ]; then +# cp "$cib_cmp_output" "$cib_cmp_expected" +# fi + + if [ -f $cib_cmp_expected ]; then + diff $cib_cmp_diff_opts -q $cib_cmp_expected $cib_cmp_output >/dev/null + rc=$? + fi + +# if [ "$create_mode" = "true" ]; then +# echo "Test $cib_cmp_base... Created expected output" +# el + if [ ! -f $cib_cmp_expected ]; then + echo "==== Raw results for CIB test ($cib_cmp_base) ====" >> $cib_cmp_failed + cat $cib_cmp_output 2>/dev/null >> $cib_cmp_failed + cat $cib_cmp_output + + elif [ "$rc" = 0 ]; then + do_cmd echo "Test $cib_cmp_base... Passed"; + elif [ "$rc" = 1 ]; then + do_cmd echo "Test $cib_cmp_base... * Failed"; + diff $cib_cmp_diff_opts $cib_cmp_expected $cib_cmp_output 2>/dev/null >> $cib_cmp_failed + diff $cib_cmp_diff_opts $cib_cmp_expected $cib_cmp_output + else + do_cmd echo "Test $cib_cmp_base... Error (diff: $rc)"; + do_cmd echo "==== Raw results for test ($cib_cmp_base) ====" >> $cib_cmp_failed + cat $cib_cmp_output 2>/dev/null >> $cib_cmp_failed + cat $cib_cmp_output + fi + + rm $cib_cmp_output .gres.$cib_cmp_output $cib_cmp_expected + + do_cts_assert $rc 0 "diff for $cib_cmp_base found differences" +} + + +#---- + +do_cmd echo "wait for HA to start" +crm_log_pos=$(stat -L -c %s $logfile) +do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & +do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & +do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & +do_cmd ${testdir}/testutils.pl -l ${logfile} -p $crm_log_pos --search -a -m 1500 \ + -s "${test_node_1} heartbeat(.*) info: Starting(.*)cib" \ + -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" \ + -s "${test_node_2} heartbeat(.*) info: Starting(.*)cib" \ + -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" \ + -s "${test_node_3} heartbeat(.*) info: Starting(.*)cib" \ + -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast" +cts_assert "Startup of Heartbeat on ${test_node_1}, ${test_node_2} and ${test_node_3} failed." + +#---- + +do_cmd echo "is master instance?" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -M" +do_cts_assert $? 35 "CIB service on $test_node_1 should not be in master mode by default" + +#---- + +do_cmd echo "local query" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -l" +cts_assert "Could not query local CIB service on $test_node_1" + +#---- + +do_cmd echo "queries with hostname" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -h $test_node_2" +cts_assert "Could not query CIB service on $test_node_2 from $test_node_1" + +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -l -h $test_node_1" +cts_assert "Could not query CIB service on $test_node_1 from $test_node_1" + +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -h $test_node_1" +cts_assert "Could not query CIB service on $test_node_1 from $test_node_1 without -l flag" + +#---- + +do_cmd echo "make sure all instances are slaves" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -r" +cts_assert "Could not set CIB services to slaves" + +#---- + +do_cmd echo "make local the master instance" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -w" +cts_assert "Could not set CIB service on $test_node_1 to master" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -M" +cts_assert "Could not set CIB service on $test_node_1 to master" + +#---- + +do_cmd echo "erase CIB contents and sync" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -E" +cts_assert "Could not erase CIB contents" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -S" +cts_assert "Could not sync CIB to all nodes" + +#---- + +do_cmd echo "verify the CIB contents" +compare_cibs erase_sync_1 $test_node_1 -l +compare_cibs erase_sync_2 $test_node_1 -h $test_node_2 +compare_cibs erase_sync_3 $test_node_1 -l -h $test_node_3 + +#---- + +do_cmd echo "make some nodes in the CIB" +do_cmd make_node $test_node_1 $test_node_1 "member" +cts_assert "Could not create $test_node_1" +do_cmd make_node $test_node_1 $test_node_2 "member" +cts_assert "Could not create $test_node_2" + +#---- + +do_cmd echo "verify the CIB contents" +compare_cibs create_1 $test_node_1 -l +compare_cibs create_2 $test_node_2 -l +compare_cibs create_3 $test_node_3 -l + +#---- + +do_cmd make_node_local $test_node_1 $test_node_3 "member" +cts_assert "Could not create $test_node_3 locally" + +#---- + +do_cmd echo "verify the CIB contents on all nodes" +compare_cibs create_local_1 $test_node_1 -l +compare_cibs create_local_2 $test_node_2 -l +compare_cibs create_local_3 $test_node_3 -l + +#---- + +do_cmd echo "make all slaves" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -r" +cts_assert "Could not set CIB services to slaves" + +do_cmd echo "make node2 master" +do_cmd remote_cmd $CRMD_USER $test_node_2 "$HALIB_DIR/cibadmin -w" +cts_assert "Could not set CIB service on $test_node_1 to master" + +#---- + +do_cmd echo "bump master twice (so we can use it in a sync later)" + +#---- + +do_cmd remote_cmd $CRMD_USER $test_node_2 "$HALIB_DIR/cibadmin -B" +cts_assert "Could not perform bump on $test_node_2" + +do_cmd echo "verify the CIB contents on all nodes" +compare_cibs bump_1 $test_node_1 -l +compare_cibs bump_2 $test_node_2 -l +compare_cibs bump_3 $test_node_3 -l + +#---- + +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -B" +cts_assert "Could not perform bump on $test_node_2 via $test_node_1" + +do_cmd echo "verify the CIB contents on all nodes" +compare_cibs bump_remote_1 $test_node_1 -l +compare_cibs bump_remote_2 $test_node_2 -l +compare_cibs bump_remote_3 $test_node_3 -l + +#---- + +do_cmd echo "make all slaves" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -r" +cts_assert "Could not set CIB services to slaves" + +do_cmd echo "make node1 master" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -w" +cts_assert "Could not set CIB service on $test_node_1 to master" + +#---- + +do_cmd echo "sync from node2" +do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -S -h $test_node_2" +cts_assert "Could not sync from $test_node_2" + +#---- + +do_cmd echo "verify the CIB contents on all nodes" +compare_cibs sync_from_1 $test_node_1 -l +compare_cibs sync_from_2 $test_node_2 -l +compare_cibs sync_from_3 $test_node_3 -l + +echo "test: PASSED" + diff --git a/crm/test/helper.sh.in b/crm/test/helper.sh.in index ad7f9a3ba4..6b3b040cbf 100644 --- a/crm/test/helper.sh.in +++ b/crm/test/helper.sh.in @@ -1,468 +1,518 @@ #!/bin/bash # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # [[ $0 != *helper.sh ]] || { echo >&2 "helper.sh expects to be sourced!" exit 1 } # no more exit below here, use return instead; gives more flexibility ... test_type=`basename $0` test_nodes=1 dump_dir=~/BUILD/tests HALIB_DIR=@libdir@/heartbeat HAVAR_DIR=@HA_VARLIBDIR@/heartbeat HAINIT_DIR=@INITDIR@ INIT_USER=root CRMD_USER=root ADMIN_USER=root test_node_1=fish test_node_2=chips test_node_3=mayo CRM_OPTS="-VV" CRM_ERR_SHUTDOWN=0 OUTPUT_STDERR=0 OUTPUT_STDOUT=0 OUTPUT_CMDS=0 OUTPUT_ECHO=1 repeats=1 iteration=0 done=0 ip_rsc_1=192.168.9.161 ip_rsc_2=192.168.9.162 logfile=/var/log/messages if [ ! -z "$CRM_LOCAL_OPTS" -a -f "$CRM_LOCAL_OPTS" ]; then . "$CRM_LOCAL_OPTS" fi while test "$done" = "0"; do case "$1" in --debug | -x) set -x; shift;; --silent | -s) OUTPUT_ECHO=0; shift;; --stdout | -o) OUTPUT_STDOUT=1; shift;; --stderr | -e) OUTPUT_STDERR=1; shift;; --verbose | -v) OUTPUT_CMDS=1; shift;; --logfile | -l) logfile=${2?no logfile??} shift 2;; --dumpdir | -d) dump_dir=${2?no dumpdir??} shift 2;; --repeat | -r) repeats=${2?no max repetitions??} || return 1; shift 2;; --init | -i) INIT_USER=${2?no init user??} || return 1; shift 2;; --admin | -a) ADMIN_USER=${2?no admin??} || return 1; shift 2;; --crmd | -c) CRMD_USER=${2?no crmd user??} || return 1; shift 2;; --node1 | -1) test_node_1=${2?no node1??} || return 1; shift 2;; --node2 | -2) test_node_2=${2?no node2??} || return 1; shift 2;; --node3 | -3) test_node_3=${2?no node2??} || return 1; shift 2;; --ip1) ip_rsc_1=${2?no ip1??} || return 1; shift 2;; --ip2) ip_rsc_2=${2?no ip2??} || return 1; shift 2;; ""|--) done=1;; *) echo "unknown option: $1"; return 1;; esac done function test_init() { export date_str=`date "+%Y-%b-%d_%H:%M"` export test_dump_dir="$dump_dir/$test_type/$date_str" mkdir -p "$test_dump_dir" echo Test dir: $test_dump_dir } function remote_cmd() { user=$1 host=$2 shift shift `which ssh` ${user}@${host} "$*" return $?; } function is_state() { ret=1 host=$1 state=$2 output=`remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host | grep $state` if [ ! -z "$output" ]; then echo "$host is in $state" return 0 fi echo "$host is NOT in $state" return $ret } function is_running() { rsc=$1 askhost=$2 host=$3 if [ -z $host ]; then output=`remote_cmd $ADMIN_USER $askhost $HALIB_DIR/crmadmin -W $rsc | grep -v $HALIB_DIR/crmadmin` output=`echo $output | grep -v NOT` else output=`remote_cmd $ADMIN_USER $askhost $HALIB_DIR/crmadmin -W $rsc | grep -v $HALIB_DIR/crmadmin` output=`echo $output | grep $host` fi if [ -z "$output" ]; then if [ -z $host ]; then echo "$rsc is NOT running" else echo "$rsc is NOT running on host $host" fi return 1 fi if [ -z $host ]; then echo "$rsc is running" else echo "$rsc is running on host $host" fi return 0 } function is_dc() { host=$1 output=`remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host | grep -v S_PENDING | grep -v S_NOT_DC | grep -v S_ELECTION | grep -v S_RECOVERY` if [ ! -z "$output" ]; then echo "$host is DC" return 0 fi echo "$host is NOT DC" remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host return 1 } function wait_for_state() { state=$1 max=$2 host=$3 target=$4 sleep 1 if [ "$host" = "DC" ]; then host=`remote_cmd $ADMIN_USER $target $HALIB_DIR/crmadmin -D` host=${host#*: } fi count=1 until is_state $host $state; do if [ $count -gt $max ]; then echo "Attempt to reach $state on $host failed" return 1 fi count=`expr $count + 1` sleep 1 done echo "$state reached on $host" return 0 } function make_node() { host=$1 uname=$2 type=$3 uuid=`uuidgen` if [ -z $type ]; then type="member" fi node_xml="''"; remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -C -o node -X $node_xml" } +function make_node_local() +{ + host=$1 + uname=$2 + type=$3 + uuid=`uuidgen` + + if [ -z $type ]; then + type="member" + fi + + node_xml="''"; + + remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -l -C -o node -X $node_xml" +} + function make_resource() { host=$1 id=$2 class=$3 type=$4 priority=$5 version=$6 stopfail=$7 shift 7 args=$* uuid=`uuidgen` node_xml="'&1 > /dev/null } function make_constraint_adv() { host=$1 shift node_xml=$* node_xml=`echo $node_xml | tr '\n' ' '` do_cmd $QUIET remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -C -o constraint -X $node_xml" 2>&1 > /dev/null } function make_constraint() { host=$1 rsc=$2 result=$3 uuid1=`uuidgen` uuid2=`uuidgen` node_xml="''" make_constraint_adv $host $node_xml } function cts_assert() { do_cts_assert $? 0 "Assert failed - " $* } function cts_assert_false() { do_cts_assert $? 1 "AssertFalse failed - " $* } function do_cts_assert() { result=$1 expected=$2 shift shift if [ $result -ne $expected ]; then + do_cmd echo "Expected rc ($expected) != $result" err $* fi } function err() { if [ ! -z $HOST ]; then if [ $CRM_ERR_SHUTDOWN -eq 1 ]; then remote_cmd $ADMIN_USER $test_node_1 killall -9 crmd remote_cmd $ADMIN_USER $test_node_2 killall -9 crmd fi fi do_cmd echo Dumping test nodes to $dump_dir dump_node 1 $test_node_1 dump_node 2 $test_node_2 dump_node 3 $test_node_3 do_cmd echo "ERROR: $*" echo "test $test_type: FAILED" echo "ERROR: $*" > $test_dump_dir/error.txt exit 1 } function dump_node() { a_node_index=$1 a_node=$2 - dump_file="${a_node_index}-${a_node}.tgz" + dump_file="/tmp/${a_node_index}-${a_node}.tgz" `which ssh` ${ADMIN_USER}@${a_node} "ps axf > /tmp/crm/processes" - `which ssh` ${ADMIN_USER}@${a_node} \ - "tar zcvf /tmp/${dump_file} /tmp/crm ~${CRMD_USER}/core" - - scp -r ${ADMIN_USER}@${a_node}:/tmp/${dump_file} $test_dump_dir + `which ssh` ${ADMIN_USER}@${a_node} "tar zcvf ${dump_file} /tmp/crm" + scp -r ${ADMIN_USER}@${a_node}:${dump_file} $test_dump_dir + `which ssh` ${ADMIN_USER}@${a_node} "rm ${dump_file}" } function do_cmd() { cmdline=$* old_OUTPUT_STDOUT=$OUTPUT_STDOUT cmd=$1 if [ $OUTPUT_STDERR -eq 1 -o $OUTPUT_STDOUT -eq 1 -o $OUTPUT_CMDS -eq 1 ]; then if [ $cmd = "remote_cmd" ]; then user=$2 host=$3 shift 3 echo `date` ": Running '$*' as ${user}@${host}" | tee -a $test_dump_dir/test.txt; elif [ $cmd = "wait_for_state" ]; then host=$4 if [ -z $host ]; then host="the DC" fi echo `date` ": Waiting for state $2 on $host (max $3 attempts)" | tee -a $test_dump_dir/test.txt; elif [ $cmd = "${HALIB_DIR}/crmtest/testutils.pl" ]; then echo `date` ": Searching '$cmdline'" | tee -a $test_dump_dir/test.txt; elif [ $cmd = "echo" ]; then echo -n "" # if [ $OUTPUT_CMDS -eq 1 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then # echo -n `date` ": " # $cmdline # elif [ $OUTPUT_ECHO -eq 1 -a $OUTPUT_CMDS -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then # $cmdline # fi else echo `date` ": Running '$cmdline'" | tee -a $test_dump_dir/test.txt; fi fi if [ $cmd = "echo" ]; then if [ $OUTPUT_CMDS -eq 1 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then echo -n `date` ": " | tee -a $test_dump_dir/test.txt echo -n -e "\033[01;33m" $cmdline | tee -a $test_dump_dir/test.txt echo -n -e "\033[00m" elif [ $OUTPUT_ECHO -eq 1 -a $OUTPUT_CMDS -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then echo -n `date` ": " | tee -a $test_dump_dir/test.txt echo -n -e "\033[01;33m" $cmdline | tee -a $test_dump_dir/test.txt echo -n -e "\033[00m" fi elif [ $OUTPUT_STDERR -eq 1 -a $OUTPUT_STDOUT -eq 1 ]; then $cmdline 2>&1 >> $test_dump_dir/test.txt rc=$? elif [ $OUTPUT_STDERR -eq 0 -a $OUTPUT_STDOUT -eq 0 ]; then $cmdline 2>&1 >/dev/null rc=$? elif [ $OUTPUT_STDOUT -eq 0 ]; then $cmdline >/dev/null 2>> $test_dump_dir/test.txt rc=$? elif [ $OUTPUT_STDERR -eq 0 ]; then $cmdline 2>/dev/null >> $test_dump_dir/test.txt rc=$? else $cmdline 2>&1 >> $test_dump_dir/test.txt rc=$? fi if [ $cmd = "${HALIB_DIR}/crmtest/testutils.pl" ]; then # for the logs... if [ $OUTPUT_STDERR -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_CMDS -eq 0 ]; then echo `date` ": Searching '$cmdline'" >> $test_dump_dir/test.txt; fi $cmdline -v 2>&1 >> $test_dump_dir/test.txt if [ $rc -lt 0 ]; then echo `date` ": Searching '$cmdline'" $cmdline fi fi OUTPUT_STDOUT=$old_OUTPUT_STDOUT # echo Result: $rc return $rc } function crm-cleanup() { if [ -z $1 ]; then test_init fi do_cmd echo Cleaning up on $test_node_1, $test_node_2 and $test_node_3 crm-cleanup-node $test_node_1 $1 crm-cleanup-node $test_node_2 $1 crm-cleanup-node $test_node_3 $1 } function crm-cleanup-node() { ha_host=$1 do_cmd echo Cleaning up on $ha_host # stop all running HAs do_cmd remote_cmd $INIT_USER $ha_host $HALIB_DIR/heartbeat "-k" "2>&1 >/dev/null" # be *very* sure everything has stopped do_cmd remote_cmd $INIT_USER $ha_host "killall -q9 heartbeat ccm lrmd crmd" # resources too do_cmd remote_cmd $INIT_USER $ha_host "/etc/ha.d/resource.d/IPaddr $ip_rsc_1 stop" 2>&1 > /dev/null do_cmd remote_cmd $INIT_USER $ha_host "/etc/ha.d/resource.d/IPaddr $ip_rsc_2 stop" 2>&1 > /dev/null if [ -z $2 ]; then # make *sure* theres nothing left over from last time do_cmd remote_cmd $INIT_USER $ha_host "rm -f $HAVAR_DIR/crm/cib*.xml" - remote_cmd $INIT_USER $ha_host "rm -f /tmp/crm/*" 2>&1 > /dev/null - remote_cmd $CRMD_USER $ha_host "touch /tmp/crm/cluster.log" + `which ssh` ${INIT_USER}@${ha_host} "rm -f /tmp/crm/*" 2>/dev/null remote_cmd $CRMD_USER $ha_host "rm -f /tmp/crm/*/core" 2>&1 > /dev/null do_cmd remote_cmd $INIT_USER $ha_host "/etc/init.d/syslog restart" + do_cmd remote_cmd $CRMD_USER $ha_host "logger -i -p local7.info __crmtest_${test_type}_log_mark__" + do_cmd sleep 2 # zero out logs, maybe if [ ${test_node_1} = "fish" ]; then do_cmd remote_cmd $INIT_USER $ha_host "cat /dev/null > /var/log/messages" fi fi } function mark_log() { export crm_log_pos=$(stat -L -c %s $logfile) } +function gres() +{ + if [ $# -lt "3" ] + then + echo Usage: gres pattern replacement files + return 1 + fi + pattern=$1 + replacement=$2 + shift; shift; + filelist=$* + A="`echo | tr '\012' '\001' `" + + for file in $filelist; do + + fname=`basename $file` + dname=`dirname $file` + + is_backup=${fname##.gres.} + if [ "$is_backup" = "$fname" ]; then + grep -q "$pattern" "$file" + isthere=$? + + if [ $isthere = 0 ]; then +# echo Processing $file... + cp "$file" "$dname/.gres.$fname" && sed -e "s$A$pattern$A$replacement$A" "$dname/.gres.$fname" > "$file" + fi +# else +# echo "Ignoring backup of $is_backup in $dname" + fi + done +} + # successfully sourced true