diff --git a/crm/Makefile.am b/crm/Makefile.am index eca9a7db84..3f4511d980 100644 --- a/crm/Makefile.am +++ b/crm/Makefile.am @@ -1,43 +1,43 @@ # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # MAINTAINERCLEANFILES = Makefile.in ## Subdirectories -SUBDIRS = cib crmd pengine tengine admin test -DIST_SUBDIRS = cib crmd pengine tengine admin test +SUBDIRS = cib crmd pengine tengine admin +DIST_SUBDIRS = cib crmd pengine tengine admin dtddir = $(HA_LIBDIR)/$(HB_PKG) crm_varlibdir = $(HA_VARLIBDIR)/$(HB_PKG)/crm crm_varrundir = $(HA_VARRUNDIR)/$(HB_PKG)/crm noinst_HEADERS = dmalloc_wrapper.h EXTRA_DIST = crm-1.0.dtd dtd_SCRIPTS = crm.dtd crm.dtd: crm-1.0.dtd cp $(top_srcdir)/crm/crm-1.0.dtd crm.dtd install-exec-local: $(mkinstalldirs) $(DESTDIR)/$(crm_varlibdir) -chown $(HA_CCMUSER) $(DESTDIR)/$(crm_varlibdir) -chgrp $(HA_APIGROUP) $(DESTDIR)/$(crm_varlibdir) -chmod 750 $(DESTDIR)/$(crm_varlibdir) $(mkinstalldirs) $(DESTDIR)/$(crm_varrundir) -chown $(HA_CCMUSER) $(DESTDIR)/$(crm_varrundir) -chgrp $(HA_APIGROUP) $(DESTDIR)/$(crm_varrundir) -chmod 750 $(DESTDIR)/$(crm_varrundir) diff --git a/crm/test/1node.sh.in b/crm/test/1node.sh.in deleted file mode 100644 index 3651d998e8..0000000000 --- a/crm/test/1node.sh.in +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -required_nodes=1 -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -while [ $iteration -lt $repeats ]; do - iteration=`expr $iteration + 1` - echo -ne "$test_type : Iteration $iteration of $repeats" - echo "########### Begining iteration $iteration of $repeats ###########" -( - -# make *sure* theres nothing left over from last time -crm-cleanup - -#---- - -start_node $test_node_1 - -#---- - -do_cmd echo "Analyse CRMd startup" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90 \ - -s "crmd(.*) State transition (.*) \-> S_ELECTION" \ - -s "crmd(.*) State transition (.*) \-> S_INTEGRATION" -cts_assert "DC election on ${test_node_1} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 900 \ - -s "Updating node state to member for ${test_node_1}" \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -cts_assert "IDLE and CRMd join on ${test_node_1} failed." - -do_cmd wait_for_state S_IDLE 30 $test_node_1 -cts_assert "S_IDLE not reached on $test_node_1 (startup)!" - -#---- - -do_cmd echo Create the first constraint and wait for S_IDLE -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint $test_node_1 rsc1 0 -cts_assert "Creation of first constraint failed." -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 120 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -#cts_assert S_IDLE not reached after first constraint created - -#---- - -do_cmd echo Create the second constraint and wait for S_IDLE -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint $test_node_1 rsc2 0 -cts_assert "Creation of second constraint failed." -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 120 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -#cts_assert S_IDLE not reached after second constraint created - -#---- - -do_cmd echo Create the first resource and wait for S_IDLE after start -args="" -args="$args " -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc1 ocf IPaddr - - - $args -cts_assert "Creation of first resource failed." -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 300 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" -cts_assert S_IDLE not reached after first resource created - -#---- - -do_cmd echo Create the second resource and wait for S_IDLE after start -args="" -args="$args " -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc2 ocf IPaddr - - - $args -cts_assert "Creation of second resource failed." -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 300 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" -cts_assert S_IDLE not reached after second resource created - -#---- - -do_cmd echo Check for S_IDLE, the DC and running resources -do_cmd wait_for_state S_IDLE 30 DC $test_node_1 -cts_assert "S_IDLE not reached on $test_node_1 after CIB create" - -do_cmd is_running rsc1 $test_node_1 -cts_assert "rsc1 NOT running" - -do_cmd wait_for_state S_IDLE 30 $test_node_1 -cts_assert "S_IDLE not maintained on $test_node_1!" - -do_cmd is_running rsc2 $test_node_1 -cts_assert "rsc2 NOT running" - -do_cmd is_dc $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -do_cmd is_running rsc1 $test_node_1 x$test_node_1 -cts_assert_false "rsc1 IS running on x$test_node_1" - -do_cmd is_running rsc1 $test_node_1 $test_node_1 -cts_assert "rsc1 NOT running on $test_node_1" - -do_cmd is_running rsc2 $test_node_1 $test_node_1 -cts_assert "rsc2 NOT running on $test_node_1" - -#---- - -stop_node $test_node_1 - -#---- - -do_cmd remote_cmd $CRMD_USER $test_node_1 killall -9q crmd - -echo "test: PASSED" -) - -done diff --git a/crm/test/2node-empty.sh.in b/crm/test/2node-empty.sh.in deleted file mode 100644 index 30b30fc7f3..0000000000 --- a/crm/test/2node-empty.sh.in +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -required_nodes=2 -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -CRM_ERR_SHUTDOWN=0 -test_nodes=2 - -while [ $iteration -lt $repeats ]; do - iteration=`expr $iteration + 1` - echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" - echo "########### Begining iteration $iteration of $repeats ###########" -( -# make *sure* theres nothing left over from last time -crm-cleanup - -#---- - -start_node $test_node_1 - -#---- - -do_cmd echo "Analyse CRMd startup" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -cts_assert "CRMd startup on ${test_node_1} failed." - -do_cmd wait_for_state S_IDLE 3 $test_node_1 -cts_assert "S_IDLE not reached on $test_node_1 (startup)!" - -#---- - -do_cmd echo Various sanity checks - stage 1 - -do_cmd is_running rsc1 $test_node_1 -cts_assert_false "rsc1 IS running" - -do_cmd is_running rsc2 $test_node_1 -cts_assert_false "rsc2 IS running" - -do_cmd is_dc $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -#---- - -start_node $test_node_2 - -#---- - -do_cmd echo "Analyse CRMd startup" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "${test_node_1} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${test_node_2} crmd(.*) State transition S_PENDING \-> S_NOT_DC" -cts_assert "CRMd on ${test_node_2} did not join the cluster." - -#---- - -do_cmd echo Various sanity checks - stage 2 -do_cmd wait_for_state S_IDLE 3 $test_node_1 -cts_assert "S_IDLE not reached on $test_node_1" - -do_cmd is_running rsc1 $test_node_1 -cts_assert_false "rsc1 IS running" - -do_cmd is_running rsc2 $test_node_1 -cts_assert_false "rsc2 IS running" - -do_cmd is_dc $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -#---- - -stop_node $test_node_1 crmd - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_2} crmd(.*) State transition (.*) \-> S_ELECTION" \ - -s "${test_node_1} crmd(.*) State transition (.*) \-> S_NOT_DC" -cts_assert "Transition of the DC from ${test_node_1} to ${test_node_2} failed." - -#---- - -do_cmd echo Various sanity checks - stage 3 -do_cmd wait_for_state S_IDLE 60 $test_node_2 -cts_assert "S_IDLE not reached on $test_node_2 after shutdown!" - -do_cmd is_running rsc1 $test_node_1 -cts_assert_false "rsc1 IS running" - -do_cmd is_running rsc2 $test_node_1 -cts_assert_false "rsc2 IS running" - -do_cmd is_dc $test_node_2 -cts_assert "$test_node_2 is supposed to be the DC" - -#---- - -stop_node $test_node_2 crmd - -#---- - -do_cmd remote_cmd $CRMD_USER $test_node_2 killall -9 crmd - -echo "test: PASSED" -) -done diff --git a/crm/test/2node-fail.sh.in b/crm/test/2node-fail.sh.in deleted file mode 100644 index 6a177cccf3..0000000000 --- a/crm/test/2node-fail.sh.in +++ /dev/null @@ -1,337 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -required_nodes=2 -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -CRM_ERR_SHUTDOWN=0 -test_nodes=2 - - -function 2node_fail_test() { - - test_type=$1 - fail_pieces=$2 - fail_node=$3 - good_node=$4 - do_failback=$5 - accelerated=$6 - - if [ $fail_node = $test_node_1 ]; then - moved_rsc=rsc1 - else - moved_rsc=rsc2 - fi - -#---- - - echo -ne "\033]0;$test_type: Iteration $iteration of $repeats\007" - crm-cleanup - do_cmd echo "#############################" - do_cmd echo "$test_type: Iteration $iteration of $repeats" -# make *sure* theres nothing left over from last time - -#---- - - do_cmd echo "wait for HA to start on ${test_node_1}" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ - -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" - cts_assert "Startup of Heartbeat on ${test_node_1} failed." - -#---- - - do_cmd echo "wait for CRMd to start on ${test_node_1}" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert "CRMd startup on ${test_node_1} failed." - - do_cmd wait_for_state S_IDLE 3 $test_node_1 - cts_assert "S_IDLE not reached on $test_node_1 (startup)!" - -#---- - - do_cmd echo Create the first constraint and wait for S_IDLE - rsc=rsc1 - uuid1=`uuidgen` - uuid2=`uuidgen` - uuid3=`uuidgen` - node_xml=" - - - - - " - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd make_constraint_adv $test_node_1 $node_xml - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert Adding constraint1 did not pass - -#---- - - do_cmd echo Create the second constraint and wait for S_IDLE - rsc=rsc2 - uuid1=`uuidgen` - uuid2=`uuidgen` - uuid3=`uuidgen` - node_xml=" - - - - - " - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd make_constraint_adv $test_node_1 $node_xml - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert Adding constraint2 did not pass - -#---- - - do_cmd echo Create the first resource and wait for S_IDLE after start - args="" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - ignore $args - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" - cts_assert Adding rsc1 did not pass - -#---- - - do_cmd echo Create the second resource and wait for S_IDLE after start - args="" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - ignore $args - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" - cts_assert Adding rsc2 did not pass - -#---- - - do_cmd echo Various sanity checks - stage 1 - do_cmd wait_for_state S_IDLE 3 $test_node_1 - cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" - - do_cmd is_running rsc1 $test_node_1 - cts_assert "rsc1 NOT running" - - do_cmd is_running rsc2 $test_node_1 - cts_assert "rsc2 NOT running" - - do_cmd is_dc $test_node_1 - cts_assert "$test_node_1 is supposed to be the DC" - - do_cmd is_running rsc1 $test_node_1 x$test_node_1 - cts_assert_false "rsc1 IS running on x$test_node_1" - - do_cmd is_running rsc1 $test_node_1 $test_node_1 - cts_assert "rsc1 NOT running on $test_node_1" - - do_cmd is_running rsc2 $test_node_1 $test_node_1 - cts_assert "rsc2 NOT running on $test_node_1" - -#---- - - do_cmd echo "wait for HA to start on $test_node_2" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" \ - -s "${test_node_2} heartbeat(.*) info: Starting (.*)lrmd" \ - -e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast" - cts_assert "Startup of Heartbeat on ${test_node_2} failed." - -#---- - - do_cmd echo "wait for CRMd to start on $test_node_2" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" \ - -s "${test_node_2} crmd(.*) State transition S_PENDING \-> S_NOT_DC" - cts_assert "CRMd startup on ${test_node_2} failed." - - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 8000 \ - -s "${test_node_1} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" - cts_assert "rsc2 was not transferred to ${test_node_2} on startup." - -#---- - - do_cmd echo Various sanity checks - stage 2 - do_cmd wait_for_state S_NOT_DC 30 $test_node_2 - cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" - - do_cmd wait_for_state S_IDLE 30 $test_node_1 - cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" - - do_cmd is_running rsc1 $test_node_1 - cts_assert "rsc1 NOT running" - - do_cmd is_running rsc2 $test_node_1 - cts_assert "rsc2 NOT running" - - do_cmd is_running rsc1 $test_node_1 $test_node_1 - cts_assert "rsc1 NOT running on $test_node_1" - - do_cmd is_running rsc2 $test_node_1 $test_node_2 - cts_assert "rsc2 NOT running on $test_node_2" - -#---- - - is_dc $fail_node 2>&1 > /dev/null - test_for_election=$? - - do_cmd echo Killing $fail_pieces on $fail_node - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $ADMIN_USER $fail_node "killall -9 $fail_pieces" & - - if [ $test_for_election = 0 ]; then - do_cmd echo Killed the DC... checking for DC Failover - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90000 \ - -s "${good_node} crmd(.*) State transition" \ - -s "${good_node} crmd(.*) State transition (.*) \-> S_ELECTION" \ - -s "${good_node} crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert "Transition of the DC from ${fail_node} to ${good_node} failed." - else - do_cmd echo Killed slave node... checking the DC noticed - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90000 \ - -s "${good_node} crmd(.*) State transition" \ - -s "${good_node} crmd(.*) State transition (.*) \-> S_POLICY_ENGINE" \ - -s "${good_node} crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert "Failure of slave node ${fail_node} was not noticed on the DC (${good_node})." - fi - - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 80000 \ - -s "${god_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" - cts_assert "Move of ${moved_rsc} to ${good_node} failed." - -#---- - - do_cmd echo Various sanity checks - stage 3 - do_cmd wait_for_state S_IDLE 60 $good_node - cts_assert "S_IDLE not reached on $good_node after kill!" - - do_cmd is_running rsc1 $good_node - cts_assert "rsc1 NOT running on $good_node" - - do_cmd is_running rsc2 $good_node - cts_assert "rsc2 NOT running on $good_node" - - do_cmd is_running rsc1 $good_node $fail_node - cts_assert_false "rsc1 IS running on $fail_node" - - do_cmd is_running rsc2 $good_node $good_node - cts_assert "rsc2 NOT running on $good_node" - -#---- - - if [ $do_failback = 1 ]; then - do_cmd echo "Re-Starting on failed node $fail_node" - - if [ "$fail_pieces" = "crmd" ]; then - do_cmd echo "HA still running, skipping restart" - else - do_cmd echo "wait for HA to start on $fail_node" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $INIT_USER $fail_node $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${fail_node} ccm(.*) Hostname: ${fail_node}" \ - -s "${fail_node} heartbeat(.*) info: Starting (.*)lrmd" \ - -e "${fail_node} heartbeat(.*) Client (.*) respawning too fast" - cts_assert "Startup of Heartbeat on ${fail_node} failed." - fi - -#---- - - do_cmd echo "wait for CRMd to start on $fail_node" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $CRMD_USER $fail_node $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 20000 \ - -s "${fail_node} crmd(.*)FSA Hostname: ${fail_node}" \ - -s "${fail_node} crmd(.*) State transition S_PENDING \-> S_NOT_DC" - cts_assert "CRMd startup on ${fail_node} failed." - - do_cmd echo "Checking ${moved_rsc} was moved back" - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 20000 \ - -s "${good_node} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${fail_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" - cts_assert "$moved_rsc was not transferred to ${fail_node} on startup." - - do_cmd echo Various sanity checks - stage 4 - do_cmd wait_for_state S_NOT_DC 30 $fail_node - cts_assert "S_NOT_DC not reached on $fail_node (restart)!" - - do_cmd wait_for_state S_IDLE 30 $good_node - cts_assert "S_IDLE not reached on $good_node (restart)!" - - do_cmd is_running rsc1 $test_node_1 - cts_assert "rsc1 NOT running" - - do_cmd is_running rsc2 $test_node_1 - cts_assert "rsc2 NOT running" - - do_cmd is_running rsc1 $test_node_1 $test_node_1 - cts_assert "rsc1 NOT running on $test_node_1" - - do_cmd is_running rsc2 $test_node_1 $test_node_2 - cts_assert "rsc2 NOT running on $test_node_2" - - fi - -#---- - do_cmd echo "test ${test_type}: PASSED" - -} - - -while [ $iteration -lt $repeats ]; do - iteration=`expr $iteration + 1` - echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" - echo "########### $test_type : Begining iteration $iteration of $repeats ###########" -( - 2node_fail_test 2node__failback_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 1 0 - - 2node_fail_test 2node__fail_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 0 0 - 2node_fail_test 2node__fail_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 0 0 - 2node_fail_test 2node__failback_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 1 0 - 2node_fail_test 2node__failback_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 1 0 - - 2node_fail_test 2node__fail_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 0 0 - 2node_fail_test 2node__fail_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 0 0 - 2node_fail_test 2node__failback_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 1 0 - 2node_fail_test 2node__failback_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 1 0 - -#2node_fail_test 2node__fail_slave_LRM "lrmd" ${test_node_2} ${test_node_1} -#2node_fail_test 2node__fail_DC_LRM "lrmd" ${test_node_1} ${test_node_2} - - echo "test suite: PASSED" -) -done - diff --git a/crm/test/2node-fence.sh.in b/crm/test/2node-fence.sh.in deleted file mode 100644 index 60e7bd468c..0000000000 --- a/crm/test/2node-fence.sh.in +++ /dev/null @@ -1,347 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -required_nodes=2 -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -CRM_ERR_SHUTDOWN=0 -test_nodes=2 - - -function 2node_fail_test() { - - test_type=$1 - fail_pieces=$2 - fail_node=$3 - good_node=$4 - do_failback=$5 - accelerated=$6 - - if [ $fail_node = $test_node_1 ]; then - moved_rsc=rsc1 - else - moved_rsc=rsc2 - fi - -#---- - - echo -ne "\033]0;$test_type: Iteration $iteration of $repeats\007" - crm-cleanup - do_cmd echo "#############################" - do_cmd echo "$test_type: Iteration $iteration of $repeats" -# make *sure* theres nothing left over from last time - -#---- - - do_cmd echo "wait for HA to start on ${test_node_1}" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ - -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" - cts_assert "Startup of Heartbeat on ${test_node_1} failed." - -#---- - - do_cmd echo "wait for CRMd to start on ${test_node_1}" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert "CRMd startup on ${test_node_1} failed." - - do_cmd wait_for_state S_IDLE 3 $test_node_1 - cts_assert "S_IDLE not reached on $test_node_1 (startup)!" - -#---- - - do_cmd echo Create the first constraint and wait for S_IDLE - rsc=rsc1 - uuid1=`uuidgen` - uuid2=`uuidgen` - uuid3=`uuidgen` - node_xml="' - - - - - '" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd make_constraint_adv $test_node_1 $node_xml - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert Adding constraint1 did not pass - -#---- - - do_cmd echo Create the second constraint and wait for S_IDLE - rsc=rsc2 - uuid1=`uuidgen` - uuid2=`uuidgen` - uuid3=`uuidgen` - node_xml="' - - - - - '" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd make_constraint_adv $test_node_1 $node_xml - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert Adding constraint2 did not pass - -#---- - - do_cmd echo Create the first resource and wait for S_IDLE after start - args="" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - stonith $args - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" - cts_assert Adding rsc1 did not pass - -#---- - - do_cmd echo Create the second resource and wait for S_IDLE after start - args="" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - stonith $args - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" - cts_assert Adding rsc2 did not pass - -#---- - - do_cmd echo Create the fencing resource and wait for S_IDLE after start - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd make_incarnation $test_node_1 fence1 stonith null 2 - - ignore - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on fence1" - cts_assert Adding fence1 did not pass - -#---- - - do_cmd echo Various sanity checks - stage 1 - do_cmd wait_for_state S_IDLE 3 $test_node_1 - cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" - - do_cmd is_running rsc1 $test_node_1 - cts_assert "rsc1 NOT running" - - do_cmd is_running rsc2 $test_node_1 - cts_assert "rsc2 NOT running" - - do_cmd is_dc $test_node_1 - cts_assert "$test_node_1 is supposed to be the DC" - - do_cmd is_running rsc1 $test_node_1 x$test_node_1 - cts_assert_false "rsc1 IS running on x$test_node_1" - - do_cmd is_running rsc1 $test_node_1 $test_node_1 - cts_assert "rsc1 NOT running on $test_node_1" - - do_cmd is_running rsc2 $test_node_1 $test_node_1 - cts_assert "rsc2 NOT running on $test_node_1" - -#---- - - do_cmd echo "wait for HA to start on $test_node_2" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_2} ccm(.*) Hostname: ${test_node_2}" \ - -s "${test_node_2} heartbeat(.*) info: Starting (.*)lrmd" \ - -e "${test_node_2} heartbeat(.*) Client (.*) respawning too fast" - cts_assert "Startup of Heartbeat on ${test_node_2} failed." - -#---- - - do_cmd echo "wait for CRMd to start on $test_node_2" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_2} crmd(.*)FSA Hostname: ${test_node_2}" \ - -s "${test_node_2} crmd(.*) State transition S_PENDING \-> S_NOT_DC" - cts_assert "CRMd startup on ${test_node_2} failed." - - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 8000 \ - -s "${test_node_1} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" - cts_assert "rsc2 was not transferred to ${test_node_2} on startup." - -#---- - - do_cmd echo Various sanity checks - stage 2 - do_cmd wait_for_state S_NOT_DC 30 $test_node_2 - cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" - - do_cmd wait_for_state S_IDLE 30 $test_node_1 - cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" - - do_cmd is_running rsc1 $test_node_1 - cts_assert "rsc1 NOT running" - - do_cmd is_running rsc2 $test_node_1 - cts_assert "rsc2 NOT running" - - do_cmd is_running rsc1 $test_node_1 $test_node_1 - cts_assert "rsc1 NOT running on $test_node_1" - - do_cmd is_running rsc2 $test_node_1 $test_node_2 - cts_assert "rsc2 NOT running on $test_node_2" - -#---- - - is_dc $fail_node 2>&1 > /dev/null - test_for_election=$? - - do_cmd echo Killing $fail_pieces on $fail_node - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $ADMIN_USER $fail_node "killall -9 $fail_pieces" & - - if [ $test_for_election = 0 ]; then - do_cmd echo Killed the DC... checking for DC Failover - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90000 \ - -s "${good_node} crmd(.*) State transition" \ - -s "${good_node} crmd(.*) State transition (.*) \-> S_ELECTION" \ - -s "${good_node} crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert "Transition of the DC from ${fail_node} to ${good_node} failed." - else - do_cmd echo Killed slave node... checking the DC noticed - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90000 \ - -s "${good_node} crmd(.*) State transition" \ - -s "${good_node} crmd(.*) State transition (.*) \-> S_POLICY_ENGINE" \ - -s "${good_node} crmd(.*) State transition (.*) \-> S_IDLE" - cts_assert "Failure of slave node ${fail_node} was not noticed on the DC (${good_node})." - fi - - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 80000 \ - -s "${god_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" - cts_assert "Move of ${moved_rsc} to ${good_node} failed." - -#---- - - do_cmd echo Various sanity checks - stage 3 - do_cmd wait_for_state S_IDLE 60 $good_node - cts_assert "S_IDLE not reached on $good_node after kill!" - - do_cmd is_running rsc1 $good_node - cts_assert "rsc1 NOT running on $good_node" - - do_cmd is_running rsc2 $good_node - cts_assert "rsc2 NOT running on $good_node" - - do_cmd is_running rsc1 $good_node $fail_node - cts_assert_false "rsc1 IS running on $fail_node" - - do_cmd is_running rsc2 $good_node $good_node - cts_assert "rsc2 NOT running on $good_node" - -#---- - - if [ $do_failback = 1 ]; then - do_cmd echo "Re-Starting on failed node $fail_node" - - if [ "$fail_pieces" = "crmd" ]; then - do_cmd echo "HA still running, skipping restart" - else - do_cmd echo "wait for HA to start on $fail_node" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $INIT_USER $fail_node $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${fail_node} ccm(.*) Hostname: ${fail_node}" \ - -s "${fail_node} heartbeat(.*) info: Starting (.*)lrmd" \ - -e "${fail_node} heartbeat(.*) Client (.*) respawning too fast" - cts_assert "Startup of Heartbeat on ${fail_node} failed." - fi - -#---- - - do_cmd echo "wait for CRMd to start on $fail_node" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $CRMD_USER $fail_node $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 20000 \ - -s "${fail_node} crmd(.*)FSA Hostname: ${fail_node}" \ - -s "${fail_node} crmd(.*) State transition S_PENDING \-> S_NOT_DC" - cts_assert "CRMd startup on ${fail_node} failed." - - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 20000 \ - -s "${good_node} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${fail_node} crmd(.*) Performing op start(.*) on ${moved_rsc}" - cts_assert "$moved_rsc was not transferred to ${fail_node} on startup." - - do_cmd echo Various sanity checks - stage 4 - do_cmd wait_for_state S_NOT_DC 30 $fail_node - cts_assert "S_NOT_DC not reached on $fail_node (restart)!" - - do_cmd wait_for_state S_IDLE 30 $good_node - cts_assert "S_IDLE not reached on $good_node (restart)!" - - do_cmd is_running rsc1 $test_node_1 - cts_assert "rsc1 NOT running" - - do_cmd is_running rsc2 $test_node_1 - cts_assert "rsc2 NOT running" - - do_cmd is_running rsc1 $test_node_1 $test_node_1 - cts_assert "rsc1 NOT running on $test_node_1" - - do_cmd is_running rsc2 $test_node_1 $test_node_2 - cts_assert "rsc2 NOT running on $test_node_2" - - fi - -#---- - - do_cmd echo "test ${test_type}: PASSED" - -} - - -while [ $iteration -lt $repeats ]; do - iteration=`expr $iteration + 1` - echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" - echo "########### $test_type : Begining iteration $iteration of $repeats ###########" -( - - 2node_fail_test 2node__fail_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 0 0 - 2node_fail_test 2node__fail_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 0 0 - 2node_fail_test 2node__failback_DC_All "heartbeat ccm lrmd crmd" ${test_node_1} ${test_node_2} 1 0 - 2node_fail_test 2node__failback_slave_All "heartbeat ccm lrmd crmd" ${test_node_2} ${test_node_1} 1 0 - - 2node_fail_test 2node__fail_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 0 0 - 2node_fail_test 2node__fail_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 0 0 - 2node_fail_test 2node__failback_DC_CRMd "crmd" ${test_node_1} ${test_node_2} 1 0 - 2node_fail_test 2node__failback_slave_CRMd "crmd" ${test_node_2} ${test_node_1} 1 0 - -#2node_fail_test 2node__fail_slave_LRM "lrmd" ${test_node_2} ${test_node_1} -#2node_fail_test 2node__fail_DC_LRM "lrmd" ${test_node_1} ${test_node_2} - - echo "test suite: PASSED" -) -done - - diff --git a/crm/test/2node.sh.in b/crm/test/2node.sh.in deleted file mode 100644 index 7b8d2c9f23..0000000000 --- a/crm/test/2node.sh.in +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -required_nodes=2 -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -CRM_ERR_SHUTDOWN=0 -test_nodes=2 - -while [ $iteration -lt $repeats ]; do - iteration=`expr $iteration + 1` - echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" - echo "########### Begining iteration $iteration of $repeats ###########" -( -# make *sure* theres nothing left over from last time -crm-cleanup - -#---- - -start_node $test_node_1 - -#---- - -do_cmd echo "Analyse CRMd startup" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 90 \ - -s "crmd(.*) State transition (.*) \-> S_ELECTION" \ - -s "crmd(.*) State transition (.*) \-> S_INTEGRATION" -cts_assert "DC election on ${test_node_1} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "Updating node state to member for ${test_node_1}" \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -cts_assert "IDLE and CRMd join on ${test_node_1} failed." - -do_cmd wait_for_state S_IDLE 30 $test_node_1 -cts_assert "S_IDLE not reached on $test_node_1 (startup)!" - -#---- - -do_cmd echo Create the first constraint and wait for S_IDLE -rsc=rsc1 -uuid1=`uuidgen` -uuid2=`uuidgen` -uuid3=`uuidgen` -node_xml=" - - - - - " -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint_adv $test_node_1 $node_xml -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 100 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -#cts_assert Adding constraint1 did not pass - -#---- - -do_cmd echo Create the second constraint and wait for S_IDLE -rsc=rsc2 -uuid1=`uuidgen` -uuid2=`uuidgen` -uuid3=`uuidgen` -node_xml=" - - - - - " -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint_adv $test_node_1 $node_xml -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 100 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -#cts_assert Adding constraint2 did not pass - -#---- - -do_cmd echo Create the first resource and wait for S_IDLE after start -args="" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 200 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" -cts_assert Adding rsc1 did not pass - -#---- - -do_cmd echo Create the second resource and wait for S_IDLE after start -args="" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 200 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" -cts_assert Adding rsc2 did not pass - -#---- - -do_cmd echo Various sanity checks - stage 1 -do_cmd wait_for_state S_IDLE 3 $test_node_1 -cts_assert "S_IDLE not reached on $test_node_1 (CIB create)!" - -do_cmd is_running rsc1 $test_node_1 -cts_assert "rsc1 NOT running" - -do_cmd is_running rsc2 $test_node_1 -cts_assert "rsc2 NOT running" - -do_cmd is_dc $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -do_cmd is_running rsc1 $test_node_1 x$test_node_1 -cts_assert_false "rsc1 IS running on x$test_node_1" - -do_cmd is_running rsc1 $test_node_1 $test_node_1 -cts_assert "rsc1 NOT running on $test_node_1" - -do_cmd is_running rsc2 $test_node_1 $test_node_1 -cts_assert "rsc2 NOT running on $test_node_1" - -#---- - -start_node $test_node_2 - -#---- - -do_cmd echo "Analyse CRMd startup" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 400 \ - -s "${test_node_2} crmd(.*) State transition S_PENDING \-> S_NOT_DC" -cts_assert "CRMd startup on ${test_node_2} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2000 \ - -s "${test_node_1} crmd(.*) State transition(.*) \-> S_IDLE" \ - -s "${test_node_2} crmd(.*) Performing op start(.*) on rsc2" -cts_assert "rsc2 was not transferred to ${test_node_2} on startup." - -#---- - -do_cmd echo Various sanity checks - stage 2 -do_cmd wait_for_state S_NOT_DC 30 $test_node_2 -cts_assert "S_NOT_DC not reached on $test_node_2 (startup - 2)!" - -do_cmd wait_for_state S_IDLE 30 $test_node_1 -cts_assert "S_IDLE not reached on $test_node_1 (startup - 2)!" - -do_cmd is_running rsc1 $test_node_1 -cts_assert "rsc1 NOT running" - -do_cmd is_running rsc2 $test_node_1 -cts_assert "rsc2 NOT running" - -do_cmd is_running rsc1 $test_node_1 $test_node_1 -cts_assert "rsc1 NOT running on $test_node_1" - -do_cmd is_running rsc2 $test_node_1 $test_node_2 -cts_assert "rsc2 NOT running on $test_node_2" - -#---- - -stop_node $test_node_1 crmd - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2000 \ - -s "${test_node_2} crmd(.*) State transition (.*) \-> S_ELECTION" \ - -s "${test_node_1} crmd(.*) State transition (.*) \-> S_NOT_DC" -cts_assert "Transition of the DC from ${test_node_1} to ${test_node_2} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 4000 \ - -s "crmd(.*) Performing op start(.*) on rsc1" -cts_assert "rsc1 was not transferred to ${test_node_2} on shutdown." - -#---- - -do_cmd echo Various sanity checks - stage 3 -do_cmd wait_for_state S_IDLE 60 $test_node_2 -cts_assert "S_IDLE not reached on $test_node_2 after shutdown!" - -do_cmd is_running rsc1 $test_node_2 -cts_assert "rsc1 NOT running" - -do_cmd is_running rsc2 $test_node_2 -cts_assert "rsc2 NOT running" - -do_cmd is_running rsc1 $test_node_2 $test_node_1 -cts_assert_false "rsc1 IS running on $test_node_1" - -do_cmd is_running rsc2 $test_node_2 $test_node_2 -cts_assert "rsc2 NOT running on $test_node_2" - -#---- - -stop_node $test_node_2 crmd - -#---- - -do_cmd remote_cmd $CRMD_USER $test_node_2 killall -9 crmd - -echo "test: PASSED" -) -done diff --git a/crm/test/3node-fail.sh.in b/crm/test/3node-fail.sh.in deleted file mode 100644 index c89275632f..0000000000 --- a/crm/test/3node-fail.sh.in +++ /dev/null @@ -1,267 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -required_nodes=3 -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -CRM_ERR_SHUTDOWN=0 -test_nodes=3 - -while [ $iteration -lt $repeats ]; do - iteration=`expr $iteration + 1` - echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" - echo "########### Begining iteration $iteration of $repeats ###########" -( -# make *sure* theres nothing left over from last time -crm-cleanup - -#---- - -do_cmd echo "wait for HA to start on ${test_node_1}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ - -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_1} failed." - -#---- - -do_cmd echo "wait for HA to start on ${test_node_2}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "${test_node_2} ccm(.*) info: Hostname: ${test_node_2}" \ - -s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_2} failed." - -#---- - -do_cmd echo "wait for HA to start on ${test_node_3}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "${test_node_3} ccm(.*) info: Hostname: ${test_node_3}" \ - -s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_3} failed." - -#---- - -do_cmd echo "wait for CRMd to start on ${test_node_1}, ${test_node_2} and ${test_node_3}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & -do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & -do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ - -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" -cts_assert "CRMd startup on ${test_node_1} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ - -s "${test_node_2} crmd(.*) info:(.*)FSA Hostname: ${test_node_2}" -cts_assert "CRMd startup on ${test_node_2} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ - -s "${test_node_3} crmd(.*) info:(.*)FSA Hostname: ${test_node_3}" -cts_assert "CRMd startup on ${test_node_3} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 25000 \ - -s "${test_node_1} crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "${test_node_2} crmd(.*) State transition (.*) \-> S_NOT_DC" \ - -s "${test_node_3} crmd(.*) State transition (.*) \-> S_NOT_DC" -cts_assert "CRMd did not start up into correct modes." - -#---- - -do_cmd echo Various sanity checks - stage 1 -do_cmd is_dc $test_node_1 $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 DC $test_node_1 -cts_assert "S_IDLE not reached on the DC!" - -#---- - -do_cmd echo Create the first constraint and wait for S_IDLE -rsc=rsc1 -uuid1=`uuidgen` -uuid2=`uuidgen` -uuid3=`uuidgen` -uuid4=`uuidgen` -node_xml=" - - - - - - - - " -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint_adv $test_node_1 $node_xml -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -cts_assert Adding constraint1 did not pass - -#---- - -do_cmd echo Create the second constraint and wait for S_IDLE -rsc=rsc2 -uuid1=`uuidgen` -uuid2=`uuidgen` -uuid3=`uuidgen` -uuid4=`uuidgen` -node_xml=" - - - - - - - - " -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint_adv $test_node_1 $node_xml -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -cts_assert Adding constraint2 did not pass - -#---- - -do_cmd echo Create the first resource and wait for S_IDLE after start -args="" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - ignore $args -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" -cts_assert Adding rsc1 did not pass - -#---- - -do_cmd echo Create the second resource and wait for S_IDLE after start -args="" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - ignore $args -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" -cts_assert Adding rsc2 did not pass - -#---- - -do_cmd echo Various sanity checks - stage 2 -do_cmd is_dc $test_node_1 $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 DC $test_node_1 -cts_assert "S_IDLE not reached on the DC!" - -do_cmd is_running rsc1 $test_node_1 $test_node_1 -cts_assert "rsc1 NOT running on $test_node_1" - -do_cmd is_running rsc2 $test_node_1 $test_node_2 -cts_assert "rsc2 NOT running on $test_node_2" - -#---- - -do_cmd echo Kill CRM on $test_node_1 -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $ADMIN_USER $test_node_1 "killall -9 crmd" & -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) State transition (.*) \-> S_ELECTION" -cts_assert "Transition of the DC away from ${test_node_1} failed." -# -s "crmd(.*) State transition (.*) \->" \ - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" -cts_assert "No DC Declared" - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" \ - -s "crmd(.*)State transition(.*) \-> S_IDLE" -cts_assert "DC not idle" - -#---- - -do_cmd echo Various sanity checks - stage 3 -do_cmd is_dc $test_node_2 $test_node_2 -cts_assert "$test_node_2 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 $test_node_2 -cts_assert "S_IDLE not reached on $test_node_2!" - -do_cmd is_running rsc1 $test_node_2 -cts_assert "rsc1 NOT running on $test_node_2" - -do_cmd is_running rsc2 $test_node_2 $test_node_2 -cts_assert "rsc2 NOT running on $test_node_2" - -#---- - -do_cmd echo Kill the CRM, CCM and Heartbeat on $test_node_2 -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $ADMIN_USER $test_node_2 "killall -9 heartbeat crmd lrmd ccm" & -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "crmd(.*) State transition S_NOT_DC \-> S_ELECTION" -cts_assert "Transition of the DC away from ${test_node_2} failed." -# -s "crmd(.*) State transition(.*) \->" \ - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" \ - -s "crmd(.*)State transition(.*) \-> S_IDLE" -cts_assert "No DC Declared" - -#---- - -do_cmd echo Various sanity checks - stage 4 -do_cmd is_dc $test_node_3 $test_node_3 -cts_assert "$test_node_3 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 $test_node_3 -cts_assert "S_IDLE not reached on $test_node_3!" - -do_cmd is_running rsc1 $test_node_3 -cts_assert "rsc1 NOT running on $test_node_3" - -do_cmd is_running rsc2 $test_node_3 $test_node_3 -cts_assert "rsc2 NOT running on $test_node_3" - -#---- - -do_cmd echo Shutdown $test_node_3 -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_3} crmd(.*)State transition (.*) \-> S_PENDING" \ - -s "${test_node_3} crmd(.*)State transition S_PENDING \-> S_STOPPING" \ - -s "${test_node_3} crmd(.*)\[crmd\] stopped" -cts_assert "Shutdown of ${test_node_3} failed." - -#---- - -do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd - -echo "test: PASSED" -) -done diff --git a/crm/test/3node-rapid.sh.in b/crm/test/3node-rapid.sh.in deleted file mode 100644 index ca09a2fe54..0000000000 --- a/crm/test/3node-rapid.sh.in +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -required_nodes=3 -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -CRM_ERR_SHUTDOWN=0 -test_nodes=3 - -while [ $iteration -lt $repeats ]; do - iteration=`expr $iteration + 1` - echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" - echo "########### Begining iteration $iteration of $repeats ###########" -( -# make *sure* theres nothing left over from last time -crm-cleanup - -#---- - -do_cmd echo "wait for HA to start on ${test_node_1}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ - -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_1} failed." - -#---- - -do_cmd echo "wait for HA to start on ${test_node_2}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "${test_node_2} ccm(.*) info: Hostname: ${test_node_2}" \ - -s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_2} failed." - -#---- - -do_cmd echo "wait for HA to start on ${test_node_3}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "${test_node_3} ccm(.*) info: Hostname: ${test_node_3}" \ - -s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_3} failed." - -#---- - -do_cmd echo "wait for CRMd to start on ${test_node_1}, ${test_node_2} and ${test_node_3}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & -do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & -do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ - -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" -cts_assert "CRMd startup on ${test_node_1} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ - -s "${test_node_2} crmd(.*) info:(.*)FSA Hostname: ${test_node_2}" -cts_assert "CRMd startup on ${test_node_2} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ - -s "${test_node_3} crmd(.*) info:(.*)FSA Hostname: ${test_node_3}" -cts_assert "CRMd startup on ${test_node_3} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 25000 \ - -s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ - -s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" \ - -s "${test_node_3} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" -cts_assert "CRMd did not start up into correct modes." - -#---- - -do_cmd echo Various sanity checks - stage 1 -do_cmd is_dc $test_node_1 $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 DC $test_node_1 -cts_assert "S_IDLE not reached on the DC!" - -#---- - -do_cmd echo Create the first constraint and wait for S_IDLE -rsc=rsc1 -uuid1=`uuidgen` -uuid2=`uuidgen` -uuid3=`uuidgen` -uuid4=`uuidgen` -node_xml="' - - - - - - - - '" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint_adv $test_node_1 $node_xml -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" -cts_assert Adding constraint1 did not pass - -#---- - -do_cmd echo Create the second constraint and wait for S_IDLE -rsc=rsc2 -uuid1=`uuidgen` -uuid2=`uuidgen` -uuid3=`uuidgen` -uuid4=`uuidgen` -node_xml="' - - - - - - - - '" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint_adv $test_node_1 $node_xml -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" -cts_assert Adding constraint2 did not pass - -#---- - -do_cmd echo Create the first resource and wait for S_IDLE after start -args="" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ - -s "crmd(.*) Performing op start(.*) on rsc1" -cts_assert Adding rsc1 did not pass - -#---- - -do_cmd echo Create the second resource and wait for S_IDLE after start -args="" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \ - -s "crmd(.*) Performing op start(.*) on rsc2" -cts_assert Adding rsc2 did not pass - -#---- - -do_cmd echo Various sanity checks - stage 2 -do_cmd is_dc $test_node_1 $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 DC $test_node_1 -cts_assert "S_IDLE not reached on the DC!" - -do_cmd is_running rsc1 $test_node_1 $test_node_1 -cts_assert "rsc1 NOT running on $test_node_1" - -do_cmd is_running rsc2 $test_node_1 $test_node_2 -cts_assert "rsc2 NOT running on $test_node_2" - -#---- - -do_cmd echo Shutdown $test_node_1 -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 & -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \ - -s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" -cts_assert "Transition of the DC away from ${test_node_1} failed." -# -s "crmd(.*) State transition (.*) \->" \ - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "${test_node_1} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \ - -s "${test_node_1} crmd(.*)\[crmd\] stopped" \ - -e "${test_node_1} crmd(.*)Timer I_TERMINATE just popped" -cts_assert "Shutdown of ${test_node_1} failed" - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" -cts_assert "No DC Declared" - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \ - -s "crmd(.*)State transition(.*) \-> \"S_IDLE\"" -cts_assert "DC not idle" - -#---- - -do_cmd echo Various sanity checks - stage 3 -do_cmd is_dc $test_node_2 $test_node_2 -cts_assert "$test_node_2 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 $test_node_2 -cts_assert "S_IDLE not reached on $test_node_2!" - -do_cmd is_running rsc1 $test_node_2 $test_node_2 -cts_assert "rsc1 NOT running on $test_node_2" - -do_cmd is_running rsc2 $test_node_2 $test_node_2 -cts_assert "rsc2 NOT running on $test_node_2" - -#---- - -do_cmd echo Shutdown $test_node_2 -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 & -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \ - -s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" -cts_assert "Transition of the DC away from ${test_node_2} failed." -# -s "crmd(.*) State transition(.*) \->" \ - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "${test_node_2} crmd(.*)State transition" \ - -s "${test_node_2} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \ - -s "${test_node_2} crmd(.*)\[crmd\] stopped" \ - -e "${test_node_2} crmd(.*)Timer I_TERMINATE just popped" -cts_assert "Shutdown of ${test_node_2} failed" - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 11000 \ - -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \ - -s "crmd(.*)State transition(.*) \-> \"S_IDLE\"" -cts_assert "No DC Declared" - -#---- - -do_cmd echo Various sanity checks - stage 4 -do_cmd is_dc $test_node_3 $test_node_3 -cts_assert "$test_node_3 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 $test_node_3 -cts_assert "S_IDLE not reached on $test_node_3!" - -do_cmd is_running rsc1 $test_node_3 $test_node_3 -cts_assert "rsc1 NOT running on $test_node_3" - -do_cmd is_running rsc2 $test_node_3 $test_node_3 -cts_assert "rsc2 NOT running on $test_node_3" - -#---- - -do_cmd echo Shutdown $test_node_3 -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \ - -s "${test_node_3} crmd(.*)State transition (.*) \-> \"S_PENDING\"" \ - -s "${test_node_3} crmd(.*)State transition \"S_PENDING\" \-> \"S_STOPPING\"" \ - -s "${test_node_3} crmd(.*)\[crmd\] stopped" -cts_assert "Shutdown of ${test_node_3} failed." - -#---- - -do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd - -echo "test: PASSED" -) -done diff --git a/crm/test/3node.sh.in b/crm/test/3node.sh.in deleted file mode 100644 index b8e5b6e4e3..0000000000 --- a/crm/test/3node.sh.in +++ /dev/null @@ -1,275 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -required_nodes=3 -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -CRM_ERR_SHUTDOWN=0 -test_nodes=3 - -while [ $iteration -lt $repeats ]; do - iteration=`expr $iteration + 1` - echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007" - echo "########### Begining iteration $iteration of $repeats ###########" -( -# make *sure* theres nothing left over from last time -crm-cleanup - -#---- - -do_cmd echo "wait for HA to start on ${test_node_1}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \ - -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_1} failed." - -#---- - -do_cmd echo "wait for HA to start on ${test_node_2}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "${test_node_2} ccm(.*) info: Hostname: ${test_node_2}" \ - -s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_2} failed." - -#---- - -do_cmd echo "wait for HA to start on ${test_node_3}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \ - -s "${test_node_3} ccm(.*) info: Hostname: ${test_node_3}" \ - -s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_3} failed." - -#---- - -do_cmd echo "wait for CRMd to start on ${test_node_1}, ${test_node_2} and ${test_node_3}" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & -do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & -do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" & - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ - -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}" -cts_assert "CRMd startup on ${test_node_1} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ - -s "${test_node_2} crmd(.*) info:(.*)FSA Hostname: ${test_node_2}" -cts_assert "CRMd startup on ${test_node_2} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 3000 \ - -s "${test_node_3} crmd(.*) info:(.*)FSA Hostname: ${test_node_3}" -cts_assert "CRMd startup on ${test_node_3} failed." - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 25000 \ - -s "${test_node_1} crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "${test_node_2} crmd(.*) State transition (.*) \-> S_NOT_DC" \ - -s "${test_node_3} crmd(.*) State transition (.*) \-> S_NOT_DC" -cts_assert "CRMd did not start up into correct modes." - -#---- - -do_cmd echo Various sanity checks - stage 1 -do_cmd is_dc $test_node_1 $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 DC $test_node_1 -cts_assert "S_IDLE not reached on the DC!" - -#---- - -do_cmd echo Create the first constraint and wait for S_IDLE -rsc=rsc1 -uuid1=`uuidgen` -uuid2=`uuidgen` -uuid3=`uuidgen` -uuid4=`uuidgen` -node_xml=" - - - - - - - - " -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint_adv $test_node_1 $node_xml -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -cts_assert Adding constraint1 did not pass - -#---- - -do_cmd echo Create the second constraint and wait for S_IDLE -rsc=rsc2 -uuid1=`uuidgen` -uuid2=`uuidgen` -uuid3=`uuidgen` -uuid4=`uuidgen` -node_xml="' - - - - - - - - " -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_constraint_adv $test_node_1 $node_xml -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" -cts_assert Adding constraint2 did not pass - -#---- - -do_cmd echo Create the first resource and wait for S_IDLE after start -args="" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc1" -cts_assert Adding rsc1 did not pass - -#---- - -do_cmd echo Create the second resource and wait for S_IDLE after start -args="" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \ - -s "crmd(.*) State transition (.*) \-> S_IDLE" \ - -s "crmd(.*) Performing op start(.*) on rsc2" -cts_assert Adding rsc2 did not pass - -#---- - -do_cmd echo Various sanity checks - stage 2 -do_cmd is_dc $test_node_1 $test_node_1 -cts_assert "$test_node_1 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 DC $test_node_1 -cts_assert "S_IDLE not reached on the DC!" - -do_cmd is_running rsc1 $test_node_1 $test_node_1 -cts_assert "rsc1 NOT running on $test_node_1" - -do_cmd is_running rsc2 $test_node_1 $test_node_2 -cts_assert "rsc2 NOT running on $test_node_2" - -#---- - -do_cmd echo Shutdown $test_node_1 -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 & -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) State transition (.*) \-> S_ELECTION" \ - -s "${test_node_1} crmd(.*) State transition (.*) \-> S_NOT_DC" -cts_assert "Transition of the DC away from ${test_node_1} failed." -# -s "crmd(.*) State transition (.*) \->" \ - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "${test_node_1} crmd(.*)State transition S_NOT_DC \-> S_STOPPING" \ - -s "${test_node_1} crmd(.*)\[crmd\] stopped" \ - -e "${test_node_1} crmd(.*)Timer I_TERMINATE just popped" -cts_assert "Shutdown of ${test_node_1} failed" - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" -cts_assert "No DC Declared" - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" \ - -s "crmd(.*)State transition(.*) \-> S_IDLE" -cts_assert "DC not idle" - -#---- - -do_cmd echo Various sanity checks - stage 3 -do_cmd is_dc $test_node_2 $test_node_2 -cts_assert "$test_node_2 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 $test_node_2 -cts_assert "S_IDLE not reached on $test_node_2!" - -do_cmd is_running rsc1 $test_node_2 $test_node_2 -cts_assert "rsc1 NOT running on $test_node_2" - -do_cmd is_running rsc2 $test_node_2 $test_node_2 -cts_assert "rsc2 NOT running on $test_node_2" - -#---- - -do_cmd echo Shutdown $test_node_2 -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 & -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "crmd(.*) State transition (.*) \-> S_ELECTION" \ - -s "${test_node_2} crmd(.*) State transition (.*) \-> S_NOT_DC" -cts_assert "Transition of the DC away from ${test_node_2} failed." -# -s "crmd(.*) State transition(.*) \->" \ - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \ - -s "${test_node_2} crmd(.*)State transition" \ - -s "${test_node_2} crmd(.*)State transition S_NOT_DC \-> S_STOPPING" \ - -s "${test_node_2} crmd(.*)\[crmd\] stopped" \ - -e "${test_node_2} crmd(.*)Timer I_TERMINATE just popped" -cts_assert "Shutdown of ${test_node_2} failed" - -do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 11000 \ - -s "crmd(.*)State transition(.*) \-> S_INTEGRATION" \ - -s "crmd(.*)State transition(.*) \-> S_IDLE" -cts_assert "No DC Declared" - -#---- - -do_cmd echo Various sanity checks - stage 4 -do_cmd is_dc $test_node_3 $test_node_3 -cts_assert "$test_node_3 is supposed to be the DC" - -do_cmd wait_for_state S_IDLE 3 $test_node_3 -cts_assert "S_IDLE not reached on $test_node_3!" - -do_cmd is_running rsc1 $test_node_3 $test_node_3 -cts_assert "rsc1 NOT running on $test_node_3" - -do_cmd is_running rsc2 $test_node_3 $test_node_3 -cts_assert "rsc2 NOT running on $test_node_3" - -#---- - -stop_node $test_node_3 crmd - -#---- - -do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd - -echo "test: PASSED" -) -done diff --git a/crm/test/Makefile.am b/crm/test/Makefile.am deleted file mode 100644 index 06fd4dd943..0000000000 --- a/crm/test/Makefile.am +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -MAINTAINERCLEANFILES = Makefile.in - -testdir = $(libdir)/heartbeat/crmtest -test_SCRIPTS = helper.sh testutils.pl stop.sh cib_suite.sh \ - 1node.sh 2node.sh 2node-fail.sh 2node-empty.sh \ - 2node-fence.sh 3node.sh 3node-rapid.sh 3node-fail.sh \ - extract.logs -EXTRA_DIST = $(test_SCRIPTS) - -install-data-local: - -chown $(HA_CCMUSER) $(DESTDIR)/$(testdir) - -chgrp $(HA_APIGROUP) $(DESTDIR)/$(testdir) - -chmod g+w $(DESTDIR)/$(testdir) diff --git a/crm/test/cib_suite.sh.in b/crm/test/cib_suite.sh.in deleted file mode 100644 index eadc3ae970..0000000000 --- a/crm/test/cib_suite.sh.in +++ /dev/null @@ -1,599 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -erase_sync_1=" - - - - - - - - - - - -" - -erase_sync_2=" - - - - - - - - - - - -" - -erase_sync_3=" - - - - - - - - - - - -" - -create_1=" - - - - - - - - - - - - - - -" - -create_2=" - - - - - - - - - - - - - - -" - -create_3=" - - - - - - - - - - - - - - -" - -create_local_1=" - - - - - - - - - - - - - - - -" - -create_local_2=" - - - - - - - - - - - - - - -" - -create_local_3=" - - - - - - - - - - - - - - -" - -bump_local_1=" - - - - - - - - - - - - - - - -" - -bump_local_2=" - - - - - - - - - - - - - - -" - -bump_local_3=" - - - - - - - - - - - - - - -" - -bump_remote_1=" - - - - - - - - - - - - - - - -" - -bump_remote_2=" - - - - - - - - - - - - - - -" - -bump_remote_3=" - - - - - - - - - - - - - - -" - -bump_1=" - - - - - - - - - - - - - - - -" - -bump_2=" - - - - - - - - - - - - - - -" - -bump_3=" - - - - - - - - - - - - - - -" - - -sync_from_1=" - - - - - - - - - - - - - - -" - -sync_from_2=" - - - - - - - - - - - - - - -" - -sync_from_3=" - - - - - - - - - - - - - - -" - -# make *sure* theres nothing left over from last time -crm-cleanup - -function compare_cibs() -{ - cib_cmp_base=$1; shift - cib_cmp_host=$1; shift - cib_cmp_opts=$* - cib_cmp_output=${cib_cmp_base}.out - cib_cmp_expected=${cib_cmp_base}.exp - cib_cmp_failed=$test_dump_dir/test.txt - cib_cmp_diff_opts="--ignore-all-space -U 1 -u" - - remote_cmd $CRMD_USER $cib_cmp_host "$HALIB_DIR/cibadmin -Q ${cib_cmp_opts}" > $cib_cmp_output - cts_assert "compare_cibs(): Could not perform query: opts=$cib_cmp_opts" - - gres ' timestamp=\"[0123456789]*\"' "" ${cib_cmp_output} - gres ' id=\"[0123456789abcdef-]*\"' "" ${cib_cmp_output} - - # this is the value stored in a variable named $cib_cmp_base - echo -e "${!cib_cmp_base}" > $cib_cmp_expected - -# if [ "$create_mode" = "true" -a ! -f $cib_cmp_expected ]; then -# cp "$cib_cmp_output" "$cib_cmp_expected" -# fi - - if [ -f $cib_cmp_expected ]; then - diff $cib_cmp_diff_opts -q $cib_cmp_expected $cib_cmp_output >/dev/null - rc=$? - fi - -# if [ "$create_mode" = "true" ]; then -# echo "Test $cib_cmp_base... Created expected output" -# el - if [ ! -f $cib_cmp_expected ]; then - echo "==== Raw results for CIB test ($cib_cmp_base) ====" >> $cib_cmp_failed - cat $cib_cmp_output 2>/dev/null >> $cib_cmp_failed - cat $cib_cmp_output - - elif [ "$rc" = 0 ]; then - do_cmd echo "Test $cib_cmp_base... Passed"; - elif [ "$rc" = 1 ]; then - do_cmd echo "Test $cib_cmp_base... * Failed"; - diff $cib_cmp_diff_opts $cib_cmp_expected $cib_cmp_output 2>/dev/null >> $cib_cmp_failed - diff $cib_cmp_diff_opts $cib_cmp_expected $cib_cmp_output - else - do_cmd echo "Test $cib_cmp_base... Error (diff: $rc)"; - do_cmd echo "==== Raw results for test ($cib_cmp_base) ====" >> $cib_cmp_failed - cat $cib_cmp_output 2>/dev/null >> $cib_cmp_failed - cat $cib_cmp_output - fi - - rm $cib_cmp_output .gres.$cib_cmp_output $cib_cmp_expected - - do_cts_assert $rc 0 "diff for $cib_cmp_base found differences" -} - - -#---- - -do_cmd echo "wait for HA to start" -crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & -do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & -do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & -do_cmd ${testdir}/testutils.pl -l ${logfile} -p $crm_log_pos --search -a -m 3500 \ - -s "${test_node_1} heartbeat(.*) info: Starting child client(.*)cib" \ - -e "${test_node_1} heartbeat(.*)Client(.*)/cib \"respawning too fast" \ - -s "${test_node_2} heartbeat(.*) info: Starting child client(.*)cib" \ - -e "${test_node_2} heartbeat(.*)Client(.*)/cib \"respawning too fast" \ - -s "${test_node_3} heartbeat(.*) info: Starting child client(.*)cib" \ - -e "${test_node_3} heartbeat(.*)Client(.*)/cib \"respawning too fast" -cts_assert "Startup of Heartbeat on ${test_node_1}, ${test_node_2} and ${test_node_3} failed." - -#---- - -do_cmd echo "is master instance?" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -M" -do_cts_assert $? 35 "CIB service on $test_node_1 should not be in master mode by default" - -#---- - -do_cmd echo "local query" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -l" -cts_assert "Could not query local CIB service on $test_node_1" - -#---- - -do_cmd echo "queries with hostname" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -h $test_node_2" -cts_assert "Could not query CIB service on $test_node_2 from $test_node_1" - -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -l -h $test_node_1" -cts_assert "Could not query CIB service on $test_node_1 from $test_node_1" - -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -Q -h $test_node_1" -cts_assert "Could not query CIB service on $test_node_1 from $test_node_1 without -l flag" - -#---- - -do_cmd echo "make sure all instances are slaves" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -r" -cts_assert "Could not set CIB services to slaves" - -#---- - -do_cmd echo "make local the master instance" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -w" -cts_assert "Could not set CIB service on $test_node_1 to master" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -M" -cts_assert "Could not set CIB service on $test_node_1 to master" - -#---- - -do_cmd echo "erase CIB contents and sync" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -E" -cts_assert "Could not erase CIB contents" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -S" -cts_assert "Could not sync CIB to all nodes" - -#---- - -do_cmd echo "verify the CIB contents" -compare_cibs erase_sync_1 $test_node_1 -l -compare_cibs erase_sync_2 $test_node_1 -h $test_node_2 -compare_cibs erase_sync_3 $test_node_1 -l -h $test_node_3 - -#---- - -do_cmd echo "make some nodes in the CIB" -do_cmd make_node $test_node_1 $test_node_1 "member" -cts_assert "Could not create $test_node_1" -do_cmd make_node $test_node_1 $test_node_2 "member" -cts_assert "Could not create $test_node_2" - -#---- - -do_cmd echo "verify the CIB contents" -compare_cibs create_1 $test_node_1 -l -compare_cibs create_2 $test_node_2 -l -compare_cibs create_3 $test_node_3 -l - -#---- - -do_cmd make_node_local $test_node_1 $test_node_3 "member" -cts_assert "Could not create $test_node_3 locally" - -#---- - -do_cmd echo "verify the CIB contents on all nodes" -compare_cibs create_local_1 $test_node_1 -l -compare_cibs create_local_2 $test_node_2 -l -compare_cibs create_local_3 $test_node_3 -l - -#---- - -do_cmd echo "make all slaves" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -r" -cts_assert "Could not set CIB services to slaves" - -do_cmd echo "make node2 master" -do_cmd remote_cmd $CRMD_USER $test_node_2 "$HALIB_DIR/cibadmin -w" -cts_assert "Could not set CIB service on $test_node_1 to master" - -#---- - -do_cmd echo "bump master twice (so we can use it in a sync later)" - -#---- - -do_cmd remote_cmd $CRMD_USER $test_node_2 "$HALIB_DIR/cibadmin -B" -cts_assert "Could not perform bump on $test_node_2" - -do_cmd echo "verify the CIB contents on all nodes" -compare_cibs bump_1 $test_node_1 -l -compare_cibs bump_2 $test_node_2 -l -compare_cibs bump_3 $test_node_3 -l - -#---- - -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -B" -cts_assert "Could not perform bump on $test_node_2 via $test_node_1" - -do_cmd echo "verify the CIB contents on all nodes" -compare_cibs bump_remote_1 $test_node_1 -l -compare_cibs bump_remote_2 $test_node_2 -l -compare_cibs bump_remote_3 $test_node_3 -l - -#---- - -do_cmd echo "make all slaves" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -r" -cts_assert "Could not set CIB services to slaves" - -do_cmd echo "make node1 master" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -w" -cts_assert "Could not set CIB service on $test_node_1 to master" - -#---- - -do_cmd echo "sync from node2" -do_cmd remote_cmd $CRMD_USER $test_node_1 "$HALIB_DIR/cibadmin -S -h $test_node_2" -cts_assert "Could not sync from $test_node_2" - -#---- - -do_cmd echo "verify the CIB contents on all nodes" -compare_cibs sync_from_1 $test_node_1 -l -compare_cibs sync_from_2 $test_node_2 -l -compare_cibs sync_from_3 $test_node_3 -l - -#---- - -do_cmd remote_cmd $CRMD_USER $test_node_2 "$HALIB_DIR/cibadmin -B -l" -cts_assert "Could not perform local bump on $test_node_2" - -do_cmd echo "verify the CIB contents on all nodes" -compare_cibs bump_local_1 $test_node_1 -l -compare_cibs bump_local_2 $test_node_2 -l -compare_cibs bump_local_3 $test_node_3 -l - - -echo "test: PASSED" - diff --git a/crm/test/extract.logs b/crm/test/extract.logs deleted file mode 100755 index 0f0d52ae06..0000000000 --- a/crm/test/extract.logs +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# License: GNU General Public License (GPL) -# - -dirname=$1; shift; -prefix=$1; shift; -filename=$1; shift; - -systems="crmd cib tengine pengine cib cibmon" -tags="" -strip_tags="live.dot cib.updates" - -for subsys in $systems; do - echo Extracting ${subsys} logs... - grep "${subsys}:" "${filename}" > "${dirname}/${subsys}.${prefix}.log" -done - -for tag in $tags; do - echo Extracting ${tag} logs... - grep "${tag}" "${filename}" > "${dirname}/${tag}.${prefix}" -done - -#for strip_tag in $strip_tags; do -strip_tag="live.dot" - echo Extracting ${strip_tag}... - grep "${strip_tag}:" "${filename}" | awk '{ FS="live.dot:"; print $2 }' > "${dirname}/${strip_tag}.${prefix}" -strip_tag="cib.updates" - echo Extracting ${strip_tag}... - grep "${strip_tag}:" "${filename}" | awk '{ FS="cib.updates:"; print $2 }' > "${dirname}/${strip_tag}.${prefix}" -#done - diff --git a/crm/test/helper.sh.in b/crm/test/helper.sh.in deleted file mode 100644 index bd40bb94eb..0000000000 --- a/crm/test/helper.sh.in +++ /dev/null @@ -1,726 +0,0 @@ -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -[[ $0 != *helper.sh ]] || { - echo >&2 "helper.sh expects to be sourced!" - exit 1 -} - -# no more exit below here, except in err(), use return instead; -# gives more flexibility ... - -: ${required_nodes=0} - -test_type=`basename $0` -test_nodes=1 -dump_dir=~/ha-tests - -HALIB_DIR=@libdir@/heartbeat -HAVAR_DIR=@HA_VARLIBDIR@/heartbeat -HAINIT_DIR=@INITDIR@ - -INIT_USER=root -CRMD_USER=hacluster -ADMIN_USER=root - -test_node_1="" -test_node_2="" -test_node_3="" - -#CRM_OPTS="-VVVVVV" - -CRM_ERR_SHUTDOWN=0 - -OUTPUT_STDERR=0 -OUTPUT_STDOUT=0 -OUTPUT_CMDS=0 -OUTPUT_ECHO=1 - -repeats=1 -iteration=0 - -done=0 - -ip_rsc_1=192.168.9.161 -ip_rsc_2=192.168.9.162 - -logfile=/var/log/messages -local_version=0 -bsc_mode=0 -crm_only=0 - -STAT_OPTS="-c %s" -KILL_OPTS="-q9" -PS_OPTS="axf" -arch=`uname -s` -if [ "z$arch" = "zDarwin" ]; then - STAT_OPTS="-f %z" - KILL_OPTS="-9" - PS_OPTS="-ax" -fi - -if [ ! -z "$CRM_LOCAL_OPTS" ]; then - echo "Checking $CRM_LOCAL_OPTS for local options" - if [ -f "$CRM_LOCAL_OPTS" ]; then - . "$CRM_LOCAL_OPTS" - else - echo "File $CRM_LOCAL_OPTS does not exist" - fi -else - echo "Hint: Define CRM_LOCAL_OPTS=/config/file/name.path to automatically set options for your environment" -fi - -while test "$done" = "0"; do - case "$1" in - --debug | -x) set -x; shift;; - --silent | -s) OUTPUT_ECHO=0; shift;; - --stdout | -o) OUTPUT_STDOUT=1; shift;; - --stderr | -e) OUTPUT_STDERR=1; shift;; - --verbose | -v) OUTPUT_CMDS=1; shift;; - - --crm-only | -C) crm_only=1; shift;; - --local | -L) local_version=1; shift;; - --bsc | -B) local_version=1; bsc_mode=1; shift;; - - --logfile | -l) logfile=${2?no logfile??} || return 1; shift 2;; - --dumpdir | -d) dump_dir=${2?no dumpdir??} || return 1; shift 2;; - - --repeat | -r) repeats=${2?no max repetitions??} || return 1; shift 2;; - - --init | -i) INIT_USER=${2?no init user??} || return 1; shift 2;; - --admin | -a) ADMIN_USER=${2?no admin??} || return 1; shift 2;; - --crmd | -c) CRMD_USER=${2?no crmd user??} || return 1; shift 2;; - - --node1 | -1) test_node_1=`echo ${2?no node1??} | tr 'A-Z' 'a-z'` || return 1; shift 2;; - --node2 | -2) test_node_2=`echo ${2?no node2??} | tr 'A-Z' 'a-z'` || return 1; shift 2;; - --node3 | -3) test_node_3=`echo ${2?no node3??} | tr 'A-Z' 'a-z'` || return 1; shift 2;; - - --ip1) ip_rsc_1=${2?no ip1??} || return 1; shift 2;; - --ip2) ip_rsc_2=${2?no ip2??} || return 1; shift 2;; - - ""|--) done=1;; - *) echo "unknown option: $1"; return 1;; - esac -done - -function test_init() { - export date_str=`date "+%Y-%b-%d_%H:%M"` - export test_dump_dir="$dump_dir/$test_type/$date_str" - mkdir -p "$test_dump_dir" - echo Test dir: $test_dump_dir - crm_test_log_start=$(stat -L ${STAT_OPTS} $logfile) -} - -function remote_cmd() { - user=$1 - host=$2 - - shift - shift - - if [ $local_version = "1" ]; then - su - ${user} -c "$*" - else - `which ssh` ${user}@${host} "$*" - fi - return $?; -} - -function start_node() { - ha_node="$1" - srch_node="$ha_node " - if [ $local_version = "1" ]; then - srch_node="" - fi - do_cmd echo "wait for HA to start on $ha_node" - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd remote_cmd $INIT_USER $ha_node $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" & - if [ $local_version = "1" ]; then - do_cmd ${testdir}/testutils.pl -l ${logfile} -p $crm_log_pos --search -a -m 100 \ - -s "${srch_node}cib.*Hostname: ${ha_node}" - cts_assert "Startup of Heartbeat on ${ha_node} failed." - else - do_cmd ${testdir}/testutils.pl -l ${logfile} -p $crm_log_pos --search -a -m 100 \ - -s "${srch_node}ccm(.*) info: Hostname: ${ha_node}" \ - -s "${srch_node}cib(.*) Hostname: ${ha_node}" \ - -s "${srch_node}heartbeat(.*) info: Starting(.*)lrmd" \ - -e "${srch_node}heartbeat(.*)Client(.*) respawning too fast" - cts_assert "Startup of Heartbeat on ${ha_node} failed." - fi -#---- - - do_cmd echo "Check if CRMd was started too" - do_cmd remote_cmd $INIT_USER $ha_node "killall -INT crmd" - if [ $? = 1 ]; then - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - do_cmd echo "wait for CRMd to start on $ha_node" - do_cmd remote_cmd $CRMD_USER $ha_node $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" & - fi - - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 20 \ - -s "${srch_node}crmd(.*) info:(.*)Hostname: ${ha_node}" \ - -s "${srch_node}crmd(.*)State transition S_STARTING \-> S_PENDING" - cts_assert "CRMd startup on ${ha_node} failed." -} - -function stop_node() -{ - a_node=$1; shift; - down_cmd=$1; shift; - do_cmd echo Shutdown $a_node - crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) - - srch_node="$a_node " - if [ $local_version = "1" ]; then - srch_node="" - fi - - if [ $crm_only = 1 ]; then - do_cmd echo "Shutting down CRMd" - do_cmd remote_cmd $CRMD_USER $a_node @sbindir@/crmadmin -K $a_node - else - do_cmd echo "Shutting down Heartbeat" - do_cmd remote_cmd $INIT_USER $a_node $HALIB_DIR/heartbeat -k - fi - do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -t 120 \ - -s "${srch_node} crmd(.*)State transition" \ - -s "${srch_node} crmd(.*)State transition (.*) \-> S_STOPPING" \ - -e "${srch_node} crmd(.*)Timer I_TERMINATE just popped" - cts_assert "Shutdown of ${a_node} failed." - -# This pattern does not always get printed... most annoying -# -s "${srch_node} crmd(.*)\[crmd\] stopped" \ - - sleep 5 - do_cmd echo "Check if CRMd _really_ stopped" - do_cmd remote_cmd $INIT_USER $ha_node "killall -INT crmd" - cts_assert_false "CRMd is still running on ${a_node}." - - -} - -function is_state() { - ret=1 - host=$1 - state=$2 - - output=`remote_cmd $ADMIN_USER $host @sbindir@/crmadmin -S $host | grep $state` - - if [ ! -z "$output" ]; then - echo "$host is in $state" - return 0 - fi - echo "$host is NOT in $state" - return $ret -} - -function is_running() { - rsc=$1 - askhost=$2 - host=$3 - - if [ -z $host ]; then - output=`remote_cmd $ADMIN_USER $askhost @sbindir@/crm_resource -W -r $rsc | grep -v @sbindir@/crm_resource` - output=`echo $output | grep -v NOT` - else - output=`remote_cmd $ADMIN_USER $askhost @sbindir@/crm_resource -W -r $rsc | grep -v @sbindir@/crm_resource` - output=`echo $output | grep $host` - fi - - if [ -z "$output" ]; then - if [ -z $host ]; then - echo "$rsc is NOT running" - else - echo "$rsc is NOT running on host $host" - fi - return 1 - fi - - if [ -z $host ]; then - echo "$rsc is running" - else - echo "$rsc is running on host $host" - fi - - return 0 -} - -function is_dc() { - host=$1 - output=`remote_cmd $ADMIN_USER $host @sbindir@/crmadmin -S $host | grep -v S_PENDING | grep -v S_NOT_DC | grep -v S_ELECTION | grep -v S_RECOVERY` - - if [ ! -z "$output" ]; then - echo "$host is DC" - return 0 - fi - - echo "$host is NOT DC" - remote_cmd $ADMIN_USER $host @sbindir@/crmadmin -S $host - - return 1 -} - -function wait_for_state() { - state=$1 - max=$2 - host=$3 - target=$4 - - sleep 1 - - if [ "$host" = "DC" ]; then - host=`remote_cmd $ADMIN_USER $target @sbindir@/crmadmin -D` - host=${host#*: } - fi - - count=1 - until is_state $host $state; do - if [ $count -gt $max ]; then - echo "Attempt to reach $state on $host failed" - return 1 - fi - count=`expr $count + 1` - sleep 1 - done - echo "$state reached on $host" - return 0 -} - - -function make_node() -{ - host=$1 - uname=$2 - type=$3 - uuid=`uuidgen` - - if [ -z $type ]; then - type="member" - fi - - node_xml="''"; - - remote_cmd $ADMIN_USER $host "@sbindir@/cibadmin -C -o nodes -X $node_xml" - -} - -function make_node_local() -{ - host=$1 - uname=$2 - type=$3 - uuid=`uuidgen` - - if [ -z $type ]; then - type="member" - fi - - node_xml="''"; - - remote_cmd $ADMIN_USER $host "@sbindir@/cibadmin -l -C -o nodes -X $node_xml" -} - -function make_incarnation() -{ - host=$1; shift - id=$1; shift - class=$1; shift - type=$1; shift - max=$1; shift - priority=$1; shift - version=$1; shift - stopfail=$1; shift - - args=$* - - uuid=`uuidgen` - node_xml="'" - - if [ ! -z $priority -a "x$priority" != "x-" ]; then - node_xml="$node_xml priority=\\\"${priority}\\\"" - fi - - node_xml="$node_xml &1 > /dev/null -} - -function make_resource() -{ - host=$1 - id=$2 - class=$3 - type=$4 - priority=$5 - version=$6 - stopfail=$7 - - shift 7 - - args=$* - - uuid=`uuidgen` - - node_xml=" ${args} " - fi - node_xml="$node_xml "; - - do_cmd remote_cmd ${ADMIN_USER} ${host} "@sbindir@/cibadmin -VVVV -C -o resources -X \"$node_xml\"" -# 2>&1 > /dev/null -} - -function make_constraint_adv() -{ - host=$1 - shift - node_xml=$* - - node_xml=`echo $node_xml | tr '\n' ' '` - - do_cmd remote_cmd ${ADMIN_USER} ${host} "@sbindir@/cibadmin -C -o constraints -VVVV -X \"$node_xml\"" -} - -function make_constraint() -{ - host=$1 - rsc=$2 - result=$3 - uuid1=`uuidgen` - - node_xml="" - - make_constraint_adv $host "$node_xml" -} - -function cts_assert() { - do_cts_assert $? 0 "Assert failed - " $* -} - -function cts_assert_false() { - do_cts_assert $? 1 "AssertFalse failed - " $* -} - -function do_cts_assert() { - result=$1 - expected=$2 - shift - shift - do_err=0 - - if [ $result -ne $expected ]; then - do_cmd echo "Expected rc ($expected) != $result" - do_err=1 - else - # this is a hack for the fact that stat dumps core with MALLOC_CHECK_=2 - if [ $local_version = 1 ]; then - file $HADIR/core* | grep -e "from 'stat'" | rm -f `awk '{ print $1 }' | tr ':' ' '` - fi - for a_host in $node_list; do - remote_cmd ${ADMIN_USER} ${a_host} "ls -al ${HAVAR_DIR}/cores/*/core*" 2>/dev/null - if [ $? -eq 0 ]; then - do_cmd echo "Core file(s) found on $a_host" - do_cmd remote_cmd ${ADMIN_USER} ${a_host} "file ${HAVAR_DIR}/cores/*/core*" - do_err=1 - fi - done - fi - - if [ $do_err = 1 ]; then - err $* - fi -} - -function err() { - do_cmd echo Dumping test nodes to $dump_dir - node_num=1 - for a_host in $node_list; do - dump_node $node_num $a_host - node_num=`expr $node_num + 1` - done - - crm_test_log_end=$(stat -L ${STAT_OPTS} $logfile) - do_cmd echo "ERROR: $*" - do_cmd echo "test $test_type: FAILED" - if [ $bsc_mode = 0 ]; then - echo "see ${test_dump_dir}/test.txt for details..." - echo "ERROR: $*" > $test_dump_dir/error.txt - $HALIB_DIR/crmtest/testutils.pl --dump -p $crm_test_log_start -ep $crm_test_log_end -l $logfile > $test_dump_dir/cluster.log - fi - exit 1 -} - -function dump_node() { - - a_node_index=$1 - a_node=$2 - - dump_file="/tmp/${a_node_index}-${a_node}.tgz" - - remote_cmd $ADMIN_USER ${a_node} "mkdir /tmp/crm" - remote_cmd $ADMIN_USER ${a_node} "stat -L ${STAT_OPTS} $logfile > /tmp/crm/.logend" - remote_cmd ${ADMIN_USER} ${a_node} "ps ${PS_OPTS} > /tmp/crm/processes" - remote_cmd ${ADMIN_USER} ${a_node} "$HALIB_DIR/crmtest/testutils.pl --dump -pf /tmp/crm/.logstart -l $logfile > /tmp/crm/cluster-test.log" -# `which ssh` ${ADMIN_USER}@${a_node} "$HALIB_DIR/crmtest/testutils.pl --dump -pf /tmp/crm/.logstart -epf /tmp/crm/.logend -l $logfile > /tmp/crm/cluster-test.log" - - remote_cmd ${ADMIN_USER} ${a_node} "ln -s /tmp/crm /tmp/crm-${a_node_index}" - remote_cmd ${ADMIN_USER} ${a_node} "ln -s ${HAVAR_DIR}/cores /tmp/crm" - remote_cmd ${ADMIN_USER} ${a_node} "cp ${HAVAR_DIR}/crm/cib.xml /tmp/crm" - remote_cmd ${ADMIN_USER} ${a_node} "tar -zhcvf ${dump_file} /tmp/crm-${a_node_index}/ " - remote_cmd ${ADMIN_USER} ${a_node} "rm -f /tmp/crm-${a_node_index}" - - cd $test_dump_dir - if [ $local_version = 1 ]; then - cp -r ${dump_file} . - else - scp -r ${ADMIN_USER} ${a_node}:${dump_file} . - fi - tar zxvf ${a_node_index}-${a_node}.tgz - cd - - remote_cmd ${ADMIN_USER} ${a_node} "rm ${dump_file}" - - if [ $bsc_mode = 0 ]; then - $HALIB_DIR/crmtest/extract.logs ${test_dump_dir} node-${a_node_index} ${test_dump_dir}/tmp/crm-${a_node_index}/cluster-test.log - fi -} - -function do_cmd() { - - cmdline=$* - old_OUTPUT_STDOUT=$OUTPUT_STDOUT - cmd=$1 - - test_log_file=$test_dump_dir/test.txt - if [ $bsc_mode = 1 ]; then - test_log_file=/tmp/linux-ha.testlog - fi - - if [ $OUTPUT_STDERR -eq 1 -o $OUTPUT_STDOUT -eq 1 -o $OUTPUT_CMDS -eq 1 ]; then - - if [ $cmd = "remote_cmd" ]; then - user=$2 - host=$3 - shift 3 - echo `date` ": Running '$*' as ${user}@${host}" | tee -a $test_log_file; - - elif [ $cmd = "wait_for_state" ]; then - host=$4 - if [ -z $host ]; then - host="the DC" - fi - echo `date` ": Waiting for state $2 on $host (max $3 attempts)" | tee -a $test_log_file; - - elif [ $cmd = "${HALIB_DIR}/crmtest/testutils.pl" ]; then - echo `date` ": Searching '$cmdline'" | tee -a $test_log_file; - - elif [ $cmd = "echo" ]; then - echo -n "" -# if [ $OUTPUT_CMDS -eq 1 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then -# echo -n `date` ": " -# $cmdline -# elif [ $OUTPUT_ECHO -eq 1 -a $OUTPUT_CMDS -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then -# $cmdline -# fi - else - echo `date` ": Running '$cmdline'" | tee -a $test_log_file; - fi - fi - - if [ $cmd = "echo" ]; then - if [ $OUTPUT_CMDS -eq 1 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then - echo -n `date` ": " | tee -a $test_log_file - if [ $local_version != 1 ]; then - echo -n -e "\033[01;32m" - fi - $cmdline | tee -a $test_log_file - if [ $local_version != 1 ]; then - echo -n -e "\033[00m" - fi - elif [ $OUTPUT_ECHO -eq 1 -a $OUTPUT_CMDS -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0 ]; then - echo -n `date` ": " | tee -a $test_log_file - if [ $local_version != 1 ]; then - echo -n -e "\033[01;32m" - fi - $cmdline | tee -a $test_log_file - if [ $local_version != 1 ]; then - echo -n -e "\033[00m" - fi - fi - - elif [ $cmd = "${HALIB_DIR}/crmtest/testutils.pl" ]; then - # for the logs... - if [ $OUTPUT_STDERR -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_CMDS -eq 0 ]; then - : Searching for $cmdline -# echo `date` ": Searching '$cmdline'" | tee -a $test_log_file; - fi - $cmdline -v 2>&1 >> $test_log_file - rc=$? - - elif [ $OUTPUT_STDERR -eq 1 -a $OUTPUT_STDOUT -eq 1 ]; then - $cmdline 2>&1 >> $test_log_file - rc=$? - elif [ $OUTPUT_STDERR -eq 0 -a $OUTPUT_STDOUT -eq 0 ]; then - $cmdline 2>&1 >/dev/null - rc=$? - elif [ $OUTPUT_STDOUT -eq 0 ]; then - $cmdline >/dev/null 2>> $test_log_file - rc=$? - elif [ $OUTPUT_STDERR -eq 0 ]; then - $cmdline 2>/dev/null >> $test_log_file - rc=$? - else - $cmdline 2>&1 >> $test_log_file - rc=$? - fi - - OUTPUT_STDOUT=$old_OUTPUT_STDOUT -# echo Result: $rc >> $test_log_file - return $rc -} - -function crm-cleanup() { - - if [ -z $1 ]; then - test_init - fi - - for a_host in $node_list; do - crm-cleanup-node $a_host $1 - done -} - -function crm-cleanup-node() { - ha_host=$1 - do_cmd echo Cleaning up on $ha_host - - # be *very* sure everything has stopped - do_cmd remote_cmd $INIT_USER $ha_host "killall ${KILL_OPTS} heartbeat ccm lrmd crmd crmadmin cibadmin ha_logd stonithd cib pengine tengine" - - # resources too - do_cmd remote_cmd $INIT_USER $ha_host "/etc/ha.d/resource.d/IPaddr $ip_rsc_1 stop" 2>&1 > /dev/null - do_cmd remote_cmd $INIT_USER $ha_host "/etc/ha.d/resource.d/IPaddr $ip_rsc_2 stop" 2>&1 > /dev/null - - if [ -z $2 ]; then - # make *sure* theres nothing left over from last time - do_cmd remote_cmd $INIT_USER $ha_host "rm -f $HAVAR_DIR/crm/cib*.xml" - - remote_cmd ${INIT_USER} ${ha_host} "rm -f /tmp/crm/cores" 2>/dev/null - remote_cmd ${INIT_USER} ${ha_host} "rm -f ${HAVAR_DIR}/cores/*/*" 2>/dev/null - - do_cmd remote_cmd ${INIT_USER} ${ha_host} "ls -al ${HAVAR_DIR}/cores/" - remote_cmd ${INIT_USER} ${ha_host} "rm -rf /tmp/crm/* /tmp/crm/.log*" 2>/dev/null - do_cmd remote_cmd ${INIT_USER} ${ha_host} "ls -al ${HAVAR_DIR}/cores/" - - if [ `dirname $logfile` = "/tmp/crm" ]; then - do_cmd remote_cmd $INIT_USER $ha_host "/etc/init.d/syslog restart" - fi - remote_cmd $ADMIN_USER $ha_host "stat -L ${STAT_OPTS} $logfile > /tmp/crm/.logstart" - do_cmd remote_cmd $ADMIN_USER $ha_host "logger -i -p local7.info __crmtest_${test_type}_log_mark__" - do_cmd sleep 2 - do_cmd remote_cmd $ADMIN_USER $ha_host "echo 1 > /proc/sys/kernel/core_uses_pid" - -# zero out logs, maybe - fi -} - -function mark_log() -{ - export crm_log_pos=$(stat -L ${STAT_OPTS} $logfile) -} - -function gres() -{ - if [ $# -lt "3" ] - then - echo Usage: gres pattern replacement files - return 1 - fi - pattern=$1 - replacement=$2 - shift; shift; - filelist=$* - A="`echo | tr '\012' '\001' `" - - for file in $filelist; do - - fname=`basename $file` - dname=`dirname $file` - - is_backup=${fname##.gres.} - if [ "$is_backup" = "$fname" ]; then - grep -q "$pattern" "$file" - isthere=$? - - if [ $isthere = 0 ]; then -# echo Processing $file... - cp "$file" "$dname/.gres.$fname" && sed -e "s$A$pattern$A$replacement$A" "$dname/.gres.$fname" > "$file" - fi -# else -# echo "Ignoring backup of $is_backup in $dname" - fi - done -} - -# successfully sourced -if [ $local_version = 1 ]; then - test_node_1=`uname -n | tr 'A-Z' 'a-z'` - test_node_2="" - test_node_3="" - echo "Set test_node_1=\"$test_node_1\"" - echo "Unset test_node_2 and test_node_3" -fi - -node_list="$test_node_1 $test_node_2 $test_node_3" -node_count=0 -for a_host in $node_list; do - node_count=`expr $node_count + 1` -done - -if [ $node_count -lt $required_nodes ]; then - do_cmd echo "$required_nodes required for this test. You specified only $node_list" - do_cmd echo "test: FAILED" - return false -fi - -true diff --git a/crm/test/stop.sh.in b/crm/test/stop.sh.in deleted file mode 100644 index c2556c251c..0000000000 --- a/crm/test/stop.sh.in +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2004 Andrew Beekhof -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# - -testdir=@libdir@/heartbeat/crmtest -. ${testdir}/helper.sh || exit 1 - -# make *sure* theres nothing left over from last time -crm-cleanup 0 diff --git a/crm/test/testutils.pl b/crm/test/testutils.pl deleted file mode 100755 index 2690ec6bdc..0000000000 --- a/crm/test/testutils.pl +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/perl - - # Copyright (C) 2004 Andrew Beekhof - # - # This program is free software; you can redistribute it and/or - # modify it under the terms of the GNU General Public - # License as published by the Free Software Foundation; either - # version 2.1 of the License, or (at your option) any later version. - # - # This software is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of - # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - # General Public License for more details. - # - # You should have received a copy of the GNU General Public - # License along with this library; if not, write to the Free Software - # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - # - -$in_exp=0; -$in_err_exp=0; -$match_all=0; -$max_lines=0; -$log_file="/var/log/messages"; -$start_pos=-1; -$end_pos=-1; -$verbose=0; -$timeout=90; -@search_for = (); -@errors = (); - -while ( $_ = @ARGV[0], /^-/ ) { - shift; - if ( /^--search/ ) { - $do_search = 1; - - } elsif ( /^--dump/ ) { - $do_dump = 1; - - } elsif ( /^-t/ ) { - $timeout = $ARGV[0]; - shift; - - } elsif ( /^-pf/ ) { - $start_pos = `cat $ARGV[0]`; -# chop $start_pos; - shift; - - } elsif ( /^-p/ ) { - $start_pos = $ARGV[0]; - shift; - - } elsif ( /^-epf/ ) { - $end_pos = `cat $ARGV[0]`; -# chop $end_pos; - shift; - - } elsif ( /^-ep/ ) { - $end_pos = $ARGV[0]; - shift; - - } elsif ( /^-v/ ) { - $verbose = 1; - - } elsif ( /^-m/ ) { - $max_lines = $ARGV[0]; - shift; - - } elsif ( /^-l/ ) { - $log_file = $ARGV[0]; - shift; - - } elsif ( /^-a/ ) { - $match_all = 1; - - } elsif ( /^-s/ ) { - $this_exp=""; - while( @ARGV ) { - last if $ARGV[0] =~ /^-/; - $this_exp=$this_exp." ".$ARGV[0]; - shift; - } - $this_exp=substr($this_exp, 1); -# print STDOUT "Found search expression: _${this_exp}_\n"; - push @search_for, $this_exp; - - } elsif ( /^-e/ ) { - $this_exp=""; - while( @ARGV ) { - last if $ARGV[0] =~ /^-/; - $this_exp=$this_exp." ".$ARGV[0]; - shift; - } - $this_exp=substr($this_exp, 1); -# print STDOUT "Found error expression: _${this_exp}_\n"; - push @errors, $this_exp; - } else { - print STDOUT "huh? $_\n"; - } -} - -push @errors, "__crmtest_manual_abort__"; -# - -if( $do_search eq 1 ) { - $rc=string_search(); - if( $verbose > 0) { - print STDOUT "Search returned: $rc\n"; - } - exit $rc; - -} elsif( $do_dump eq 1 ) { - file_dup(); - -} - -sub remote_command() { - my ($user, $host, @command) = @_; - - my $args = ""; - - foreach $arg ( @command ) { - $args = $args." ".$arg; - } - - print STDOUT "Running \'".$args."\' as ".$user."@".$host."\n"; - $rc = system "/usr/bin/ssh", "-l", $user, $host, $args; - return $rc; -} - -sub file_dup() { - open(LOG, $log_file); - - print STDOUT "Duping $log_file from position $start_pos to $end_pos...\n"; - if( $start_pos > 0 ) { - if( $verbose > 0) { - print STDOUT "Starting search in $log_file from position $start_pos...\n"; - } - seek LOG, $start_pos, 0 - } else { - if( $verbose > 0) { - print STDOUT "Starting search in $log_file from EOF...\n"; - } - seek LOG, 0, 2; - } - - for(;;) - { -# print STDOUT "Checking $log_file for more data...\n"; - for($curpos = tell LOG; $_ = ; $curpos = tell LOG) - { - my $lpc = 0; - $line = $_; - $num_lines = $num_lines + 1; - - print STDOUT "[".$num_lines."]: ".$line; - - if($end_pos > 0 && $curpos > $end_pos) { - last; - } - } - - return; - } -} - -sub string_search() { - - my %results = {}; - my $num_lines = 0; - - open(LOG, $log_file); - $max_lines = -1; # turn off line "timeouts" for now - - if( $start_pos > 0 ) { - if( $verbose > 0) { - print STDOUT "Starting search in $log_file from position $start_pos...\n"; - } - seek LOG, $start_pos, 0 - } else { - if( $verbose > 0) { - print STDOUT "Starting search in $log_file from EOF...\n"; - } - seek LOG, 0, 2; - } - $start_time = time; - $end_time = $start_time + ${timeout}; - - for(;;) - { - if($end_time < time) { - print "Ran out of time... ${timeout} seconds\n"; - return -3000; - } - -# print STDOUT "Checking $log_file for more data...\n"; - for($curpos = tell LOG; $_ = ; $curpos = tell LOG) - { - my $lpc = 0; - $line = $_; - $num_lines = $num_lines + 1; - -# print STDOUT "Checking line[".$num_lines."]: ".$line; - if($max_lines > 0 && $num_lines > $max_lines) { - foreach $key (sort keys %results) { - print STDOUT "Found key \'".$key."\' ".$results{$key}." times.\n" if $results{$key} ne ""; - } - return -1000; - } - - foreach $regex (@search_for) { - $lpc = $lpc +1; - if ( $line =~ /$regex/ ) { - if($match_all eq "0") { - print STDOUT "Found match for (".$regex."): \n\t".$line; - return $lpc; - } else { - if( $results{$regex} ne "" ) { - $results{$regex} = $results{$regex} + 1; - } else { - $results{$regex} = 1; - } - $found = scalar(keys %results)-1; - if( $verbose > 0) { - print STDOUT "[line $num_lines]: Found match ".$found." of ".scalar(@search_for)." for (".$regex."): \n\t".$line; - } - if(scalar(@search_for) < scalar(keys %results)) { - - if( $verbose > 0) { - foreach $key (sort keys %results) { - print STDOUT "Found key \'".$key."\' ".$results{$key}." times.\n" if $results{$key} ne ""; - } - } - return 0; - } - } - } - } - - $lpc = 0; - foreach $regex ( @errors ) { - $lpc = $lpc +1; - if ( $line =~ /$regex/ ) { - print STDOUT "Found ERROR match for (".$regex."): ".$line; - return 0-$lpc; - } - } - } - sleep 3; - seek LOG, $curpos, 0; - } -# print STDOUT "No more lines\n"; - return -2000; -} -