diff --git a/tools/ocft/ocft.in b/tools/ocft/ocft.in index 957a8fc06..b84046098 100644 --- a/tools/ocft/ocft.in +++ b/tools/ocft/ocft.in @@ -1,865 +1,895 @@ #!/bin/bash # Copyright (c) 2010-2013 Novell Inc, John Shi # All Rights Reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as # published by the Free Software Foundation. # # This program is distributed in the hope that it would be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # # Further, this software is distributed without any warranty that it is # free of the rightful claim of any third person regarding infringement # or the like. Any license provided herein, whether implied or # otherwise, applies only to this software file. Patent licenses, if # any, provided herein do not apply to combinations of this program with # other software, or any other product whatsoever. # # You should have received a copy of the GNU General Public License # along with this program; if not, write the Free Software Foundation, # Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. die() { local str str="$1" echo "ERROR: $str" >&2 exit 1 } warn() { local str str="$1" echo "WARNING: $str" >&2 } parse_die() { local str str="$1" agent_parse_finish die "${agent}: line ${line_num}: ${str}" } # add quotes to string for Here Documents add_quotes() { local typ str a b typ="$1" str="$2" case "$typ" in 1) a=\'; b=\";; 2) a=\"; b=\';; esac echo "$str" | sed "s/$a/$a$b$a$b$a/g; 1 s/^/$a/; $ s/$/$a/" } # split strings explode() { local str str="$1" echo "$str" | awk -F'"' '{ if (NF > 0 && NF%2 == 0) exit(1); for (i=1; i<=NF; i++) { if (i%2 == 0) print $i; else { len = split($i, str, /[ \t]+/); for (j=1; j<=len; j++) { sb = sub(/#.*/, "", str[j]); if (str[j] != "") print str[j]; if (sb) exit(0); } } } }' } # phase 1: parse the string to 'command' and 'argument collection'. line2trunk() { trunk[0]="${line%%[[:blank:]]*}" trunk[1]="${line#*[[:blank:]]}" } # phase 2: split the argument collection. trunk2branch() { local IFS # Some of statements need one parameter at least. if [ "$line" = "${trunk[0]}" ]; then parse_die "missing parameter." fi IFS=$'\n' branch=($(explode "${trunk[1]}")) if [ $? -ne 0 ]; then parse_die "missing '\"'." fi } preparse_cfg() { local agent line trunk branch macro num host agent="$1" if [ ! -r "$opt_cfgsdir/$agent" ]; then die "${agent}: configuration file not found." fi line_num=0 while read -r line; do let line_num++ num=" $line_num" case "$line" in ""|\#*) continue;; esac line2trunk case "${trunk[0]}" in CASE-BLOCK) trunk2branch macro="$CASES_DIR/${agent}_macro.${branch[0]}" continue ;; Include|Include@*) host=$(echo "${trunk[0]}" | awk -F@ '{print $2}') trunk2branch if [ ! -r "$CASES_DIR/${agent}_macro.${branch[0]}" ]; then parse_die "Macro '${branch[0]}' not found." fi if [ -n "$host" ]; then line="$(sed -e 's/^\([^[:blank:]]*\)@[^[:blank:]]*/\1/' -e "s/^[^[:blank:]]*/&@$host/" "$CASES_DIR/${agent}_macro.${branch[0]}")" else line="$(<"$CASES_DIR/${agent}_macro.${branch[0]}")" fi num= ;; *[!A-Z-]*) : ;; *) macro= ;; esac if [ -n "$macro" ]; then echo "$line$num" >>"$macro" else echo "$line$num" >>"$CASES_DIR/${agent}_preparse" fi done <"$opt_cfgsdir/$agent" } case_parse_finish() { local host if [ -n "$sh" ]; then cat >>$sh <<EOF if [ -n "\$__OCFT__VERBOSE" ]; then echo fi # Cleanup and exit EOF for host in $hosts; do echo "backbash_stop $host" >>$sh done echo "quit 0" >>$sh fi atexit_num=0 hosts= sh= } init_cfg_vars() { cfg_agent= cfg_agent_root= cfg_install_package=() cfg_hang_timeout=20 } agent_parse_finish() { local suf for suf in preparse setup cleanup var hosts; do rm -f $CASES_DIR/${agent}_$suf done rm -f $CASES_DIR/${agent}_macro.* init_cfg_vars } need_make() { local src_time obj_time if [ ! -f "$CASES_DIR/0_${agent}.sh" ]; then return 0 fi src_time=$(stat -c '%Y' "$opt_cfgsdir/$agent") obj_time=$(stat -c '%Y' "$CASES_DIR/0_${agent}.sh") test $src_time -ge $obj_time } parse_cfg() { local agents i line stat sh trunk branch atexit_num host hosts if [ $# -eq 0 ]; then agents=($opt_cfgsdir/*) else agents=("$@") fi for agent in "${agents[@]}"; do agent="$(basename "$agent")" if ! need_make; then continue fi agent_obj_clean $agent agent_parse_finish i=0 echo "Making '$agent': " preparse_cfg "$agent" while read -r line; do line_num="${line##* }" line="${line% *}" line2trunk # state switch case "${trunk[0]}" in CONFIG) case_parse_finish stat=1 continue ;; VARIABLE) case_parse_finish stat=2 continue ;; - SETUP-AGENT) + SETUP-AGENT) case_parse_finish stat=3 continue ;; - CLEANUP-AGENT) + CLEANUP-AGENT) case_parse_finish stat=4 continue ;; CASE) case_parse_finish trunk2branch echo " - case ${i}: ${branch[0]}" sh="$CASES_DIR/${i}_${agent}.sh" cat >$sh <<EOF #!/bin/bash # Agent: $cfg_agent # Summary: ${branch[0]} . $OCFT_DIR/caselib || { echo "ERROR: '$OCFT_DIR/caselib' not found." exit 2 } $(test -r $CASES_DIR/${agent}_var && cat $CASES_DIR/${agent}_var) __OCFT__MYROOT="$cfg_agent_root" $(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}") if [ -n "\$__OCFT__VERBOSE" ]; then echo -e $(add_quotes 1 "Starting '\\033[33m${agent}\\033[0m' case $i '\\033[33m${branch[0]}\\033[0m':") else echo -n "${agent}: ${branch[0]} - " fi EOF chmod a+x $sh let i++ stat=5 continue ;; esac case "$stat" in 1) case "${trunk[0]}" in Agent) trunk2branch cfg_agent="${branch[0]}" ;; AgentRoot) trunk2branch cfg_agent_root="${branch[0]}" ;; InstallPackage) trunk2branch cfg_install_package=(${cfg_install_package[@]} ${branch[@]}) ;; HangTimeout) trunk2branch if ! echo "${branch[0]}" | grep -qxE '[0-9]+'; then parse_die "numeric argument required." fi cfg_hang_timeout="${branch[0]}" ;; *) parse_die "unimplemented statement: ${trunk[0]}" ;; esac ;; 2) if echo "$line" | grep -q '^__OCFT__'; then parse_die "reserved key word '__OCFT__'." fi echo "declare $line" >>$CASES_DIR/${agent}_var ;; 3) echo "$line" >>$CASES_DIR/${agent}_setup ;; 4) echo "$line" >>$CASES_DIR/${agent}_cleanup ;; 5) host=$(echo ${trunk[0]} | awk -F@ '{print $2}') if [ -n "$host" ]; then if ! echo "$hosts" | grep -q "$host"; then echo "$host" >>$CASES_DIR/${agent}_hosts hosts=$hosts$'\n'$host cat >>$sh <<EOF # Initialize remote shell backbash_start $host backbash $host <<CMD __OCFT__VERBOSE=\$__OCFT__VERBOSE CMD backbash $host <$OCFT_DIR/caselib backbash $host <<'CMD' $(test -r $CASES_DIR/${agent}_var && cat $CASES_DIR/${agent}_var) __OCFT__MYROOT="$cfg_agent_root" __OCFT__showhost="${host}: " $(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}") CMD EOF fi fi echo " # CASE statement: $line" >>$sh if [ -n "$host" ]; then echo "backbash $host <<'CMD'" >>$sh fi case "${trunk[0]}" in Env|Env@*) cat >>$sh <<EOF if [ -n "\$__OCFT__VERBOSE" ]; then echo $(add_quotes 2 " \${__OCFT__showhost}Setting agent environment: export ${trunk[1]}") fi export ${trunk[1]} check_success \$? $(add_quotes 1 "export ${trunk[1]}") EOF ;; Unenv|Unenv@*) cat >>$sh <<EOF if [ -n "\$__OCFT__VERBOSE" ]; then echo $(add_quotes 2 " \${__OCFT__showhost}Removing agent environment: unset ${trunk[1]}") fi unset ${trunk[1]} check_success \$? $(add_quotes 1 "unset ${trunk[1]}") EOF ;; AgentRun|AgentRun@*) trunk2branch if [ -z "${branch[1]}" ]; then if [ "${branch[0]}" = "start" ]; then cat >>$sh <<EOF agent_run $(add_quotes 1 "$cfg_agent") monitor $cfg_hang_timeout __OCFT__rc=\$? if [ \$__OCFT__rc -eq \$OCF_SUCCESS -o \$__OCFT__rc -eq \$OCF_RUNNING_MASTER ]; then : #The status I want, so I can do nothing. elif [ \$__OCFT__rc -eq \$OCF_NOT_RUNNING ]; then if [ -n "\$__OCFT__VERBOSE" ]; then echo $(add_quotes 2 " \${__OCFT__showhost}Running agent: ./$cfg_agent ${branch[0]}") fi agent_run $(add_quotes 1 "$cfg_agent") start $cfg_hang_timeout check_success \$? $(add_quotes 1 "./$cfg_agent ${branch[0]}") else check_success \$__OCFT__rc $(add_quotes 1 "./$cfg_agent monitor") fi EOF elif [ "${branch[0]}" = "stop" ]; then cat >>$sh <<EOF agent_run $(add_quotes 1 "$cfg_agent") monitor $cfg_hang_timeout __OCFT__rc=\$? if [ \$__OCFT__rc -eq \$OCF_NOT_RUNNING ]; then : #The status I want, so I can do nothing. elif [ \$__OCFT__rc -eq \$OCF_SUCCESS -o \$__OCFT__rc -eq \$OCF_RUNNING_MASTER ]; then if [ -n "\$__OCFT__VERBOSE" ]; then echo $(add_quotes 2 " \${__OCFT__showhost}Running agent: ./$cfg_agent ${branch[0]}") fi agent_run $(add_quotes 1 "$cfg_agent") stop $cfg_hang_timeout check_success \$? $(add_quotes 1 "./$cfg_agent ${branch[0]}") else check_success \$__OCFT__rc $(add_quotes 1 "./$cfg_agent monitor") fi EOF elif [ "${branch[0]}" = "monitor" ]; then cat >>$sh <<EOF if [ -n "\$__OCFT__VERBOSE" ]; then echo $(add_quotes 2 " \${__OCFT__showhost}Running agent: ./$cfg_agent ${branch[0]}") fi agent_run $(add_quotes 1 "$cfg_agent") $(add_quotes 1 "${branch[0]}") $cfg_hang_timeout EOF else cat >>$sh <<EOF if [ -n "\$__OCFT__VERBOSE" ]; then echo $(add_quotes 2 " \${__OCFT__showhost}Running agent: ./$cfg_agent ${branch[0]}") fi agent_run $(add_quotes 1 "$cfg_agent") $(add_quotes 1 "${branch[0]}") $cfg_hang_timeout check_success \$? $(add_quotes 1 "./$cfg_agent ${branch[0]}") EOF fi else cat >>$sh <<EOF test -n $(add_quotes 2 "\$${branch[1]}") check_success \$? $(add_quotes 1 "test -n \"\$${branch[1]}\"") if [ -n "\$__OCFT__VERBOSE" ]; then echo $(add_quotes 2 " \${__OCFT__showhost}Running agent: ./$cfg_agent ${branch[0]}") fi agent_run $(add_quotes 1 "$cfg_agent") $(add_quotes 1 "${branch[0]}") $cfg_hang_timeout __OCFT__ret=\$? if [ -n "\$__OCFT__VERBOSE" ]; then echo -n " \${__OCFT__showhost}Checking return value:" fi if [ -n "\${__OCFT__retval[__OCFT__ret]}" ]; then __OCFT__retstr="\${__OCFT__retval[__OCFT__ret]}" else __OCFT__retstr=\$__OCFT__ret fi if [ \$__OCFT__ret -eq \$${branch[1]} ]; then if [ -n "\$__OCFT__VERBOSE" ]; then echo -e $(add_quotes 2 " \\033[32mOK\\033[0m. The return value '\\033[34m\$__OCFT__retstr\\033[0m' == '\\033[34m${branch[1]}\\033[0m'") else echo -e "\\033[32mOK\\033[0m." fi else if [ -n "\$__OCFT__VERBOSE" ]; then echo -en $(add_quotes 2 " \\033[31mFAILED\\033[0m. The return value '\\033[34m\$__OCFT__retstr\\033[0m' != '\\033[34m${branch[1]}\\033[0m'. ") else echo -en "\\033[31mFAILED\\033[0m. Agent returns unexpected value: '\$__OCFT__retstr'. " fi echo "See details below:" cat /tmp/.ocft_runlog echo quit 1 fi EOF fi ;; Bash|Bash@*) cat >>$sh <<EOF if [ -n "\$__OCFT__VERBOSE" ]; then echo $(add_quotes 2 " \${__OCFT__showhost}Setting system environment: ${trunk[1]}") fi ${trunk[1]} check_success \$? $(add_quotes 1 "${trunk[1]}") EOF ;; BashAtExit|BashAtExit@*) let atexit_num++ cat >>$sh <<EOF atexit${atexit_num}() { if [ -n "\$__OCFT__VERBOSE" ]; then echo $(add_quotes 2 " \${__OCFT__showhost}Setting system environment: ${trunk[1]}") fi ${trunk[1]} } let __OCFT__atexit_num++ EOF ;; *) parse_die "unimplemented statement: ${trunk[0]}" ;; esac if [ -n "$host" ]; then echo 'CMD' >>$sh fi ;; *) parse_die "unimplemented statement: ${trunk[0]}" ;; esac done <$CASES_DIR/${agent}_preparse if [ -r "$CASES_DIR/${agent}_setup" ]; then cat >$CASES_DIR/setup_${agent}.sh <<EOF #!/bin/bash # Agent: $cfg_agent # Summary: SETUP before test echo "Initializing '$cfg_agent' ..." . $OCFT_DIR/caselib || { echo "ERROR: '$OCFT_DIR/caselib' not found." exit 2 } $(test -r "$CASES_DIR/${agent}_var" && cat $CASES_DIR/${agent}_var) $(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}") EOF for host in $(test -r $CASES_DIR/${agent}_hosts && cat $CASES_DIR/${agent}_hosts); do cat >>$CASES_DIR/setup_${agent}.sh <<EOF # Initialize remote shell backbash_start $host backbash $host <<CMD __OCFT__VERBOSE=\$__OCFT__VERBOSE CMD backbash $host <$OCFT_DIR/caselib backbash $host <<'CMD' $(test -r "$CASES_DIR/${agent}_var" && cat $CASES_DIR/${agent}_var) __OCFT__MYROOT="$cfg_agent_root" __OCFT__showhost="${host}: " $(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}") $(cat $CASES_DIR/${agent}_setup) check_success \$? "SETUP-AGENT" CMD backbash_stop $host EOF done cat >>$CASES_DIR/setup_${agent}.sh <<EOF $(cat $CASES_DIR/${agent}_setup) check_success \$? "SETUP-AGENT" echo "Done." echo quit 0 EOF chmod a+x $CASES_DIR/setup_${agent}.sh fi if [ -r "$CASES_DIR/${agent}_cleanup" ]; then cat >$CASES_DIR/cleanup_${agent}.sh <<EOF #!/bin/bash # Agent: $cfg_agent # Summary: CLEANUP after test echo "Cleaning '$cfg_agent' ..." . $OCFT_DIR/caselib || { echo "ERROR: '$OCFT_DIR/caselib' not found." exit 2 } $(test -r "$CASES_DIR/${agent}_var" && cat $CASES_DIR/${agent}_var) $(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}") $(cat $CASES_DIR/${agent}_cleanup) check_success \$? "CLEANUP-AGENT" EOF for host in $(test -r $CASES_DIR/${agent}_hosts && cat $CASES_DIR/${agent}_hosts); do cat >>$CASES_DIR/cleanup_${agent}.sh <<EOF # Initialize remote shell backbash_start $host backbash $host <<CMD __OCFT__VERBOSE=\$__OCFT__VERBOSE CMD backbash $host <$OCFT_DIR/caselib backbash $host <<'CMD' $(test -r "$CASES_DIR/${agent}_var" && cat $CASES_DIR/${agent}_var) __OCFT__MYROOT="$cfg_agent_root" __OCFT__showhost="${host}: " $(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}") $(cat $CASES_DIR/${agent}_cleanup) check_success \$? "CLEANUP-AGENT" CMD backbash_stop $host EOF done cat >>$CASES_DIR/cleanup_${agent}.sh <<EOF echo "Done." echo quit 0 EOF chmod a+x $CASES_DIR/cleanup_${agent}.sh fi case_parse_finish agent_parse_finish done } +set_ocf_env() +{ + local cfg=$1 + + export OCF_RA_VERSION_MAJOR=1 + export OCF_RA_VERSION_MINOR=0 + export OCF_RESOURCE_TYPE=${OCF_RESOURCE_TYPE:-"$cfg"} + export OCF_RESOURCE_INSTANCE=${OCF_RESOURCE_INSTANCE:-"ocft"} +} + start_test() { local sh shs testsh agents line ret + local varlib if ! cd $CASES_DIR >/dev/null 2>&1; then die "cases directory not found." fi if [ ! -d logs ]; then mkdir logs fi export __OCFT__VERBOSE=$opt_verbose if [ $# -eq 0 ]; then agents=($(ls -1 *.sh 2>/dev/null | sed 's/.*_\([^_]*\)\.sh$/\1/' | sort | uniq)) else agents=("$@") fi for shs in "${agents[@]}"; do + set_ocf_env $shs if [ -z "$opt_incremental" ]; then testsh="setup_${shs}.sh $(ls -1 [0-9]*_${shs}.sh 2>/dev/null | sort -n) cleanup_${shs}.sh" else testsh="setup_${shs}.sh $(ls -1 [0-9]*_${shs}.retest 2>/dev/null | sed 's/retest$/sh/' | sort -n) cleanup_${shs}.sh" fi + if [ -n "$opt_trace_ra" ]; then + varlib=${HA_VARLIB:="/var/lib/heartbeat"} + export OCF_RESKEY_trace_ra=1 + # not exactly resource type, but for this purpose ... + export OCF_RESOURCE_TYPE="ocft_$shs" + echo "RA trace on, output in $varlib/trace_ra/${OCF_RESOURCE_TYPE}" + fi + for sh in $testsh; do if [ -r "$sh" ]; then + if [ -n "$opt_trace_ra" ]; then + OCF_RESOURCE_INSTANCE="`echo $sh | sed 's/_.*//'`" + fi ./$sh ret=$? case "$sh" in setup*) if [ $ret -ne 0 ]; then warn "SETUP failed, break all tests of '$shs'." break fi ;; cleanup*) if [ $ret -ne 0 ]; then warn "CLEANUP failed." fi ;; [0-9]*) case $ret in 3) die "core function failed, break all tests." ;; 2) warn "core function failed, break all tests of '$shs'."; break ;; 1) touch ${sh%.*}.retest ;; 0) rm -f ${sh%.*}.retest ;; esac ;; esac fi done 2>&1 | while read -r line; do echo "$line" echo "$(date '+%F %T'): $line" | cat -A | sed -r 's/\^\[\[[0-9]+m|\^I|.$//g' >>logs/$shs.log done done } agent_clean() { local typ ra typ=$1 shift if [ $# -eq 0 ]; then rm -f $CASES_DIR/*.$typ else for ra in "$@"; do rm -f $CASES_DIR/*_${ra}.$typ done fi } agent_retest_clean() { agent_clean retest "$@" } agent_obj_clean() { agent_clean sh "$@" } usage() { cat <<EOF $0 ACTION [OPTION] [agent1 [agent2] [...]] ACTIONs include: make [-d dir] Generate the testing shell scripts. -d The directory that contains - configuration of cases. - test [-v|-i] Execute the testing shell scripts. + configuration of cases. + test [-v|-i|-X] Execute the testing shell scripts. -v Verbose output mode. -i Incremental mode, skip case which succeeded. If cleaning the status of incremental mode is needed, try to '$0 clean RA_NAME'. + -X Trace the RA clean Delete the testing shell scripts. help [-v] Show this help and exit. -v Show HOWTO and exit. Version 0.44 See '$OCFT_DIR/README' for detail. EOF } howto() { cat <<EOF HOW TO USE THIS TOOL - Ocft is a testing tool for resource agents. Instead of the policy of HA, it mainly concerns whether resource agents run correct locally. It can design types of complicated environments to test the reliability of resource agents. Precisely, it is to display whether resource agents can return to correct or expected value. The advantage of the tool provides us with competence to design conditions which can be recorded or reproduced. Hence it is useful to debuggers. * Components ** Test case generator (/usr/sbin/ocft) - Turning configuration files of test case to executable scripts. ** Configuration file ($CONFIGS_DIR/) - Every configuration file directs only one resource agent and share the same name with resource agent but contains more test cases. ** The testing script ($CASES_DIR/) - After the generator reads configuration files and generates many testing scripts and the script is underway, the test begins. * How to customize the environment of testing - Ocft designs the running conditions through two ways, one is changing the environment variables of resource agents (it is the interface left by OCF itself), the other is modifying the OS environment of resource agents, such as altering the permission of some key file or IP address of the machine. * How to test - Firstly, you need to sketch the all complex and uncommon environments against a certain resource agent and keep in mind what consequences may be caused by these uncommon environments. Secondly, write the designed conditions and foreknown consequences into configuration files, and then run the generator to translate the test case to executable scripts. Finally, you need running these scripts to observe the output and learn the running status of each test case, which will compares the predicated result with the actual one. If they differ, you will be able to find the bugs of the resource agent. - All of the output with test will be recorded into the log files, you can find them in $CASES_DIR/logs. EOF } export LANG=C # system variable OCFT_DIR=@datadir@/@PACKAGE_NAME@/ocft CONFIGS_DIR=@datadir@/@PACKAGE_NAME@/ocft/configs CASES_DIR=/var/lib/@PACKAGE_NAME@/ocft/cases # global variable agent= line_num= # default configuration init_cfg_vars # default option opt_verbose= opt_incremental= opt_cfgsdir=$CONFIGS_DIR command="$1" shift case "$command" in make) if [ "$1" = "-d" ]; then if [ ! -d "$2" ]; then usage exit 1 fi opt_cfgsdir="$2" shift 2 fi if [ ! -d "$CASES_DIR" ]; then mkdir -p "$CASES_DIR" || die "Can not create directory: ${CASES_DIR}." fi parse_cfg "$@" ;; test) - for v in 1 2; do + for v in 1 2 3; do case "$1" in -v) opt_verbose=1 shift ;; + -X) + opt_trace_ra=1 + shift + ;; -i) opt_incremental=1 shift ;; esac done start_test "$@" ;; clean) agent_obj_clean "$@" agent_retest_clean "$@" ;; help) if [ "$1" = "-v" ]; then howto else usage fi exit 0 ;; *) usage exit 1 ;; esac + +# vim:ts=2:sw=2:et: