diff --git a/Makefile.am b/Makefile.am index 4cce132..db1f989 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,84 +1,87 @@ SUBDIRS = src agent man tests # .gz because github doesn't support .xz yet :-( # this is modified # TAG ?= $(shell git log --pretty="format:%H" -n 1 || sed -n -e "s/%global commit //p" sbd.spec)$(shell test -n "$$(git status -s)" && echo -n "-mod") distdir = $(PACKAGE)-$(TAG) TARFILE = $(distdir).tar.gz DIST_ARCHIVES = $(TARFILE) KEEP_EXISTING_TAR = no INJECT_GIT_COMMIT = yes DISTCLEANFILES = sbd-* sbd-*/ CLEANFILES = *.rpm *.tar.* sbd-* RPM_ROOT = $(shell pwd) RPM_OPTS = --define "_sourcedir $(RPM_ROOT)" \ --define "_specdir $(RPM_ROOT)" \ --define "_srcrpmdir $(RPM_ROOT)" \ --define "_builddir $(RPM_ROOT)" \ --define "_rpmdir $(RPM_ROOT)" MOCK_TARGET ?= rhel-7.1-candidate-x86_64 MOCK_OPTIONS ?= --resultdir=$(RPM_ROOT)/mock --no-cleanup-after BUILD_COUNTER ?= build.counter LAST_COUNT = $(shell test ! -e $(BUILD_COUNTER) && echo 0; test -e $(BUILD_COUNTER) && cat $(BUILD_COUNTER)) COUNT = $(shell expr 1 + $(LAST_COUNT)) TESTS = tests/regressions.sh export SBD_BINARY := src/sbd +export SBD_PRELOAD := tests/.libs/libsbdtestbed.so +export SBD_USE_DM := no + EXTRA_DIST = sbd.spec tests/regressions.sh export: rm -f $(PACKAGE)-HEAD.tar.* if test "$(KEEP_EXISTING_TAR)" != "yes"; then \ rm -f $(TARFILE); \ fi; ! (git status -s | grep "??" && echo "untracked files present in git-repo" ) if [ -f $(TARFILE) ]; then \ echo `date`: Using existing tarball: $(TARFILE); \ else \ rm -f $(PACKAGE).tar.*; \ (git archive --prefix=$(distdir)/ $(shell echo $(TAG)|cut -f1 -d-) || tar -c --transform="s,^,$(distdir)/," --exclude="*.tar.*" --exclude="$(distdir)" --exclude="*.o" --exclude="*.8" --exclude="config.*" --exclude="Makefile" --exclude="Makefile.in" --exclude="stamp-*" --exclude="*.service" --exclude="sbd" --exclude="*.m4" --exclude="*.cache" --exclude="configure" --exclude="*.list" --exclude="depcomp" --exclude="install-sh" --exclude="missing" --exclude="compile" --exclude="sbd.sh" --exclude="~" --exclude="*.swp" --exclude="*.patch" --exclude="*.diff" --exclude="*.orig" --exclude="*.rej" --exclude="*.rpm" --exclude=".deps" --exclude="test-driver" *) | gzip > $(TARFILE); \ if test -n "$$(git status -s)" || test "$(INJECT_GIT_COMMIT)" = "yes"; then \ if test -n "$$(git status -s)"; then git diff HEAD --name-only|grep -v "^\."|xargs -n1 git diff HEAD > uncommitted.diff; fi; \ rm -rf $(distdir); tar -xzf $(TARFILE); rm $(TARFILE); \ cd $(distdir); \ if test -n "$$(git status -s)"; then patch -p1 -i ../uncommitted.diff; fi; \ cd ..; \ sed -i 's/global\ commit.*/global\ commit\ $(TAG)/' $(distdir)/$(PACKAGE).spec; \ tar -czf $(TARFILE) $(distdir); rm -rf $(distdir); \ rm -f uncommitted.diff; \ fi; \ echo `date`: Rebuilt $(TARFILE); \ fi #replace commit id in sbd.spec spec: rm -f *.src.rpm rm -rf $(distdir) mkdir $(distdir) cp $(PACKAGE).spec $(distdir) sed -i 's/global\ commit.*/global\ commit\ $(TAG)/' $(distdir)/$(PACKAGE).spec srpm: export spec if [ -e $(BUILD_COUNTER) ]; then \ sed -i 's/global\ buildnum.*/global\ buildnum\ $(COUNT)/' $(distdir)/$(PACKAGE).spec; \ echo $(COUNT) > $(BUILD_COUNTER); \ fi rpmbuild $(RPM_OPTS) -bs $(distdir)/$(PACKAGE).spec rpm: export spec rpmbuild $(RPM_OPTS) -ba $(distdir)/$(PACKAGE).spec mock: srpm -rm -rf $(RPM_ROOT)/mock @echo "mock --root=$* --rebuild $(MOCK_OPTIONS) $(RPM_ROOT)/*.src.rpm" mock --root=$(MOCK_TARGET) --rebuild $(MOCK_OPTIONS) $(RPM_ROOT)/*.src.rpm beekhof: mock cluster-helper -- 'rm -f sbd-*.x86_64.rpm' cluster-helper --copy $(RPM_ROOT)/mock/sbd-*.x86_64.rpm {}: cluster-helper -- yum install -y sbd-*.x86_64.rpm diff --git a/tests/regressions.sh b/tests/regressions.sh index c8733e2..b06166d 100755 --- a/tests/regressions.sh +++ b/tests/regressions.sh @@ -1,182 +1,332 @@ #!/bin/bash # # Copyright (C) 2013 Lars Marowsky-Bree # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # TODO: # - More tests # - Handle optional, long-running tests better # - Support for explicitly running a single test # - Verify output from commands # - Normalize uuids and device names so they are diffable # - Log to file, instead of syslog is needed # - How to test watch mode? # - Can the unit/service file be tested? or at least the wrapper? : ${SBD_BINARY:="/usr/sbin/sbd"} +: ${SBD_PRELOAD="libsbdtestbed.so"} +: ${SBD_USE_DM:="yes"} sbd() { - ${SBD_BINARY} $* + LD_PRELOAD=${SBD_PRELOAD} SBD_WATCHDOG_TIMEOUT=5 SBD_DEVICE="${SBD_DEVICE}" SBD_PRELOAD_LOG=${SBD_PRELOAD_LOG} SBD_WATCHDOG_DEV=/dev/watchdog setsid ${SBD_BINARY} -p ${SBD_PIDFILE} $* +} + +sbd_wipe_disk() { + dd if=/dev/zero of=$1 count=2048 2>/dev/null } sbd_setup() { trap sbd_teardown EXIT for N in $(seq 3) ; do F[$N]=$(mktemp /tmp/sbd.device.$N.XXXXXX) - R[$N]=$(echo ${F[$N]}|cut -f4 -d.) - dd if=/dev/zero of=${F[$N]} count=2048 - L[$N]=$(losetup -f) - losetup ${L[$N]} ${F[$N]} - D[$N]="/dev/mapper/sbd_${N}_${R[$N]}" - dmsetup create sbd_${N}_${R[$N]} --table "0 2048 linear ${L[$N]} 0" - dmsetup mknodes sbd_${N}_${R[$N]} + sbd_wipe_disk ${F[$N]} + if [[ "${SBD_USE_DM}" == "yes" ]]; then + R[$N]=$(echo ${F[$N]}|cut -f4 -d.) + L[$N]=$(losetup -f) + losetup ${L[$N]} ${F[$N]} + D[$N]="/dev/mapper/sbd_${N}_${R[$N]}" + dmsetup create sbd_${N}_${R[$N]} --table "0 2048 linear ${L[$N]} 0" + dmsetup mknodes sbd_${N}_${R[$N]} + else + D[$N]=${F[$N]} + fi done + if [[ "${SBD_USE_DM}" != "yes" ]]; then + SBD_DEVICE="${F[1]};${F[2]};${F[3]}" + fi + SBD_PIDFILE=$(mktemp /tmp/sbd.pidfile.XXXXXX) + SBD_PRELOAD_LOG=$(mktemp /tmp/sbd.logfile.XXXXXX) } sbd_teardown() { for N in $(seq 3) ; do - dmsetup remove sbd_${N}_${R[$N]} - losetup -d ${L[$N]} + if [[ "${SBD_USE_DM}" == "yes" ]]; then + dmsetup remove sbd_${N}_${R[$N]} + losetup -d ${L[$N]} + fi rm -f ${F[$N]} + sbd_daemon_cleanup + rm -f ${SBD_PIDFILE} + rm -f ${SBD_PRELOAD_LOG} done } sbd_dev_fail() { - dmsetup wipe_table sbd_${1}_${R[$1]} + if [[ "${SBD_USE_DM}" == "yes" ]]; then + dmsetup wipe_table sbd_${1}_${R[$1]} + else + D[$1]=/tmp/fail123456789 + fi } sbd_dev_resume() { - dmsetup suspend sbd_${1}_${R[$1]} - dmsetup load sbd_${1}_${R[$1]} --table "0 2048 linear ${L[$1]} 0" - dmsetup resume sbd_${1}_${R[$1]} + if [[ "${SBD_USE_DM}" == "yes" ]]; then + dmsetup suspend sbd_${1}_${R[$1]} + dmsetup load sbd_${1}_${R[$1]} --table "0 2048 linear ${L[$1]} 0" + dmsetup resume sbd_${1}_${R[$1]} + else + D[$1]=${F[$1]} + fi +} + +sbd_daemon_cleanup() { + echo > ${SBD_PRELOAD_LOG} + pkill -TERM --pidfile ${SBD_PIDFILE} 2>/dev/null + sleep 5 + pkill -KILL --pidfile ${SBD_PIDFILE} 2>/dev/null + pkill -KILL --parent $(cat ${SBD_PIDFILE} 2>/dev/null) 2>/dev/null + echo > ${SBD_PIDFILE} } _ok() { echo -- $@ $@ rc=$? if [ $rc -ne 0 ]; then echo "$@ failed with $rc" exit $rc fi } _no() { echo -- $@ $@ rc=$? if [ $rc -eq 0 ]; then echo "$@ did NOT fail ($rc)" exit $rc fi return 0 } +_in_log() { + grep "$@" ${SBD_PRELOAD_LOG} >/dev/null + if [ $? -ne 0 ]; then + echo "didn't find '$@' in log:" + cat ${SBD_PRELOAD_LOG} + sbd_daemon_cleanup + exit 1 + fi +} + test_1() { echo "Creating three devices" _ok sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} create _ok sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} dump } test_2() { echo "Basic functionality" for S in `seq 2` ; do _ok sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} allocate "test-$S" done _ok sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} -n test-1 message test-2 reset _ok sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} list } test_3() { echo "Start mode (expected not to start, because reset was written in test_2)" _no sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} -n test-2 -Z -Z -Z -S 1 watch } test_4() { echo "Deliver message with 1 failure" sbd_dev_fail 1 _no sbd -d ${D[1]} -n test-1 message test-2 exit _no sbd -d ${D[1]} -d ${D[2]} -n test-1 message test-2 exit _ok sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} -n test-1 message test-2 exit sbd_dev_resume 1 } test_5() { echo "Deliver message with 2 failures" sbd_dev_fail 1 sbd_dev_fail 2 _no sbd -d ${D[1]} -d ${D[2]} -n test-1 message test-2 exit _no sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} -n test-1 message test-2 exit sbd_dev_resume 1 sbd_dev_resume 2 } test_6() { echo "Deliver message with 3 failures" sbd_dev_fail 1 sbd_dev_fail 2 sbd_dev_fail 3 _no sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} -n test-1 message test-2 exit sbd_dev_resume 1 sbd_dev_resume 2 sbd_dev_resume 3 } test_101() { echo "Creating one device" _ok sbd -d ${D[1]} create } test_102() { echo "Creating two devices" _ok sbd -d ${D[1]} -d ${D[2]} create } test_7() { echo "Allocate all slots plus 1" _ok sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} -2 0 create for S in `seq 255` ; do _ok sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} allocate "test-$S" done _no sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} allocate "test-256" } test_8() { echo "Non-existent device path" _no sbd -d /dev/kfdifdifdfdlfd -create 2>/dev/null } test_9() { echo "Basic sbd invocation" _no sbd _ok sbd -h } +test_watchdog() { + echo "Basic watchdog test" + echo > ${SBD_PRELOAD_LOG} + sbd test-watchdog < /dev/null + _in_log "watchdog fired" +} + +test_stall_inquisitor() { + echo "Stall inquisitor test" + sbd_daemon_cleanup + sbd -d ${D[1]} -d ${D[2]} -d ${D[3]} -n test-1 watch + sleep 10 + _ok kill -0 $(cat ${SBD_PIDFILE}) + kill -STOP $(cat ${SBD_PIDFILE}) + sleep 10 + kill -CONT $(cat ${SBD_PIDFILE}) 2>/dev/null + _in_log "watchdog fired" +} + +test_wipe_slots1() { + echo "Wipe slots test (with watchdog)" + sbd_daemon_cleanup + sbd -d ${D[1]} -n test-1 watch + sleep 2 + sbd_wipe_disk ${D[1]} + sleep 15 + _in_log "watchdog fired" +} + +test_wipe_slots2() { + echo "Wipe slots test (without watchdog)" + sbd_daemon_cleanup + sbd -d ${D[1]} create + sbd -d ${D[1]} -w /dev/null -n test-1 watch + sleep 2 + sbd_wipe_disk ${D[1]} + sleep 15 + _in_log "sysrq-trigger ('b')" + _in_log "reboot (reboot)" +} + +test_message1() { + echo "Message test (reset)" + sbd_daemon_cleanup + sbd -d ${D[1]} create + sbd -d ${D[1]} -w /dev/null -n test-1 watch + sleep 2 + sbd -d ${D[1]} message test-1 reset + sleep 2 + _in_log "sysrq-trigger ('b')" + _in_log "reboot (reboot)" +} + +test_message2() { + echo "Message test (off)" + sbd_daemon_cleanup + sbd -d ${D[1]} create + sbd -d ${D[1]} -w /dev/null -n test-1 watch + sleep 2 + sbd -d ${D[1]} message test-1 off + sleep 2 + _in_log "sysrq-trigger ('o')" + _in_log "reboot (poweroff)" +} + +test_message3() { + echo "Message test (crashdump)" + sbd_daemon_cleanup + sbd -d ${D[1]} create + sbd -d ${D[1]} -w /dev/null -n test-1 watch + sleep 2 + sbd -d ${D[1]} message test-1 crashdump + sleep 2 + _in_log "sysrq-trigger ('c')" +} + +test_timeout_action1() { + echo "Timeout action test (off)" + sbd_daemon_cleanup + sbd -d ${D[1]} create + SBD_TIMEOUT_ACTION=off sbd -d ${D[1]} -w /dev/null -n test-1 watch + sleep 2 + sbd_wipe_disk ${D[1]} + sleep 10 + _in_log "sysrq-trigger ('o')" + _in_log "reboot (poweroff)" +} + +test_timeout_action2() { + echo "Timeout action test (crashdump)" + sbd_daemon_cleanup + sbd -d ${D[1]} create + SBD_TIMEOUT_ACTION=crashdump sbd -d ${D[1]} -w /dev/null -n test-1 watch + sleep 2 + sbd_wipe_disk ${D[1]} + sleep 10 + _in_log "sysrq-trigger ('c')" +} + sbd_setup -for T in $(seq 9); do +if [[ "${SBD_PRELOAD}" != "" ]]; then + SBD_DAEMON_TESTS="watchdog stall_inquisitor wipe_slots1 wipe_slots2 message1 message2 message3 timeout_action1 timeout_action2" +fi + +for T in $(seq 9) ${SBD_DAEMON_TESTS}; do if ! test_$T ; then echo "FAILURE: Test $T" break fi echo "SUCCESS: Test $T" done echo "SUCCESS: All tests completed"