diff --git a/.travis.yml b/.travis.yml index 51f146e..547adb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,70 +1,70 @@ dist: bionic sudo: required language: c os: - linux arch: - amd64 - ppc64le - s390x - arm64 env: global: - PACKAGE=sbd # appealing idea to go with centos 8 as build-host but unfortunately that isn't available for all platforms # and the docker-image isn't there for anything else but x86_64 # - BUILD_OS_TYPE="centos:" BUILD_OS_DIST=centos BUILD_OS_VERSION=8 # - BUILD_OS_PREPARE="yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && yum install -y mock yum-utils &&" - BUILD_OS_TYPE="fedora:" BUILD_OS_DIST= BUILD_OS_VERSION=32 - BUILD_OS_PREPARE="echo timeout=300 >> /etc/dnf/dnf.conf && dnf install -y mock dnf-utils findutils patch &&" matrix: exclude: - arch: amd64 - arch: ppc64le - arch: s390x - arch: arm64 include: - arch: amd64 - env: OS_ARCH=x86_64 OS_TYPE="centos:" OS_MOCK=epel OS_DIST=centos OS_VERSION=7 OS_INSTALL="yum install -y" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=yes" + env: OS_ARCH=x86_64 OS_TYPE="centos:" OS_MOCK=epel OS_DIST=centos OS_VERSION=7 OS_INSTALL="yum install -y" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=no" - arch: amd64 - env: OS_ARCH=x86_64 OS_TYPE="centos:" OS_MOCK=epel OS_DIST=centos OS_VERSION=6 OS_INSTALL="yum install -y" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=yes" + env: OS_ARCH=x86_64 OS_TYPE="centos:" OS_MOCK=epel OS_DIST=centos OS_VERSION=6 OS_INSTALL="yum install -y" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=no" - arch: amd64 - env: OS_ARCH=x86_64 OS_TYPE="fedora:" OS_MOCK=fedora OS_DIST= OS_VERSION=32 OS_INSTALL="dnf install -y" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=yes" + env: OS_ARCH=x86_64 OS_TYPE="fedora:" OS_MOCK=fedora OS_DIST= OS_VERSION=32 OS_INSTALL="dnf install -y" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=no" - arch: amd64 - env: OS_ARCH=x86_64 OS_TYPE="fedora:" OS_MOCK=fedora OS_DIST= OS_VERSION=rawhide OS_INSTALL="dnf install -y" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=yes" + env: OS_ARCH=x86_64 OS_TYPE="fedora:" OS_MOCK=fedora OS_DIST= OS_VERSION=rawhide OS_INSTALL="dnf install -y" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=no" - arch: ppc64le env: OS_ARCH=ppc64le OS_TYPE="fedora:" OS_MOCK=fedora OS_DIST= OS_VERSION=32 OS_INSTALL="dnf install --setopt=timeout=300 -y" DOCKER_OPTS="--cap-add=sys_admin" TEST_ENV="SBD_USE_DM=no" MOCK_OPTS="--config-opts=internal_dev_setup=False" - arch: s390x env: OS_ARCH=s390x OS_TYPE="fedora:" OS_MOCK=fedora OS_DIST= OS_VERSION=32 OS_INSTALL="dnf install -y" DOCKER_OPTS="--cap-add=sys_admin" TEST_ENV="SBD_USE_DM=no" MOCK_OPTS="--config-opts=internal_dev_setup=False" - arch: arm64 env: OS_ARCH=aarch64 OS_TYPE="fedora:" OS_MOCK=fedora OS_DIST= OS_VERSION=32 OS_INSTALL="dnf install -y" DOCKER_OPTS="--cap-add=sys_admin" TEST_ENV="SBD_USE_DM=no" MOCK_OPTS="--config-opts=internal_dev_setup=False" - arch: amd64 - env: OS_ARCH=x86_64 OS_TYPE="opensuse/" OS_MOCK=opensuse-leap OS_DIST="leap:" OS_VERSION=15.2 OS_INSTALL="zypper --no-gpg-checks --non-interactive install" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=yes" + env: OS_ARCH=x86_64 OS_TYPE="opensuse/" OS_MOCK=opensuse-leap OS_DIST="leap:" OS_VERSION=15.2 OS_INSTALL="zypper --no-gpg-checks --non-interactive install" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=no" - arch: amd64 - env: OS_ARCH=x86_64 OS_TYPE="opensuse/" OS_MOCK=opensuse OS_DIST= OS_VERSION=tumbleweed OS_INSTALL="zypper --no-gpg-checks --non-interactive install" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=yes" + env: OS_ARCH=x86_64 OS_TYPE="opensuse/" OS_MOCK=opensuse OS_DIST= OS_VERSION=tumbleweed OS_INSTALL="zypper --no-gpg-checks --non-interactive install" DOCKER_OPTS="--privileged" TEST_ENV="SBD_USE_DM=no" services: - docker install: true script: - BUILD_SUCCESS=false - make -f Makefile.am srpm PACKAGE=${PACKAGE} - docker pull ${BUILD_OS_TYPE}${BUILD_OS_DIST}${BUILD_OS_VERSION} - docker run --cap-add=sys_admin -v ${PWD}:/rpms -v /proc:/var/lib/mock/${OS_MOCK}-${OS_VERSION}-${OS_ARCH}/root/proc -v ${PWD}:/rpms -v /sys:/var/lib/mock/${OS_MOCK}-${OS_VERSION}-${OS_ARCH}/root/sys ${BUILD_OS_TYPE}${BUILD_OS_DIST}${BUILD_OS_VERSION} /bin/bash -c "${BUILD_OS_PREPARE} if test $OS_VERSION = rawhide; then sed -i /etc/mock/${OS_MOCK}-${OS_VERSION}-${OS_ARCH}.cfg -e s/gpgcheck.*/gpgcheck=0/g; fi && rpm -ql mock|grep "/mounts.py\$"|xargs -n1 sed -e "/USE_NSPAWN/d" -e "/self.mountall_essential/d" -i && mkdir -p /var/lib/mock/${OS_MOCK}-${OS_VERSION}-${OS_ARCH}/root/etc/dnf && cp /etc/dnf/dnf.conf /var/lib/mock/${OS_MOCK}-${OS_VERSION}-${OS_ARCH}/root/etc/dnf && mock --no-clean -r ${OS_MOCK}-${OS_VERSION}-${OS_ARCH} --resultdir=/rpms ${MOCK_OPTS} --disable-plugin=root_cache --disable-plugin=tmpfs --disable-plugin=yum_cache --disable-plugin=selinux --no-bootstrap-chroot --isolation=simple --config-opts=dnf_warning=False /rpms/sbd*.src.rpm" - ls ${PWD}/${PACKAGE}*.${OS_ARCH}.rpm && BUILD_SUCCESS=true - ${BUILD_SUCCESS} && docker pull ${OS_TYPE}${OS_DIST}${OS_VERSION} - ${BUILD_SUCCESS} && docker run ${DOCKER_OPTS} -v ${PWD}:/rpms ${OS_TYPE}${OS_DIST}${OS_VERSION} /bin/bash -c "if test $OS_VERSION = rawhide; then dnf update -y --nogpgcheck; fi && ${OS_INSTALL} device-mapper /rpms/${PACKAGE}*.${OS_ARCH}.rpm && ${TEST_ENV} /usr/share/sbd/regressions.sh && touch /rpms/regressions.sh.SUCCESS" - ls ${PWD}/regressions.sh.SUCCESS addons: apt: packages: - rpm diff --git a/configure.ac b/configure.ac index 3391c5f..23547cf 100644 --- a/configure.ac +++ b/configure.ac @@ -1,289 +1,302 @@ dnl dnl autoconf for Agents dnl dnl License: GNU General Public License (GPL) dnl =============================================== dnl Bootstrap dnl =============================================== AC_PREREQ(2.63) dnl Suggested structure: dnl information on the package dnl checks for programs dnl checks for libraries dnl checks for header files dnl checks for types dnl checks for structures dnl checks for compiler characteristics dnl checks for library functions dnl checks for system services AC_INIT([sbd], [1.4.1], [lmb@suse.com]) m4_include([tests-opt.m4]) AC_CANONICAL_HOST AC_CONFIG_AUX_DIR(.) AC_CONFIG_HEADERS(config.h) m4_ifdef([AM_SILENT_RULES],[AM_SILENT_RULES([no])]) AM_INIT_AUTOMAKE(1.11.1 foreign TESTS_OPTION) AM_PROG_CC_C_O PKG_CHECK_MODULES(glib, [glib-2.0]) PKG_CHECK_MODULES(libxml, [libxml-2.0]) PKG_CHECK_MODULES(cmap, [libcmap], HAVE_cmap=1, HAVE_cmap=0) PKG_CHECK_MODULES(votequorum, [libvotequorum], HAVE_votequorum=1, HAVE_votequorum=0) dnl pacemaker > 1.1.8 PKG_CHECK_MODULES(pacemaker, [pacemaker, pacemaker-cib], HAVE_pacemaker=1, HAVE_pacemaker=0) dnl pacemaker <= 1.1.8 PKG_CHECK_MODULES(pcmk, [pcmk, pcmk-cib], HAVE_pcmk=1, HAVE_pcmk=0) PKG_CHECK_MODULES(libqb, [libqb]) CPPFLAGS="$CPPFLAGS -Werror $glib_CFLAGS $libxml_CFLAGS" LIBS="$LIBS $glib_LIBS $libxml_LIBS" if test $HAVE_pacemaker = 0 -a $HAVE_pcmk = 0; then AC_MSG_ERROR(No package 'pacemaker' found) elif test $HAVE_pacemaker = 1; then CPPFLAGS="$CPPFLAGS $glib_CFLAGS $pacemaker_CFLAGS" if test $HAVE_cmap = 0; then AC_MSG_NOTICE(No library 'cmap' found) else CPPFLAGS="$CPPFLAGS $cmap_CFLAGS" LIBS="$LIBS $cmap_LIBS" fi if test $HAVE_votequorum = 0; then AC_MSG_NOTICE(No library 'votequorum' found) else CPPFLAGS="$CPPFLAGS $votequorum_CFLAGS" LIBS="$LIBS $votequorum_LIBS" fi fi CPPFLAGS="$CPPFLAGS $libqb_CFLAGS $pacemaker_CFLAGS $pcmk_CFLAGS" LIBS="$LIBS $libqb_LIBS $pacemaker_LIBS $pcmk_LIBS" dnl checks for libraries AC_CHECK_LIB(c, dlopen) dnl if dlopen is in libc... AC_CHECK_LIB(dl, dlopen) dnl -ldl (for Linux) AC_CHECK_LIB(aio, io_setup, , missing="yes") AC_CHECK_LIB(qb, qb_ipcs_connection_auth_set, , missing="yes") AC_CHECK_LIB(cib, cib_new, , missing="yes") AC_CHECK_LIB(crmcommon, set_crm_log_level, , missing="yes") AC_CHECK_LIB(pe_status, pe_find_node, , missing="yes") AC_CHECK_LIB(pe_rules, test_rule, , missing="yes") AC_CHECK_LIB(crmcluster, crm_peer_init, , missing="yes") AC_CHECK_LIB(uuid, uuid_unparse, , missing="yes") AC_CHECK_LIB(cmap, cmap_initialize, , HAVE_cmap=0) AC_CHECK_LIB(votequorum, votequorum_getinfo, , HAVE_votequorum=0) dnl pacemaker >= 1.1.8 AC_CHECK_HEADERS(crm/cluster.h) AC_CHECK_LIB(crmcommon, pcmk_strerror, , missing="yes") AC_CHECK_LIB(cib, cib_apply_patch_event, , missing="yes") dnl pacemaker-2.0 removed support for corosync 1 cluster layer AC_CHECK_DECLS([pcmk_cluster_classic_ais, pcmk_cluster_cman],,, [#include ]) dnl check for additional no-quorum-policies dnl AC_TEST_NO_QUORUM_POLICY(POLICY) AC_DEFUN([AC_TEST_NO_QUORUM_POLICY],[ AC_MSG_CHECKING([whether enum pe_quorum_policy defines value $1]) AC_LANG_PUSH([C]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM( [#include ], [enum pe_quorum_policy policy = $1; return policy;])], AC_DEFINE_UNQUOTED(m4_toupper(HAVE_ENUM_$1), 1, [Does pe_types.h have $1 value in enum pe_quorum_policy?]) AC_MSG_RESULT([yes]), AC_MSG_RESULT([no])) AC_LANG_POP([C]) ]) AC_TEST_NO_QUORUM_POLICY(no_quorum_demote) dnl check for new pe-API AC_CHECK_FUNCS(pe_new_working_set) +dnl check if votequorum comes with default for qdevice-sync_timeout +AC_CHECK_DECLS([VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT], + HAVE_DECL_VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT=1, + HAVE_DECL_VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT=0, + [#include ]) + if test "$missing" = "yes"; then AC_MSG_ERROR([Missing required libraries or functions.]) fi AC_PATH_PROGS(POD2MAN, pod2man, pod2man) AC_ARG_ENABLE([shared-disk], [ --enable-shared-disk Turn on functionality that requires shared disk [default=yes]]) DISK=0 if test "x${enable_shared_disk}" != xno ; then DISK=1 fi AC_DEFINE_UNQUOTED(SUPPORT_SHARED_DISK, $DISK, Turn on functionality that requires shared disk) AM_CONDITIONAL(SUPPORT_SHARED_DISK, test "$DISK" = "1") if test -e /proc/$$ then echo "/proc/{pid} is supported" AC_DEFINE_UNQUOTED(HAVE_PROC_PID, 1, Define to 1 if /proc/{pid} is supported.) fi AC_DEFINE_UNQUOTED(CHECK_TWO_NODE, $HAVE_cmap, Turn on checking for 2-node cluster) AM_CONDITIONAL(CHECK_TWO_NODE, test "$HAVE_cmap" = "1") AC_DEFINE_UNQUOTED(CHECK_VOTEQUORUM_HANDLE, $HAVE_votequorum, Turn on periodic checking of votequorum-handle) AM_CONDITIONAL(CHECK_VOTEQUORUM_HANDLE, test "$HAVE_votequorum" = "1") +AC_DEFINE_UNQUOTED(CHECK_QDEVICE_SYNC_TIMEOUT, + ($HAVE_DECL_VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT && $HAVE_cmap), + Turn on checking if watchdog-timeout and qdevice-sync_timeout are matching) +AM_CONDITIONAL(CHECK_QDEVICE_SYNC_TIMEOUT, + test "$HAVE_DECL_VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT" = "1" && + test "$HAVE_cmap" = "1") + CONFIGDIR="" AC_ARG_WITH(configdir, [ --with-configdir=DIR Directory for SBD configuration file [${CONFIGDIR}]], [ CONFIGDIR="$withval" ] ) # # Where is dlopen? # if test "$ac_cv_lib_c_dlopen" = yes; then LIBADD_DL="" elif test "$ac_cv_lib_dl_dlopen" = yes; then LIBADD_DL=-ldl else LIBADD_DL=${lt_cv_dlopen_libs} fi dnl ********************************************************************** dnl Check for various argv[] replacing functions on various OSs dnl dnl Borrowed from Proftpd dnl Proftpd is Licenced under the terms of the GNU General Public Licence dnl and is available from http://www.proftpd.org/ dnl AC_CHECK_FUNCS(setproctitle) AC_CHECK_HEADERS(libutil.h) AC_CHECK_LIB(util, setproctitle, [AC_DEFINE(HAVE_SETPROCTITLE,1,[ ]) ac_cv_func_setproctitle="yes" ; LIBS="$LIBS -lutil"]) if test "$ac_cv_func_setproctitle" = "yes"; then pf_argv_set="PF_ARGV_NONE" fi if test "$pf_argv_set" = ""; then AC_CHECK_HEADERS(sys/pstat.h) if test "$ac_cv_header_pstat_h" = "yes"; then AC_CHECK_FUNCS(pstat) if test "$ac_cv_func_pstat" = "yes"; then pf_argv_set="PF_ARGV_PSTAT" else pf_argv_set="PF_ARGV_WRITEABLE" fi fi if test "$pf_argv_set" = ""; then AC_EGREP_HEADER([#define.*PS_STRINGS.*],sys/exec.h, have_psstrings="yes",have_psstrings="no") if test "$have_psstrings" = "yes"; then pf_argv_set="PF_ARGV_PSSTRINGS" fi fi if test "$pf_argv_set" = ""; then AC_CACHE_CHECK(whether __progname and __progname_full are available, pf_cv_var_progname, AC_TRY_LINK([extern char *__progname, *__progname_full;], [__progname = "foo"; __progname_full = "foo bar";], pf_cv_var_progname="yes", pf_cv_var_progname="no")) if test "$pf_cv_var_progname" = "yes"; then AC_DEFINE(HAVE___PROGNAME,1,[ ]) fi AC_CACHE_CHECK(which argv replacement method to use, pf_cv_argv_type, AC_EGREP_CPP(yes,[ #if defined(__GNU_HURD__) yes #endif ],pf_cv_argv_type="new", pf_cv_argv_type="writeable")) if test "$pf_cv_argv_type" = "new"; then pf_argv_set="PF_ARGV_NEW" fi if test "$pf_argv_set" = ""; then pf_argv_set="PF_ARGV_WRITEABLE" fi fi fi AC_DEFINE_UNQUOTED(PF_ARGV_TYPE, $pf_argv_set, mechanism to pretty-print ps output: setproctitle-equivalent) dnl End of tests borrowed from Proftpd AC_MSG_NOTICE(Sanitizing prefix: ${prefix}) case $prefix in NONE) prefix=/usr dnl Fix default variables - "prefix" variable if not specified if test "$localstatedir" = "\${prefix}/var"; then localstatedir="/var" fi if test "$sysconfdir" = "\${prefix}/etc"; then sysconfdir="/etc" fi ;; esac AC_MSG_NOTICE(Sanitizing exec_prefix: ${exec_prefix}) case $exec_prefix in dnl For consistency with Heartbeat, map NONE->$prefix NONE) exec_prefix=$prefix;; prefix) exec_prefix=$prefix;; esac dnl Expand autoconf variables so that we dont end up with '${prefix}' dnl in #defines and python scripts dnl NOTE: Autoconf deliberately leaves them unexpanded to allow dnl make exec_prefix=/foo install dnl No longer being able to do this seems like no great loss to me... eval prefix="`eval echo ${prefix}`" eval exec_prefix="`eval echo ${exec_prefix}`" eval bindir="`eval echo ${bindir}`" eval sbindir="`eval echo ${sbindir}`" eval libexecdir="`eval echo ${libexecdir}`" eval datadir="`eval echo ${datadir}`" eval sysconfdir="`eval echo ${sysconfdir}`" eval sharedstatedir="`eval echo ${sharedstatedir}`" eval localstatedir="`eval echo ${localstatedir}`" eval libdir="`eval echo ${libdir}`" eval includedir="`eval echo ${includedir}`" eval oldincludedir="`eval echo ${oldincludedir}`" eval infodir="`eval echo ${infodir}`" eval mandir="`eval echo ${mandir}`" AC_SUBST(LIBADD_DL) dnl extra flags for dynamic linking libraries if test x"${CONFIGDIR}" = x""; then CONFIGDIR="${sysconfdir}/sysconfig" fi AC_SUBST(CONFIGDIR) dnl The Makefiles and shell scripts we output AC_CONFIG_FILES([Makefile src/Makefile agent/Makefile man/Makefile agent/sbd src/sbd.service src/sbd_remote.service src/sbd.sh]) AC_CONFIG_SUBDIRS([tests]) dnl Now process the entire list of files added by previous dnl calls to AC_CONFIG_FILES() AC_OUTPUT() diff --git a/src/sbd-cluster.c b/src/sbd-cluster.c index 13fa580..b6c5512 100644 --- a/src/sbd-cluster.c +++ b/src/sbd-cluster.c @@ -1,602 +1,770 @@ /* * Copyright (C) 2013 Lars Marowsky-Bree * * Based on crm_mon.c, which was: * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License along * with this program; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #include #include #include #include #include #include #include #include #include #include #include -#if CHECK_TWO_NODE +#if CHECK_TWO_NODE || CHECK_QDEVICE_SYNC_TIMEOUT #include #endif #include "sbd.h" //undef SUPPORT_PLUGIN //define SUPPORT_PLUGIN 1 /* binary for pacemaker-remote has changed with pacemaker 2 */ #ifdef CRM_SCORE_INFINITY #define PACEMAKER_REMOTE_BINARY "pacemaker-remoted" #else #define PACEMAKER_REMOTE_BINARY "pacemaker_remoted" #endif static bool remote_node = false; static pid_t remoted_pid = 0; static int reconnect_msec = 1000; static GMainLoop *mainloop = NULL; static guint notify_timer = 0; static crm_cluster_t cluster; static gboolean sbd_remote_check(gpointer user_data); static long unsigned int find_pacemaker_remote(void); static void sbd_membership_destroy(gpointer user_data); #if SUPPORT_PLUGIN static void sbd_plugin_membership_dispatch(cpg_handle_t handle, const struct cpg_name *groupName, uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len) { if(msg_len > 0) { set_servant_health(pcmk_health_online, LOG_INFO, "Connected to %s", name_for_cluster_type(get_cluster_type())); } else { set_servant_health(pcmk_health_unclean, LOG_WARNING, "Broken %s message", name_for_cluster_type(get_cluster_type())); } notify_parent(); return; } #endif #if SUPPORT_COROSYNC #if CHECK_VOTEQUORUM_HANDLE #include static votequorum_handle_t votequorum_handle = 0; #endif +#if CHECK_TWO_NODE static bool two_node = false; +#endif static bool ever_seen_both = false; static int cpg_membership_entries = -1; -#if CHECK_TWO_NODE +#if CHECK_QDEVICE_SYNC_TIMEOUT +#include +static bool using_qdevice = false; +static uint32_t qdevice_sync_timeout = /* in seconds */ + VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT / 1000; +#endif + +#if CHECK_TWO_NODE || CHECK_QDEVICE_SYNC_TIMEOUT #include static cmap_handle_t cmap_handle = 0; static cmap_track_handle_t track_handle = 0; static GSource *cmap_source = NULL; #endif void sbd_cpg_membership_health_update() { if(cpg_membership_entries > 0) { - bool quorum_is_suspect = +#if CHECK_TWO_NODE + bool quorum_is_suspect_two_node = (two_node && ever_seen_both && cpg_membership_entries == 1); +#endif +#if CHECK_QDEVICE_SYNC_TIMEOUT + bool quorum_is_suspect_qdevice_timing = + using_qdevice && (qdevice_sync_timeout > timeout_watchdog); +#endif - if (!quorum_is_suspect) { + do { +#if CHECK_TWO_NODE + if (quorum_is_suspect_two_node) { + /* Alternative would be asking votequorum for number of votes. + * Using pacemaker's cpg as source for number of active nodes + * avoids binding to an additional library, is definitely + * less code to write and we wouldn't have to combine data + * from 3 sources (cmap, cpg & votequorum) in a potentially + * racy environment. + */ + set_servant_health(pcmk_health_noquorum, LOG_WARNING, + "Connected to %s but requires both nodes present", + name_for_cluster_type(get_cluster_type()) + ); + break; + } +#endif +#if CHECK_QDEVICE_SYNC_TIMEOUT + if (quorum_is_suspect_qdevice_timing) { + /* We can't really trust quorum info as qdevice-sync_timeout + * makes reaction of quorum too sluggish for our + * watchdog-timeout. + */ + set_servant_health(pcmk_health_noquorum, LOG_WARNING, + "Connected to %s but quorum using qdevice is distrusted " + "for SBD as qdevice-sync_timeout (%ds) > watchdog-timeout " + "(%lus).", + name_for_cluster_type(get_cluster_type()), + qdevice_sync_timeout, timeout_watchdog + ); + break; + } +#endif set_servant_health(pcmk_health_online, LOG_INFO, - "Connected to %s (%u members)", - name_for_cluster_type(get_cluster_type()), - cpg_membership_entries - ); - } else { - /* Alternative would be asking votequorum for number of votes. - * Using pacemaker's cpg as source for number of active nodes - * avoids binding to an additional library, is definitely - * less code to write and we wouldn't have to combine data - * from 3 sources (cmap, cpq & votequorum) in a potentially - * racy environment. - */ - set_servant_health(pcmk_health_noquorum, LOG_WARNING, - "Connected to %s but requires both nodes present", - name_for_cluster_type(get_cluster_type()) - ); - } + "Connected to %s (%u members)%s", + name_for_cluster_type(get_cluster_type()), + cpg_membership_entries, +#if CHECK_QDEVICE_SYNC_TIMEOUT + using_qdevice?" using qdevice for quorum":"" +#else + "" +#endif + ); + } while (false); if (cpg_membership_entries > 1) { ever_seen_both = true; } } else { set_servant_health(pcmk_health_unclean, LOG_WARNING, "Empty %s membership", name_for_cluster_type(get_cluster_type())); } } void sbd_cpg_membership_dispatch(cpg_handle_t handle, const struct cpg_name *groupName, const struct cpg_address *member_list, size_t member_list_entries, const struct cpg_address *left_list, size_t left_list_entries, const struct cpg_address *joined_list, size_t joined_list_entries) { cpg_membership_entries = member_list_entries; sbd_cpg_membership_health_update(); notify_parent(); } -#if CHECK_TWO_NODE +#if CHECK_TWO_NODE || CHECK_QDEVICE_SYNC_TIMEOUT static void sbd_cmap_notify_fn( cmap_handle_t cmap_handle, cmap_track_handle_t cmap_track_handle, int32_t event, const char *key_name, struct cmap_notify_value new_val, struct cmap_notify_value old_val, void *user_data) { - if (new_val.type == CMAP_VALUETYPE_UINT8) { - switch (event) { - case CMAP_TRACK_ADD: - case CMAP_TRACK_MODIFY: - two_node = *((uint8_t *) new_val.data); - break; - case CMAP_TRACK_DELETE: - two_node = false; - break; - default: - return; - } - sbd_cpg_membership_health_update(); - notify_parent(); + switch (event) { + case CMAP_TRACK_ADD: + case CMAP_TRACK_MODIFY: + switch (new_val.type) { + case CMAP_VALUETYPE_UINT8: +#if CHECK_TWO_NODE + if (!strcmp(key_name, "quorum.two_node")) { + two_node = *((uint8_t *) new_val.data); + } else { + return; + } + break; +#else + return; +#endif + case CMAP_VALUETYPE_STRING: +#if CHECK_QDEVICE_SYNC_TIMEOUT + if (!strcmp(key_name, "quorum.device.model")) { + using_qdevice = + ((new_val.data) && strlen((char *) new_val.data)); + } else { + return; + } + break; +#else + return; +#endif + case CMAP_VALUETYPE_UINT32: +#if CHECK_QDEVICE_SYNC_TIMEOUT + if (!strcmp(key_name, "quorum.device.sync_timeout")) { + if (new_val.data) { + qdevice_sync_timeout = + *((uint32_t *) new_val.data) / 1000; + } else { + qdevice_sync_timeout = + VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT / 1000; + } + } else { + return; + } + break; +#else + return; +#endif + default: + return; + } + break; + case CMAP_TRACK_DELETE: + switch (new_val.type) { + case CMAP_VALUETYPE_UINT8: +#if CHECK_TWO_NODE + if (!strcmp(key_name, "quorum.two_node")) { + two_node = false; + } else { + return; + } + break; +#else + return; +#endif + case CMAP_VALUETYPE_STRING: +#if CHECK_QDEVICE_SYNC_TIMEOUT + if (!strcmp(key_name, "quorum.device.model")) { + using_qdevice = false; + } else { + return; + } + break; +#else + return; +#endif + case CMAP_VALUETYPE_UINT32: +#if CHECK_QDEVICE_SYNC_TIMEOUT + if (!strcmp(key_name, "quorum.device.sync_timeout")) { + qdevice_sync_timeout = + VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT / 1000; + } else { + return; + } + break; +#else + return; +#endif + default: + return; + } + break; + default: + return; } + sbd_cpg_membership_health_update(); + notify_parent(); } static gboolean cmap_dispatch_callback (gpointer user_data) { cmap_dispatch(cmap_handle, CS_DISPATCH_ALL); return TRUE; } static void cmap_destroy(void) { if (cmap_source) { g_source_destroy(cmap_source); cmap_source = NULL; } if (track_handle) { cmap_track_delete(cmap_handle, track_handle); track_handle = 0; } if (cmap_handle) { cmap_finalize(cmap_handle); cmap_handle = 0; } } static gboolean -sbd_get_two_node(void) +verify_against_cmap_config(void) { +#if CHECK_TWO_NODE uint8_t two_node_u8 = 0; +#endif +#if CHECK_QDEVICE_SYNC_TIMEOUT + char *qdevice_model = NULL; +#endif int cmap_fd; if (!track_handle) { if (cmap_initialize(&cmap_handle) != CS_OK) { cl_log(LOG_WARNING, "Cannot initialize CMAP service\n"); goto out; } +#if CHECK_TWO_NODE if (cmap_track_add(cmap_handle, "quorum.two_node", CMAP_TRACK_DELETE|CMAP_TRACK_MODIFY|CMAP_TRACK_ADD, sbd_cmap_notify_fn, NULL, &track_handle) != CS_OK) { cl_log(LOG_WARNING, "Failed adding CMAP tracker for 2Node-mode\n"); goto out; } +#endif + +#if CHECK_QDEVICE_SYNC_TIMEOUT + if (cmap_track_add(cmap_handle, "quorum.device.model", + CMAP_TRACK_DELETE|CMAP_TRACK_MODIFY|CMAP_TRACK_ADD, + sbd_cmap_notify_fn, NULL, &track_handle) != CS_OK) { + cl_log(LOG_WARNING, "Failed adding CMAP tracker for qdevice-model\n"); + goto out; + } + + if (cmap_track_add(cmap_handle, "quorum.device.sync_timeout", + CMAP_TRACK_DELETE|CMAP_TRACK_MODIFY|CMAP_TRACK_ADD, + sbd_cmap_notify_fn, NULL, &track_handle) != CS_OK) { + cl_log(LOG_WARNING, + "Failed adding CMAP tracker for qdevice-sync_timeout\n"); + goto out; + } +#endif /* add the tracker to mainloop */ if (cmap_fd_get(cmap_handle, &cmap_fd) != CS_OK) { cl_log(LOG_WARNING, "Failed to get a file handle for cmap\n"); goto out; } if (!(cmap_source = g_unix_fd_source_new (cmap_fd, G_IO_IN))) { cl_log(LOG_WARNING, "Couldn't create source for cmap\n"); goto out; } g_source_set_callback(cmap_source, cmap_dispatch_callback, NULL, NULL); g_source_attach(cmap_source, NULL); } - if (cmap_get_uint8(cmap_handle, "quorum.two_node", &two_node_u8) == CS_OK) { +#if CHECK_TWO_NODE + if (cmap_get_uint8(cmap_handle, "quorum.two_node", &two_node_u8) + == CS_OK) { cl_log(two_node_u8? LOG_NOTICE : LOG_INFO, "Corosync is%s in 2Node-mode", two_node_u8?"":" not"); two_node = two_node_u8; } else { cl_log(LOG_INFO, "quorum.two_node not present in cmap\n"); } +#endif + +#if CHECK_QDEVICE_SYNC_TIMEOUT + if (cmap_get_string(cmap_handle, "quorum.device.model", + &qdevice_model) == CS_OK) { + using_qdevice = qdevice_model && strlen(qdevice_model); + cl_log(using_qdevice? LOG_NOTICE : LOG_INFO, + "Corosync is%s using qdevice", using_qdevice?"":" not"); + } else { + cl_log(LOG_INFO, "quorum.device.model not present in cmap\n"); + } + + if (cmap_get_uint32(cmap_handle, "quorum.device.sync_timeout", + &qdevice_sync_timeout) == CS_OK) { + qdevice_sync_timeout /= 1000; + cl_log(LOG_INFO, + "Corosync is using qdevice-sync_timeout=%ds", + qdevice_sync_timeout); + } else { + cl_log(LOG_INFO, + "quorum.device.sync_timeout not present in cmap\n"); + } +#endif + return TRUE; out: cmap_destroy(); return FALSE; } #endif #endif static gboolean notify_timer_cb(gpointer data) { cl_log(LOG_DEBUG, "Refreshing %sstate", remote_node?"remote ":""); if(remote_node) { sbd_remote_check(NULL); return TRUE; } switch (get_cluster_type()) { #if HAVE_DECL_PCMK_CLUSTER_CLASSIC_AIS case pcmk_cluster_classic_ais: send_cluster_text(crm_class_quorum, NULL, TRUE, NULL, crm_msg_ais); break; #endif case pcmk_cluster_corosync: do { #if SUPPORT_COROSYNC && CHECK_VOTEQUORUM_HANDLE struct votequorum_info info; if (votequorum_getinfo(votequorum_handle, 0, &info) != CS_OK) { votequorum_finalize(votequorum_handle); if (votequorum_initialize(&votequorum_handle, NULL) != CS_OK) { votequorum_handle = 0; break; } if (votequorum_getinfo(votequorum_handle, 0, &info) != CS_OK) { break; } } #endif notify_parent(); } while (0); break; #if HAVE_DECL_PCMK_CLUSTER_CMAN case pcmk_cluster_cman: notify_parent(); break; #endif default: break; } return TRUE; } static void sbd_membership_connect(void) { bool connected = false; cl_log(LOG_INFO, "Attempting cluster connection"); cluster.destroy = sbd_membership_destroy; #if SUPPORT_PLUGIN cluster.cpg.cpg_deliver_fn = sbd_plugin_membership_dispatch; #endif #if SUPPORT_COROSYNC cluster.cpg.cpg_confchg_fn = sbd_cpg_membership_dispatch; #endif while(connected == false) { enum cluster_type_e stack = get_cluster_type(); if(get_cluster_type() == pcmk_cluster_unknown) { crm_debug("Attempting pacemaker remote connection"); /* Nothing is up, go looking for the pacemaker remote process */ if(find_pacemaker_remote() > 0) { connected = true; } } else { cl_log(LOG_INFO, "Attempting connection to %s", name_for_cluster_type(stack)); -#if SUPPORT_COROSYNC && CHECK_TWO_NODE - if (sbd_get_two_node()) { +#if SUPPORT_COROSYNC && (CHECK_TWO_NODE || CHECK_QDEVICE_SYNC_TIMEOUT) + if (verify_against_cmap_config()) { #endif if(crm_cluster_connect(&cluster)) { connected = true; } -#if SUPPORT_COROSYNC && CHECK_TWO_NODE +#if SUPPORT_COROSYNC && (CHECK_TWO_NODE || CHECK_QDEVICE_SYNC_TIMEOUT) } #endif } if(connected == false) { cl_log(LOG_INFO, "Failed, retrying in %ds", reconnect_msec / 1000); sleep(reconnect_msec / 1000); } } set_servant_health(pcmk_health_transient, LOG_INFO, "Connected, waiting for initial membership"); notify_parent(); notify_timer_cb(NULL); } static void sbd_membership_destroy(gpointer user_data) { cl_log(LOG_WARNING, "Lost connection to %s", name_for_cluster_type(get_cluster_type())); if (get_cluster_type() != pcmk_cluster_unknown) { -#if SUPPORT_COROSYNC && CHECK_TWO_NODE +#if SUPPORT_COROSYNC && (CHECK_TWO_NODE || CHECK_QDEVICE_SYNC_TIMEOUT) cmap_destroy(); #endif } set_servant_health(pcmk_health_unclean, LOG_ERR, "Cluster connection terminated"); notify_parent(); /* Attempt to reconnect, the watchdog will take the node down if the problem isn't transient */ sbd_membership_connect(); } /* * \internal * \brief Get process ID and name associated with a /proc directory entry * * \param[in] entry Directory entry (must be result of readdir() on /proc) * \param[out] name If not NULL, a char[16] to hold the process name * \param[out] pid If not NULL, will be set to process ID of entry * * \return 0 on success, -1 if entry is not for a process or info not found * * \note This should be called only on Linux systems, as not all systems that * support /proc store process names and IDs in the same way. * Copied from the Pacemaker implementation. */ int sbd_procfs_process_info(struct dirent *entry, char *name, int *pid) { int fd, local_pid; FILE *file; struct stat statbuf; char procpath[128] = { 0 }; /* We're only interested in entries whose name is a PID, * so skip anything non-numeric or that is too long. * * 114 = 128 - strlen("/proc/") - strlen("/status") - 1 */ local_pid = atoi(entry->d_name); if ((local_pid <= 0) || (strlen(entry->d_name) > 114)) { return -1; } if (pid) { *pid = local_pid; } /* Get this entry's file information */ strcpy(procpath, "/proc/"); strcat(procpath, entry->d_name); fd = open(procpath, O_RDONLY); if (fd < 0 ) { return -1; } if (fstat(fd, &statbuf) < 0) { close(fd); return -1; } close(fd); /* We're only interested in subdirectories */ if (!S_ISDIR(statbuf.st_mode)) { return -1; } /* Read the first entry ("Name:") from the process's status file. * We could handle the valgrind case if we parsed the cmdline file * instead, but that's more of a pain than it's worth. */ if (name != NULL) { strcat(procpath, "/status"); file = fopen(procpath, "r"); if (!file) { return -1; } if (fscanf(file, "Name:\t%15[a-zA-Z0-9 _-]", name) != 1) { fclose(file); return -1; } fclose(file); } return 0; } static gboolean sbd_remote_check(gpointer user_data) { static int have_proc_pid = 0; int running = 0; cl_log(LOG_DEBUG, "Checking pacemaker remote connection: %d/%d", have_proc_pid, remoted_pid); if(have_proc_pid == 0) { char proc_path[PATH_MAX], exe_path[PATH_MAX]; /* check to make sure pid hasn't been reused by another process */ snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", (long unsigned int)getpid()); have_proc_pid = 1; if(readlink(proc_path, exe_path, PATH_MAX - 1) < 0) { have_proc_pid = -1; } } if (remoted_pid <= 0) { set_servant_health(pcmk_health_transient, LOG_WARNING, "No Pacemaker Remote connection"); goto notify; } else if (kill(remoted_pid, 0) < 0 && errno == ESRCH) { /* Not running */ } else if(have_proc_pid == -1) { running = 1; cl_log(LOG_DEBUG, "Poccess %ld is active", (long)remoted_pid); } else { int rc = 0; char proc_path[PATH_MAX], exe_path[PATH_MAX]; /* check to make sure pid hasn't been reused by another process */ snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", (long unsigned int)remoted_pid); rc = readlink(proc_path, exe_path, PATH_MAX - 1); if (rc < 0) { crm_perror(LOG_ERR, "Could not read from %s", proc_path); goto done; } exe_path[rc] = 0; if (strcmp(exe_path, SBINDIR "/" PACEMAKER_REMOTE_BINARY) == 0) { cl_log(LOG_DEBUG, "Process %s (%ld) is active", exe_path, (long)remoted_pid); running = 1; } } done: if(running) { set_servant_health(pcmk_health_online, LOG_INFO, "Connected to Pacemaker Remote %lu", (long unsigned int)remoted_pid); } else { set_servant_health(pcmk_health_unclean, LOG_WARNING, "Connection to Pacemaker Remote %lu lost", (long unsigned int)remoted_pid); } notify: notify_parent(); if(running == 0) { sbd_membership_connect(); } return true; } static long unsigned int find_pacemaker_remote(void) { DIR *dp; char entry_name[16]; struct dirent *entry; dp = opendir("/proc"); if (!dp) { /* no proc directory to search through */ cl_log(LOG_NOTICE, "Can not read /proc directory to track existing components"); return FALSE; } while ((entry = readdir(dp)) != NULL) { int pid; if (sbd_procfs_process_info(entry, entry_name, &pid) < 0) { continue; } /* entry_name is truncated to 16 characters including the nul terminator */ cl_log(LOG_DEBUG, "Found %s at %u", entry_name, pid); if (strncmp(entry_name, PACEMAKER_REMOTE_BINARY, 15) == 0) { cl_log(LOG_NOTICE, "Found Pacemaker Remote at PID %u", pid); remoted_pid = pid; remote_node = true; break; } } closedir(dp); return remoted_pid; } static void clean_up(int rc) { #if CHECK_VOTEQUORUM_HANDLE votequorum_finalize(votequorum_handle); votequorum_handle = 0; /* there isn't really an invalid handle value * just to be back where we started */ #endif return; } static void cluster_shutdown(int nsig) { clean_up(0); } int servant_cluster(const char *diskname, int mode, const void* argp) { enum cluster_type_e cluster_stack = get_cluster_type(); crm_system_name = strdup("sbd:cluster"); cl_log(LOG_NOTICE, "Monitoring %s cluster health", name_for_cluster_type(cluster_stack)); set_proc_title("sbd: watcher: Cluster"); sbd_membership_connect(); /* stonith_our_uname = cluster.uname; */ /* stonith_our_uuid = cluster.uuid; */ mainloop = g_main_loop_new(NULL, FALSE); notify_timer = g_timeout_add(timeout_loop * 1000, notify_timer_cb, NULL); mainloop_add_signal(SIGTERM, cluster_shutdown); mainloop_add_signal(SIGINT, cluster_shutdown); g_main_loop_run(mainloop); g_main_loop_unref(mainloop); clean_up(0); return 0; /* never reached */ }