diff --git a/configure.ac b/configure.ac
index 4b20de181c..4a2fc14e97 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,2155 +1,2157 @@
 dnl
 dnl autoconf for Pacemaker
 dnl
 dnl Copyright 2009-2021 the Pacemaker project contributors
 dnl
 dnl The version control history for this file may have further details.
 dnl
 dnl This source code is licensed under the GNU General Public License version 2
 dnl or later (GPLv2+) WITHOUT ANY WARRANTY.
 
 dnl ===============================================
 dnl Bootstrap
 dnl ===============================================
 AC_PREREQ(2.64)
 
 dnl AC_CONFIG_MACRO_DIR is deprecated as of autoconf 2.70 (2020-12-08).
 dnl Once we can require that version, we can simplify this, and no longer
 dnl need ACLOCAL_AMFLAGS in Makefile.am.
 m4_ifdef([AC_CONFIG_MACRO_DIRS],
          [AC_CONFIG_MACRO_DIRS([m4])],
          [AC_CONFIG_MACRO_DIR([m4])])
 
 AC_DEFUN([AC_DATAROOTDIR_CHECKED])
 
 dnl Suggested structure:
 dnl     information on the package
 dnl     checks for programs
 dnl     checks for libraries
 dnl     checks for header files
 dnl     checks for types
 dnl     checks for structures
 dnl     checks for compiler characteristics
 dnl     checks for library functions
 dnl     checks for system services
 
 m4_include([m4/version.m4])
 AC_INIT([pacemaker], VERSION_NUMBER, [users@clusterlabs.org], [pacemaker],
         PCMK_URL)
 
 PCMK_FEATURES=""
 
 LT_CONFIG_LTDL_DIR([libltdl])
 AC_CONFIG_AUX_DIR([libltdl/config])
 AC_CANONICAL_HOST
 
 dnl Where #defines that autoconf makes (e.g. HAVE_whatever) go
 dnl
 dnl Internal header: include/config.h
 dnl   - Contains ALL defines
 dnl   - include/config.h.in is generated automatically by autoheader
 dnl   - NOT to be included in any header files except crm_internal.h
 dnl     (which is also not to be included in any other header files)
 dnl
 dnl External header: include/crm_config.h
 dnl   - Contains a subset of defines checked here
 dnl   - Manually edit include/crm_config.h.in to have configure include
 dnl     new defines
 dnl   - Should not include HAVE_* defines
 dnl   - Safe to include anywhere
 AC_CONFIG_HEADERS([include/config.h include/crm_config.h])
 
 dnl 1.13:           minimum automake version required
 dnl foreign:        don't require GNU-standard top-level files
 dnl tar-ustar:      use (older) POSIX variant of generated tar rather than v7
 dnl subdir-objects: keep .o's with their .c's (no-op in 2.0+)
 AM_INIT_AUTOMAKE([1.13 foreign tar-ustar subdir-objects])
 
 dnl Require minimum version of pkg-config
 PKG_PROG_PKG_CONFIG(0.27)
 AS_IF([test "x${PKG_CONFIG}" != x], [],
       [AC_MSG_FAILURE([Could not find required build tool pkg-config (0.27 or later)])])
 PKG_INSTALLDIR
 PKG_NOARCH_INSTALLDIR
 
 dnl Example 2.4. Silent Custom Rule to Generate a File
 dnl %-bar.pc: %.pc
 dnl	$(AM_V_GEN)$(LN_S) $(notdir $^) $@
 
 dnl Versioned attributes implementation is not yet production-ready
 AC_DEFINE_UNQUOTED(ENABLE_VERSIONED_ATTRS, 0, [Enable versioned attributes])
 
 CC_IN_CONFIGURE=yes
 export CC_IN_CONFIGURE
 
 LDD=ldd
 
 dnl ========================================================================
 dnl Compiler characteristics
 dnl ========================================================================
 
 dnl A particular compiler can be forced by setting the CC environment variable
 AC_PROG_CC
 
 dnl Use at least C99 if possible. This will generate an "obsolete" warning
 dnl since autoconf 2.70, but is needed for older versions.
 AC_PROG_CC_STDC
 
 dnl C++ is not needed for build, just maintainer utilities
 AC_PROG_CXX
 
 dnl We use md5.c from gnulib, which has its own m4 macros. Per its docs:
 dnl "The macro gl_EARLY must be called as soon as possible after verifying that
 dnl the C compiler is working. ... The core part of the gnulib checks are done
 dnl by the macro gl_INIT." In addition, prevent gnulib from introducing OpenSSL
 dnl as a dependency.
 gl_EARLY
 gl_SET_CRYPTO_CHECK_DEFAULT([no])
 gl_INIT
 
 # --enable-new-dtags: Use RUNPATH instead of RPATH.
 # It is necessary to have this done before libtool does linker detection.
 # See also: https://github.com/kronosnet/kronosnet/issues/107
 AX_CHECK_LINK_FLAG([-Wl,--enable-new-dtags],
                   [AM_LDFLAGS=-Wl,--enable-new-dtags],
                   [AC_MSG_ERROR(["Linker support for --enable-new-dtags is required"])])
 AC_SUBST([AM_LDFLAGS])
 
 saved_LDFLAGS="$LDFLAGS"
 LDFLAGS="$AM_LDFLAGS $LDFLAGS"
 LT_INIT([dlopen])
 LDFLAGS="$saved_LDFLAGS"
 LTDL_INIT([convenience])
 
 AC_TYPE_SIZE_T
 AC_CHECK_SIZEOF(char)
 AC_CHECK_SIZEOF(short)
 AC_CHECK_SIZEOF(int)
 AC_CHECK_SIZEOF(long)
 AC_CHECK_SIZEOF(long long)
 
 dnl ===============================================
 dnl Helpers
 dnl ===============================================
 cc_supports_flag() {
     local CFLAGS="-Werror $@"
     AC_MSG_CHECKING([whether $CC supports $@])
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ ]])],
                       [RC=0; AC_MSG_RESULT([yes])],
                       [RC=1; AC_MSG_RESULT([no])])
     return $RC
 }
 
 # Some tests need to use their own CFLAGS
 
 cc_temp_flags() {
     ac_save_CFLAGS="$CFLAGS"
     CFLAGS="$*"
 }
 
 cc_restore_flags() {
     CFLAGS=$ac_save_CFLAGS
 }
 
 # expand_path_option $path_variable_name $default
 expand_path_option() {
     # The first argument is the variable *name* (not value)
     ac_path_varname="$1"
 
     # Get the original value of the variable
     ac_path_value=$(eval echo "\${${ac_path_varname}}")
 
     # Expand any literal variable expressions in the value so that we don't
     # end up with something like '${prefix}' in #defines etc.
     #
     # Autoconf deliberately leaves values unexpanded to allow overriding
     # the configure script choices in make commands (for example,
     # "make exec_prefix=/foo install"). No longer being able to do this seems
     # like no great loss.
     eval ac_path_value=$(eval echo "${ac_path_value}")
 
     # Use (expanded) default if necessary
     AS_IF([test x"${ac_path_value}" = x""],
 	  [eval ac_path_value=$(eval echo "$2")])
 
     # Require a full path
     AS_CASE(["$ac_path_value"],
             [/*], [eval ${ac_path_varname}="$ac_path_value"],
             [*], [AC_MSG_ERROR([$ac_path_varname value "$ac_path_value" is not a full path])]
     )
 }
 
 # yes_no_try $user_response $default
 DISABLED=0
 REQUIRED=1
 OPTIONAL=2
 yes_no_try() {
     local value
     AS_IF([test x"$1" = x""], [value="$2"], [value="$1"])
     AS_CASE(["`echo "$value" | tr '[A-Z]' '[a-z]'`"],
             [0|no|false|disable], [return $DISABLED],
             [1|yes|true|enable], [return $REQUIRED],
             [try|check], [return $OPTIONAL]
     )
     AC_MSG_ERROR([Invalid option value "$value"])
 }
 
 check_systemdsystemunitdir() {
     AC_MSG_CHECKING([which system unit file directory to use])
     PKG_CHECK_VAR([systemdsystemunitdir], [systemd], [systemdsystemunitdir])
     AC_MSG_RESULT([${systemdsystemunitdir}])
     test x"$systemdsystemunitdir" != x""
     return $?
 }
 
 dnl ===============================================
 dnl Configure Options
 dnl ===============================================
 
 dnl Actual library checks come later, but pkg-config can be used here to grab
 dnl external values to use as defaults for configure options
 
 dnl --enable-* options: build process
 
 AC_ARG_ENABLE([quiet],
     [AS_HELP_STRING([--enable-quiet],
         [suppress make output unless there is an error @<:@no@:>@])]
 )
 yes_no_try "$enable_quiet" "no"
 enable_quiet=$?
 
 AC_ARG_ENABLE([fatal-warnings],
     [AS_HELP_STRING([--enable-fatal-warnings],
         [enable pedantic and fatal warnings for gcc @<:@try@:>@])],
 )
 yes_no_try "$enable_fatal_warnings" "try"
 enable_fatal_warnings=$?
 
 AC_ARG_ENABLE([hardening],
     [AS_HELP_STRING([--enable-hardening],
         [harden the resulting executables/libraries @<:@try@:>@])]
 )
 yes_no_try "$enable_hardening" "try"
 enable_hardening=$?
 
 dnl --enable-* options: features
 
 AC_ARG_ENABLE([systemd],
     [AS_HELP_STRING([--enable-systemd],
         [enable support for managing resources via systemd @<:@try@:>@])]
 )
 yes_no_try "$enable_systemd" "try"
 enable_systemd=$?
 
 AC_ARG_ENABLE([upstart],
     [AS_HELP_STRING([--enable-upstart],
         [enable support for managing resources via Upstart (deprecated) @<:@try@:>@])]
 )
 yes_no_try "$enable_upstart" "try"
 enable_upstart=$?
 
 dnl --enable-* options: compatibility
 
 AC_ARG_ENABLE([compat-2.0],
     [AS_HELP_STRING([--enable-compat-2.0], m4_normalize([
         preserve certain output as it was in 2.0; this option will be
         available only for the lifetime of the 2.1 series @<:@no@:>@]))]
 )
 yes_no_try "$enable_compat_2_0" "no"
 enable_compat_2_0=$?
 AS_IF([test $enable_compat_2_0 -ne $DISABLED],
       [
           AC_DEFINE_UNQUOTED([PCMK__COMPAT_2_0], [1],
                              [Keep certain output compatible with 2.0 release series])
           PCMK_FEATURES="$PCMK_FEATURES compat-2.0"
       ]
 )
 
 # Add an option to create symlinks at the pre-2.0.0 daemon name locations, so
 # that users and tools can continue to invoke those names directly (e.g., for
 # meta-data). This option will be removed in a future release.
 AC_ARG_ENABLE([legacy-links],
     [AS_HELP_STRING([--enable-legacy-links],
         [add symlinks for old daemon names (deprecated) @<:@no@:>@])]
 )
 yes_no_try "$enable_legacy_links" "no"
 enable_legacy_links=$?
 AM_CONDITIONAL([BUILD_LEGACY_LINKS], [test $enable_legacy_links -ne $DISABLED])
 
 dnl --with-* options: basic parameters
 
 dnl This argument is defined via an M4 macro so default can be a variable
 AC_DEFUN([VERSION_ARG],
     [AC_ARG_WITH([version],
         [AS_HELP_STRING([--with-version=VERSION],
             [override package version @<:@$1@:>@])],
         [ PACEMAKER_VERSION="$withval" ],
         [ PACEMAKER_VERSION="$PACKAGE_VERSION" ])]
 )
 VERSION_ARG(VERSION_NUMBER)
 
 # Redefine PACKAGE_VERSION and VERSION according to PACEMAKER_VERSION in case
 # the user used --with-version. Unfortunately, this can only affect the
 # substitution variables and later uses in this file, not the config.h
 # constants, so we have to be careful to use only PACEMAKER_VERSION in C code.
 PACKAGE_VERSION=$PACEMAKER_VERSION
 VERSION=$PACEMAKER_VERSION
 
 CRM_DAEMON_USER=""
 AC_ARG_WITH([daemon-user],
     [AS_HELP_STRING([--with-daemon-user=USER],
         [user to run unprivileged Pacemaker daemons as (advanced option: changing this may break other cluster components unless similarly configured) @<:@hacluster@:>@])],
     [ CRM_DAEMON_USER="$withval" ]
 )
 
 CRM_DAEMON_GROUP=""
 AC_ARG_WITH([daemon-group],
     [AS_HELP_STRING([--with-daemon-group=GROUP],
         [group to run unprivileged Pacemaker daemons as (advanced option: changing this may break other cluster components unless similarly configured) @<:@haclient@:>@])],
     [ CRM_DAEMON_GROUP="$withval" ]
 )
 
 BUG_URL=""
 AC_ARG_WITH([bug-url],
     [AS_HELP_STRING([--with-bug-url=DIR], m4_normalize([
         address where users should submit bug reports
         @<:@https://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker@:>@]))],
     [ BUG_URL="$withval" ]
 )
 
 dnl --with-* options: features
 
 AC_ARG_WITH([cibsecrets],
     [AS_HELP_STRING([--with-cibsecrets],
         [support separate file for CIB secrets @<:@no@:>@])]
 )
 yes_no_try "$with_cibsecrets" "no"
 with_cibsecrets=$?
 
 AC_ARG_WITH([gnutls],
     [AS_HELP_STRING([--with-gnutls],
         [support Pacemaker Remote and remote-tls-port using GnuTLS @<:@try@:>@])]
 )
 yes_no_try "$with_gnutls" "try"
 with_gnutls=$?
 
 PCMK_GNUTLS_PRIORITIES="NORMAL"
 AC_ARG_WITH([gnutls-priorities],
     [AS_HELP_STRING([--with-gnutls-priorities],
         [default GnuTLS cipher priorities @<:@NORMAL@:>@])],
     [ test x"$withval" = x"no" || PCMK_GNUTLS_PRIORITIES="$withval" ]
 )
 
 AC_ARG_WITH([concurrent-fencing-default],
     [AS_HELP_STRING([--with-concurrent-fencing-default],
         [default value for concurrent-fencing cluster option @<:@false@:>@])],
 )
 AS_CASE([$with_concurrent_fencing_default],
         [""], [with_concurrent_fencing_default="false"],
         [false], [],
         [true], [PCMK_FEATURES="$PCMK_FEATURES default-concurrent-fencing"],
         [AC_MSG_ERROR([Invalid value "$with_concurrent_fencing_default" for --with-concurrent-fencing-default])]
 )
 AC_DEFINE_UNQUOTED([PCMK__CONCURRENT_FENCING_DEFAULT],
                    ["$with_concurrent_fencing_default"],
                    [Default value for concurrent-fencing cluster option])
 
 AC_ARG_WITH([sbd-sync-default],
     [AS_HELP_STRING([--with-sbd-sync-default], m4_normalize([
         default value used by sbd if SBD_SYNC_RESOURCE_STARTUP
         environment variable is not set @<:@false@:>@]))],
 )
 AS_CASE([$with_sbd_sync_default],
         [""], [with_sbd_sync_default=false],
         [false], [],
         [true], [PCMK_FEATURES="$PCMK_FEATURES default-sbd-sync"],
         [AC_MSG_ERROR([Invalid value "$with_sbd_sync_default" for --with-sbd-sync-default])]
 )
 AC_DEFINE_UNQUOTED([PCMK__SBD_SYNC_DEFAULT],
                    [$with_sbd_sync_default],
                    [Default value for SBD_SYNC_RESOURCE_STARTUP environment variable])
 
 AC_ARG_WITH([resource-stickiness-default],
     [AS_HELP_STRING([--with-resource-stickiness-default],
         [If positive, value to add to new CIBs as explicit resource default for resource-stickiness @<:@0@:>@])],
 )
 errmsg="Invalid value \"$with_resource_stickiness_default\" for --with-resource-stickiness-default"
 AS_CASE([$with_resource_stickiness_default],
         [0|""], [with_resource_stickiness_default="0"],
         [*[[!0-9]]*], [AC_MSG_ERROR([$errmsg])],
         [PCMK_FEATURES="$PCMK_FEATURES default-resource-stickiness"]
 )
 AC_DEFINE_UNQUOTED([PCMK__RESOURCE_STICKINESS_DEFAULT],
                    [$with_resource_stickiness_default],
                    [Default value for resource-stickiness resource meta-attribute])
 
 AC_ARG_WITH([corosync],
     [AS_HELP_STRING([--with-corosync],
         [support the Corosync messaging and membership layer @<:@try@:>@])]
 )
 yes_no_try "$with_corosync" "try"
 with_corosync=$?
 
 AC_ARG_WITH([nagios],
     [AS_HELP_STRING([--with-nagios], [support nagios resources])]
 )
 yes_no_try "$with_nagios" "try"
 with_nagios=$?
 
 dnl --with-* options: directory locations
 
 AC_ARG_WITH([nagios-plugin-dir],
     [AS_HELP_STRING([--with-nagios-plugin-dir=DIR],
         [directory for nagios plugins @<:@LIBEXECDIR/nagios/plugins@:>@])],
     [ NAGIOS_PLUGIN_DIR="$withval" ]
 )
 
 AC_ARG_WITH([nagios-metadata-dir],
     [AS_HELP_STRING([--with-nagios-metadata-dir=DIR],
         [directory for nagios plugins metadata @<:@DATADIR/nagios/plugins-metadata@:>@])],
     [ NAGIOS_METADATA_DIR="$withval" ]
 )
 
 INITDIR=""
 AC_ARG_WITH([initdir],
     [AS_HELP_STRING([--with-initdir=DIR],
         [directory for init (rc) scripts])],
     [ INITDIR="$withval" ]
 )
 
 systemdsystemunitdir="${systemdsystemunitdir-}"
 AC_ARG_WITH([systemdsystemunitdir],
     [AS_HELP_STRING([--with-systemdsystemunitdir=DIR],
         [directory for systemd unit files (advanced option: must match what systemd uses)])],
     [ systemdsystemunitdir="$withval" ]
 )
 
 CONFIGDIR=""
 AC_ARG_WITH([configdir],
     [AS_HELP_STRING([--with-configdir=DIR],
         [directory for Pacemaker configuration file @<:@SYSCONFDIR/sysconfig@:>@])],
     [ CONFIGDIR="$withval" ]
 )
 
 dnl --runstatedir is available as of autoconf 2.70 (2020-12-08). When users
 dnl have an older version, they can use our --with-runstatedir.
 pcmk_runstatedir=""
 AC_ARG_WITH([runstatedir],
     [AS_HELP_STRING([--with-runstatedir=DIR],
         [modifiable per-process data @<:@LOCALSTATEDIR/run@:>@ (ignored if --runstatedir is available)])],
     [ pcmk_runstatedir="$withval" ]
 )
 
 CRM_LOG_DIR=""
 AC_ARG_WITH([logdir],
     [AS_HELP_STRING([--with-logdir=DIR],
         [directory for Pacemaker log file @<:@LOCALSTATEDIR/log/pacemaker@:>@])],
     [ CRM_LOG_DIR="$withval" ]
 )
 
 CRM_BUNDLE_DIR=""
 AC_ARG_WITH([bundledir],
     [AS_HELP_STRING([--with-bundledir=DIR],
         [directory for Pacemaker bundle logs @<:@LOCALSTATEDIR/log/pacemaker/bundles@:>@])],
     [ CRM_BUNDLE_DIR="$withval" ]
 )
 
 dnl Get default from resource-agents if possible. Otherwise, the default uses
 dnl /usr/lib rather than libdir because it's determined by the OCF project and
 dnl not Pacemaker. Even if a user wants to install Pacemaker to /usr/local or
 dnl such, the OCF agents will be expected in their usual location. However, we
 dnl do give the user the option to override it.
 PKG_CHECK_VAR([OCF_ROOT_DIR], [resource-agents], [ocfrootdir], [],
               [OCF_ROOT_DIR="/usr/lib/ocf"])
 AC_ARG_WITH([ocfdir],
     [AS_HELP_STRING([--with-ocfdir=DIR], m4_normalize([
         OCF resource agent root directory (advanced option: changing this
         may break other cluster components unless similarly configured)
         @<:@value from resource-agents package if available otherwise
         /usr/lib/ocf@:>@]))],
     [ OCF_ROOT_DIR="$withval" ]
 )
 AC_SUBST(OCF_ROOT_DIR)
 AC_DEFINE_UNQUOTED([OCF_ROOT_DIR], ["$OCF_ROOT_DIR"],
                    [OCF root directory for resource agents and libraries])
 
 PKG_CHECK_VAR([OCF_RA_PATH], [resource-agents], [ocfrapath], [],
               [OCF_RA_PATH="$OCF_ROOT_DIR/resource.d"])
 AC_ARG_WITH([ocfrapath],
     [AS_HELP_STRING([--with-ocfrapath=DIR], m4_normalize([
         OCF resource agent directories (colon-separated) to search
         @<:@value from resource-agents package if available otherwise
         OCFDIR/resource.d@:>@]))],
     [ OCF_RA_PATH="$withval" ]
 )
 AC_SUBST(OCF_RA_PATH)
 
 OCF_RA_INSTALL_DIR="$OCF_ROOT_DIR/resource.d"
 AC_ARG_WITH([ocfrainstalldir],
     [AS_HELP_STRING([--with-ocfrainstalldir=DIR], m4_normalize([
         OCF installation directory for Pacemakers resource agents
         @<:@OCFDIR/resource.d@:>@]))],
     [ OCF_RA_INSTALL_DIR="$withval" ]
 )
 AC_SUBST(OCF_RA_INSTALL_DIR)
 
 dnl Get default from fence-agents if available
 PKG_CHECK_VAR([FA_PREFIX], [fence-agents], [prefix],
               [PCMK__FENCE_BINDIR="${FA_PREFIX}/sbin"],
               [PCMK__FENCE_BINDIR="$sbindir"])
 AC_ARG_WITH([fence-bindir],
     [AS_HELP_STRING([--with-fence-bindir=DIR], m4_normalize([
         directory for executable fence agents @<:@value from fence-agents
         package if available otherwise SBINDIR@:>@]))],
     [ PCMK__FENCE_BINDIR="$withval" ]
 )
 AC_SUBST(PCMK__FENCE_BINDIR)
 
 dnl --with-* options: non-production testing
 
 AC_ARG_WITH([profiling],
     [AS_HELP_STRING([--with-profiling],
         [disable optimizations, for effective profiling @<:@no@:>@])]
 )
 yes_no_try "$with_profiling" "no"
 with_profiling=$?
 
 AC_ARG_WITH([coverage],
     [AS_HELP_STRING([--with-coverage],
         [disable optimizations, for effective profiling and coverage testing @<:@no@:>@])]
 )
 yes_no_try "$with_coverage" "no"
 with_coverage=$?
 
 AC_ARG_WITH([sanitizers],
   [AS_HELP_STRING([--with-sanitizers=...,...],
     [enable SANitizer build, do *NOT* use for production. Only ASAN/UBSAN/TSAN are currently supported])],
   [ SANITIZERS="$withval" ],
   [ SANITIZERS="" ])
 
 
 dnl ===============================================
 dnl General Processing
 dnl ===============================================
 
 AC_DEFINE_UNQUOTED(PACEMAKER_VERSION, "$VERSION",
                    [Version number of this Pacemaker build])
 
 PACKAGE_SERIES=`echo $VERSION | awk -F. '{ print $1"."$2 }'`
 AC_SUBST(PACKAGE_SERIES)
 
 AC_PROG_LN_S
 AC_PROG_MKDIR_P
 
 # Check for fatal warning support
 AS_IF([test $enable_fatal_warnings -ne $DISABLED && test "$GCC" = "yes" && cc_supports_flag -Werror],
       [WERROR="-Werror"],
       [
           WERROR=""
           AS_CASE([$enable_fatal_warnings],
                   [$REQUIRED], [AC_MSG_ERROR([Compiler does not support fatal warnings])],
                   [$OPTIONAL], [
                       AC_MSG_NOTICE([Compiler does not support fatal warnings])
                       enable_fatal_warnings=$DISABLED
                   ])
       ])
 
 AC_MSG_NOTICE([Sanitizing prefix: ${prefix}])
 AS_IF([test "$prefix" = "NONE"],
       [
           prefix=/usr
           dnl Fix default variables - "prefix" variable if not specified
           AS_IF([test "$localstatedir" = "\${prefix}/var"],
                 [localstatedir="/var"])
           AS_IF([test "$sysconfdir" = "\${prefix}/etc"],
                 [sysconfdir="/etc"])
       ])
 
 AC_MSG_NOTICE([Sanitizing exec_prefix: ${exec_prefix}])
 case $exec_prefix in
     prefix|NONE)
         exec_prefix=$prefix
         ;;
 esac
 
 AC_MSG_NOTICE([Sanitizing INITDIR: ${INITDIR}])
 case $INITDIR in
     prefix) INITDIR=$prefix;;
     "")
         AC_MSG_CHECKING([which init (rc) directory to use])
         for initdir in /etc/init.d /etc/rc.d/init.d /sbin/init.d \
             /usr/local/etc/rc.d /etc/rc.d
         do
             if
                 test -d $initdir
             then
                 INITDIR=$initdir
                 break
             fi
         done
         AC_MSG_RESULT([$INITDIR])
         ;;
 esac
 AC_SUBST(INITDIR)
 
 AC_MSG_NOTICE([Sanitizing libdir: ${libdir}])
 case $libdir in
     prefix|NONE)
         AC_MSG_CHECKING([which lib directory to use])
         for aDir in lib64 lib
         do
             trydir="${exec_prefix}/${aDir}"
             if
                 test -d ${trydir}
             then
                 libdir=${trydir}
                 break
             fi
         done
         AC_MSG_RESULT([$libdir]);
         ;;
 esac
 
 dnl Expand values of autoconf-provided directory options
 expand_path_option prefix
 expand_path_option exec_prefix
 expand_path_option bindir
 expand_path_option sbindir
 expand_path_option libexecdir
 expand_path_option datadir
 expand_path_option sysconfdir
 expand_path_option sharedstatedir
 expand_path_option localstatedir
 expand_path_option libdir
 expand_path_option includedir
 expand_path_option oldincludedir
 expand_path_option infodir
 expand_path_option mandir
 
 dnl Home-grown variables
 
 AS_IF([test x"${runstatedir}" = x""], [runstatedir="${pcmk_runstatedir}"])
 expand_path_option runstatedir "${localstatedir}/run"
 AC_DEFINE_UNQUOTED([PCMK_RUN_DIR], ["$runstatedir"],
                    [Location for modifiable per-process data])
 AC_SUBST(runstatedir)
 
 expand_path_option INITDIR
+AC_DEFINE_UNQUOTED([PCMK__LSB_INIT_DIR], ["$INITDIR"],
+                   [Location for LSB init scripts])
 
 expand_path_option docdir "${datadir}/doc/${PACKAGE}-${VERSION}"
 AC_SUBST(docdir)
 
 expand_path_option CONFIGDIR "${sysconfdir}/sysconfig"
 AC_SUBST(CONFIGDIR)
 
 expand_path_option CRM_LOG_DIR "${localstatedir}/log/pacemaker"
 AC_DEFINE_UNQUOTED(CRM_LOG_DIR,"$CRM_LOG_DIR", Location for Pacemaker log file)
 AC_SUBST(CRM_LOG_DIR)
 
 expand_path_option CRM_BUNDLE_DIR "${localstatedir}/log/pacemaker/bundles"
 AC_DEFINE_UNQUOTED(CRM_BUNDLE_DIR,"$CRM_BUNDLE_DIR", Location for Pacemaker bundle logs)
 AC_SUBST(CRM_BUNDLE_DIR)
 
 expand_path_option PCMK__FENCE_BINDIR
 AC_DEFINE_UNQUOTED(PCMK__FENCE_BINDIR,"$PCMK__FENCE_BINDIR",
                    [Location for executable fence agents])
 
 expand_path_option OCF_RA_PATH
 AC_DEFINE_UNQUOTED([OCF_RA_PATH], ["$OCF_RA_PATH"],
                    [OCF directories to search for resource agents ])
 
 AS_IF([test x"${PCMK_GNUTLS_PRIORITIES}" != x""], [],
       [AC_MSG_ERROR([--with-gnutls-priorities value must not be empty])])
 AC_DEFINE_UNQUOTED([PCMK_GNUTLS_PRIORITIES], ["$PCMK_GNUTLS_PRIORITIES"],
                    [GnuTLS cipher priorities])
 
 if test x"${BUG_URL}" = x""; then
     BUG_URL="https://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker"
 fi
 AC_SUBST(BUG_URL)
 
 for j in prefix exec_prefix bindir sbindir libexecdir datadir sysconfdir \
     sharedstatedir localstatedir libdir includedir oldincludedir infodir \
     mandir INITDIR docdir CONFIGDIR
 do
     dirname=`eval echo '${'${j}'}'`
     if
         test ! -d "$dirname"
     then
         AC_MSG_WARN([$j directory ($dirname) does not exist (yet)])
     fi
 done
 
 us_auth=
 AC_CHECK_HEADER([sys/socket.h], [
     AC_CHECK_DECL([SO_PEERCRED], [
         # Linux
         AC_CHECK_TYPE([struct ucred], [
             us_auth=peercred_ucred;
             AC_DEFINE([US_AUTH_PEERCRED_UCRED], [1],
                       [Define if Unix socket auth method is
                        getsockopt(s, SO_PEERCRED, &ucred, ...)])
         ], [
             # OpenBSD
             AC_CHECK_TYPE([struct sockpeercred], [
                 us_auth=localpeercred_sockepeercred;
                 AC_DEFINE([US_AUTH_PEERCRED_SOCKPEERCRED], [1],
                           [Define if Unix socket auth method is
                            getsockopt(s, SO_PEERCRED, &sockpeercred, ...)])
             ], [], [[#include <sys/socket.h>]])
         ], [[#define _GNU_SOURCE
              #include <sys/socket.h>]])
     ], [], [[#include <sys/socket.h>]])
 ])
 
 AS_IF([test -z "${us_auth}"], [
     # FreeBSD
     AC_CHECK_DECL([getpeereid], [
         us_auth=getpeereid;
         AC_DEFINE([US_AUTH_GETPEEREID], [1],
                   [Define if Unix socket auth method is
                    getpeereid(s, &uid, &gid)])
     ], [
         # Solaris/OpenIndiana
         AC_CHECK_DECL([getpeerucred], [
             us_auth=getpeerucred;
             AC_DEFINE([US_AUTH_GETPEERUCRED], [1],
                       [Define if Unix socket auth method is
                        getpeercred(s, &ucred)])
         ], [
             AC_MSG_FAILURE([No way to authenticate a Unix socket peer])
         ], [[#include <ucred.h>]])
     ])
 ])
 
 dnl OS-based decision-making is poor autotools practice; feature-based
 dnl mechanisms are strongly preferred. Keep this section to a bare minimum;
 dnl regard as a "necessary evil".
 INIT_EXT=""
 PROCFS=0
 case "$host_os" in
     dnl Solaris and some *BSD versions support procfs but not files we need
     *bsd*)
         INIT_EXT=".sh"
         ;;
     *linux*)
         PROCFS=1
         ;;
     darwin*)
         LIBS="$LIBS -L${prefix}/lib"
         CFLAGS="$CFLAGS -I${prefix}/include"
         ;;
 esac
 
 AC_SUBST(INIT_EXT)
 AC_DEFINE_UNQUOTED([SUPPORT_PROCFS], [$PROCFS],
                    [Define to 1 if procfs is supported])
 
 case "$host_cpu" in
     ppc64|powerpc64)
         case $CFLAGS in
          *powerpc64*)
              ;;
          *)
              if test "$GCC" = yes; then
                  CFLAGS="$CFLAGS -m64"
              fi
              ;;
         esac
         ;;
 esac
 
 # C99 doesn't guarantee uint64_t type and related format specifiers, but
 # prerequisites, corosync + libqb, use that widely, so the target platforms
 # are already pre-constrained to those "64bit-clean" (doesn't imply native
 # bit width) and hence we deliberately refrain from artificial surrogates
 # (sans manipulation through cached values).
 AC_CACHE_VAL(
     [pcmk_cv_decl_inttypes],
     [
         AC_CHECK_DECLS(
             [PRIu64, PRIu32, PRIx32,
              SCNu64],
             [pcmk_cv_decl_inttypes="PRIu64 PRIu32 PRIx32 SCNu64"],
             [
                 # test shall only react on "no" cached result & error out respectively
                 if test "x$ac_cv_have_decl_PRIu64" = xno; then
                     AC_MSG_ERROR([lack of inttypes.h based specifier serving uint64_t (PRIu64)])
                 elif test "x$ac_cv_have_decl_PRIu32" = xno; then
                     AC_MSG_ERROR([lack of inttypes.h based specifier serving uint32_t (PRIu32)])
                 elif test "x$ac_cv_have_decl_PRIx32" = xno; then
                     AC_MSG_ERROR([lack of inttypes.h based hexa specifier serving uint32_t (PRIx32)])
                 elif test "x$ac_cv_have_decl_SCNu64" = xno; then
                     AC_MSG_ERROR([lack of inttypes.h based specifier gathering uint64_t (SCNu64)])
                 fi
             ],
             [[#include <inttypes.h>]]
         )
     ]
 )
 (
     set $pcmk_cv_decl_inttypes
     AC_DEFINE_UNQUOTED([U64T],  [$1], [Correct format specifier for U64T])
     AC_DEFINE_UNQUOTED([U32T],  [$2], [Correct format specifier for U32T])
     AC_DEFINE_UNQUOTED([X32T],  [$3], [Correct format specifier for X32T])
     AC_DEFINE_UNQUOTED([U64TS], [$4], [Correct format specifier for U64TS])
 )
 
 dnl ===============================================
 dnl Program Paths
 dnl ===============================================
 
 PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin"
 export PATH
 
 dnl Replacing AC_PROG_LIBTOOL with AC_CHECK_PROG because LIBTOOL
 dnl was NOT being expanded all the time thus causing things to fail.
 AC_CHECK_PROGS(LIBTOOL, glibtool libtool libtool15 libtool13)
 AS_IF([test "x${LIBTOOL}" != "x"], [],
       [AC_MSG_FAILURE([Could not find required build tool libtool (or equivalent)])])
 
 dnl Pacemaker's executable python scripts will invoke the python specified by
 dnl configure's PYTHON variable. If not specified, AM_PATH_PYTHON will check a
 dnl built-in list with (unversioned) "python" having precedence. To configure
 dnl Pacemaker to use a specific python interpreter version, define PYTHON
 dnl when calling configure, for example: ./configure PYTHON=/usr/bin/python3.6
 
 dnl Ensure PYTHON is an absolute path
 AS_IF([test x"${PYTHON}" != x""], [AC_PATH_PROG([PYTHON], [$PYTHON])])
 
 dnl Require a minimum Python version
 AM_PATH_PYTHON([3.4])
 
 AC_PATH_PROGS([ASCIIDOC_CONV], [asciidoc asciidoctor])
 AC_PATH_PROG([HELP2MAN], [help2man])
 AC_PATH_PROG([SPHINX], [sphinx-build])
 AC_PATH_PROG([INKSCAPE], [inkscape])
 AC_PATH_PROG([XSLTPROC], [xsltproc])
 AC_PATH_PROG([XMLCATALOG], [xmlcatalog])
 
 dnl Bash is needed for building man pages and running regression tests.
 dnl BASH is already an environment variable, so use something else.
 AC_PATH_PROG([BASH_PATH], [bash])
 AS_IF([test "x${BASH_PATH}" != "x"], [],
       [AC_MSG_FAILURE([Could not find required build tool bash])])
 
 AC_PATH_PROGS(VALGRIND_BIN, valgrind, /usr/bin/valgrind)
 AC_DEFINE_UNQUOTED(VALGRIND_BIN, "$VALGRIND_BIN", Valgrind command)
 
 AM_CONDITIONAL(BUILD_HELP, test x"${HELP2MAN}" != x"")
 if test x"${HELP2MAN}" != x""; then
     PCMK_FEATURES="$PCMK_FEATURES generated-manpages"
 fi
 
 MANPAGE_XSLT=""
 if test x"${XSLTPROC}" != x""; then
     AC_MSG_CHECKING([for DocBook-to-manpage transform])
     # first try to figure out correct template using xmlcatalog query,
     # resort to extensive (semi-deterministic) file search if that fails
     DOCBOOK_XSL_URI='http://docbook.sourceforge.net/release/xsl/current'
     DOCBOOK_XSL_PATH='manpages/docbook.xsl'
     MANPAGE_XSLT=$(${XMLCATALOG} "" ${DOCBOOK_XSL_URI}/${DOCBOOK_XSL_PATH} \
                    | sed -n 's|^file://||p;q')
     if test x"${MANPAGE_XSLT}" = x""; then
         DIRS=$(find "${datadir}" -name $(basename $(dirname ${DOCBOOK_XSL_PATH})) \
                -type d 2>/dev/null | LC_ALL=C sort)
         XSLT=$(basename ${DOCBOOK_XSL_PATH})
         for d in ${DIRS}
         do
             if test -f "${d}/${XSLT}"; then
                  MANPAGE_XSLT="${d}/${XSLT}"
                  break
             fi
         done
     fi
 fi
 AC_MSG_RESULT([$MANPAGE_XSLT])
 AC_SUBST(MANPAGE_XSLT)
 
 AM_CONDITIONAL(BUILD_XML_HELP, test x"${MANPAGE_XSLT}" != x"")
 if test x"${MANPAGE_XSLT}" != x""; then
     PCMK_FEATURES="$PCMK_FEATURES agent-manpages"
 fi
 
 AM_CONDITIONAL([IS_ASCIIDOC], [echo "${ASCIIDOC_CONV}" | grep -Eq 'asciidoc$'])
 AM_CONDITIONAL([BUILD_ASCIIDOC], [test "x${ASCIIDOC_CONV}" != x])
 if test "x${ASCIIDOC_CONV}" != x; then
     PCMK_FEATURES="$PCMK_FEATURES ascii-docs"
 fi
 
 AM_CONDITIONAL([BUILD_SPHINX_DOCS],
                [test x"${SPHINX}" != x"" && test x"${INKSCAPE}" != x""])
 AM_COND_IF([BUILD_SPHINX_DOCS], [PCMK_FEATURES="$PCMK_FEATURES books"])
 
 dnl Pacemaker's shell scripts (and thus man page builders) rely on GNU getopt
 AC_MSG_CHECKING([for GNU-compatible getopt])
 IFS_orig=$IFS
 IFS=:
 for PATH_DIR in $PATH
 do
     IFS=$IFS_orig
     GETOPT_PATH="${PATH_DIR}/getopt"
     if test -f "$GETOPT_PATH" && test -x "$GETOPT_PATH" ; then
         $GETOPT_PATH -T >/dev/null 2>/dev/null
         if test $? -eq 4; then
             break
         fi
     fi
     GETOPT_PATH=""
 done
 IFS=$IFS_orig
 AS_IF([test -n "$GETOPT_PATH"], [AC_MSG_RESULT([$GETOPT_PATH])],
       [
           AC_MSG_RESULT([no])
           AC_MSG_ERROR([Could not find required build tool GNU-compatible getopt])
       ])
 AC_SUBST([GETOPT_PATH])
 
 dnl ========================================================================
 dnl checks for library functions to replace them
 dnl
 dnl     NoSuchFunctionName:
 dnl             is a dummy function which no system supplies.  It is here to make
 dnl             the system compile semi-correctly on OpenBSD which doesn't know
 dnl             how to create an empty archive
 dnl
 dnl     scandir: Only on BSD.
 dnl             System-V systems may have it, but hidden and/or deprecated.
 dnl             A replacement function is supplied for it.
 dnl
 dnl     strerror: returns a string that corresponds to an errno.
 dnl             A replacement function is supplied for it.
 dnl
 dnl     strnlen: is a gnu function similar to strlen, but safer.
 dnl            We wrote a tolerably-fast replacement function for it.
 dnl
 dnl     strndup: is a gnu function similar to strdup, but safer.
 dnl            We wrote a tolerably-fast replacement function for it.
 
 AC_REPLACE_FUNCS(alphasort NoSuchFunctionName scandir strerror strchrnul strnlen strndup)
 
 dnl ===============================================
 dnl Libraries
 dnl ===============================================
 AC_CHECK_LIB(socket, socket)                    dnl -lsocket
 AC_CHECK_LIB(c, dlopen)                         dnl if dlopen is in libc...
 AC_CHECK_LIB(dl, dlopen)                        dnl -ldl (for Linux)
 AC_CHECK_LIB(rt, sched_getscheduler)            dnl -lrt (for Tru64)
 AC_CHECK_LIB(gnugetopt, getopt_long)            dnl -lgnugetopt ( if available )
 AC_CHECK_LIB(pam, pam_start)                    dnl -lpam (if available)
 
 PKG_CHECK_MODULES([UUID], [uuid],
                   [CPPFLAGS="${CPPFLAGS} ${UUID_CFLAGS}"
                    LIBS="${LIBS} ${UUID_LIBS}"])
 
 AC_CHECK_FUNCS([sched_setscheduler])
 if test "$ac_cv_func_sched_setscheduler" != yes; then
         PC_LIBS_RT=""
 else
         PC_LIBS_RT="-lrt"
 fi
 AC_SUBST(PC_LIBS_RT)
 
 # Require minimum glib version
 PKG_CHECK_MODULES([GLIB], [glib-2.0 >= 2.42.0],
                   [CPPFLAGS="${CPPFLAGS} ${GLIB_CFLAGS}"
                    LIBS="${LIBS} ${GLIB_LIBS}"])
 
 # Check whether high-resolution sleep function is available
 AC_CHECK_FUNCS([nanosleep usleep])
 
 #
 # Where is dlopen?
 #
 if test "$ac_cv_lib_c_dlopen" = yes; then
     LIBADD_DL=""
 elif test "$ac_cv_lib_dl_dlopen" = yes; then
     LIBADD_DL=-ldl
 else
     LIBADD_DL=${lt_cv_dlopen_libs}
 fi
 
 PKG_CHECK_MODULES(LIBXML2, [libxml-2.0],
                   [CPPFLAGS="${CPPFLAGS} ${LIBXML2_CFLAGS}"
                    LIBS="${LIBS} ${LIBXML2_LIBS}"])
 
 REQUIRE_LIB([xslt], [xsltApplyStylesheet])
 
 dnl ========================================================================
 dnl Headers
 dnl ========================================================================
 
 # Some distributions insert #warnings into deprecated headers. If we will
 # enable fatal warnings for the build, then enable them for the header checks
 # as well, otherwise the build could fail even though the header check
 # succeeds. (We should probably be doing this in more places.)
 cc_temp_flags "$CFLAGS $WERROR"
 
 # Optional headers (inclusion of these should be conditional in C code)
 AC_CHECK_HEADERS([getopt.h])
 AC_CHECK_HEADERS([linux/swab.h])
 AC_CHECK_HEADERS([stddef.h])
 AC_CHECK_HEADERS([sys/signalfd.h])
 AC_CHECK_HEADERS([uuid/uuid.h])
 AC_CHECK_HEADERS([security/pam_appl.h pam/pam_appl.h])
 
 # Required headers
 REQUIRE_HEADER([arpa/inet.h])
 REQUIRE_HEADER([ctype.h])
 REQUIRE_HEADER([dirent.h])
 REQUIRE_HEADER([errno.h])
 REQUIRE_HEADER([glib.h])
 REQUIRE_HEADER([grp.h])
 REQUIRE_HEADER([limits.h])
 REQUIRE_HEADER([netdb.h])
 REQUIRE_HEADER([netinet/in.h])
 REQUIRE_HEADER([netinet/ip.h], [
     #include <sys/types.h>
     #include <netinet/in.h>
 ])
 REQUIRE_HEADER([pwd.h])
 REQUIRE_HEADER([signal.h])
 REQUIRE_HEADER([stdio.h])
 REQUIRE_HEADER([stdlib.h])
 REQUIRE_HEADER([string.h])
 REQUIRE_HEADER([strings.h])
 REQUIRE_HEADER([sys/ioctl.h])
 REQUIRE_HEADER([sys/param.h])
 REQUIRE_HEADER([sys/reboot.h])
 REQUIRE_HEADER([sys/resource.h])
 REQUIRE_HEADER([sys/socket.h])
 REQUIRE_HEADER([sys/stat.h])
 REQUIRE_HEADER([sys/time.h])
 REQUIRE_HEADER([sys/types.h])
 REQUIRE_HEADER([sys/utsname.h])
 REQUIRE_HEADER([sys/wait.h])
 REQUIRE_HEADER([time.h])
 REQUIRE_HEADER([unistd.h])
 REQUIRE_HEADER([libxml/xpath.h])
 REQUIRE_HEADER([libxslt/xslt.h])
 
 cc_restore_flags
 
 AC_CHECK_FUNCS([uuid_unparse], [],
                [AC_MSG_FAILURE([Could not find required C function uuid_unparse()])])
 
 AC_CACHE_CHECK([whether __progname and __progname_full are available],
                [pf_cv_var_progname], AC_LINK_IFELSE([
                    AC_LANG_PROGRAM([[extern char *__progname, *__progname_full;]],
                                    [[__progname = "foo"; __progname_full = "foo bar";]],
                                    [pf_cv_var_progname="yes"],
                                    [pf_cv_var_progname="no"])
                ]))
 AS_IF([test "$pf_cv_var_progname" = "yes"], [AC_DEFINE(HAVE___PROGNAME,1,[ ])])
 
 dnl ========================================================================
 dnl Generic declarations
 dnl ========================================================================
 
 AC_CHECK_DECLS([CLOCK_MONOTONIC], [PCMK_FEATURES="$PCMK_FEATURES monotonic"], [], [[
     #include <time.h>
 ]])
 
 dnl ========================================================================
 dnl Unit test declarations
 dnl ========================================================================
 
 AC_CHECK_DECLS([assert_float_equal], [], [], [[
     #include <stdarg.h>
     #include <stddef.h>
     #include <setjmp.h>
     #include <cmocka.h>
 ]])
 
 cc_temp_flags "$CFLAGS -Wl,--wrap=uname"
 
 WRAPPABLE_UNAME="no"
 
 AC_MSG_CHECKING([if uname() can be wrapped])
 AC_RUN_IFELSE([AC_LANG_SOURCE([[
 #include <sys/utsname.h>
 int __wrap_uname(struct utsname *buf) {
 return 100;
 }
 int main(int argc, char **argv) {
 struct utsname x;
 return uname(&x) == 100 ? 0 : 1;
 }
 ]])],
                    [ WRAPPABLE_UNAME="yes" ], [ WRAPPABLE_UNAME="no"])
 AC_MSG_RESULT([$WRAPPABLE_UNAME])
 AM_CONDITIONAL([WRAPPABLE_UNAME], [test "$WRAPPABLE_UNAME" = "yes"])
 
 cc_restore_flags
 
 dnl ========================================================================
 dnl Structures
 dnl ========================================================================
 
 AC_CHECK_MEMBERS([struct tm.tm_gmtoff],,,[[#include <time.h>]])
 AC_CHECK_MEMBER([struct dirent.d_type],
     AC_DEFINE(HAVE_STRUCT_DIRENT_D_TYPE,1,[Define this if struct dirent has d_type]),,
     [#include <dirent.h>])
 
 dnl ========================================================================
 dnl Functions
 dnl ========================================================================
 
 REQUIRE_FUNC([getopt])
 REQUIRE_FUNC([setenv])
 REQUIRE_FUNC([unsetenv])
 
 AC_CACHE_CHECK(whether sscanf supports %m,
                pf_cv_var_sscanf,
                AC_RUN_IFELSE([AC_LANG_SOURCE([[
 #include <stdio.h>
 const char *s = "some-command-line-arg";
 int main(int argc, char **argv) {
 char *name = NULL;
 int n = sscanf(s, "%ms", &name);
 return n == 1 ? 0 : 1;
 }
 ]])],
                                  pf_cv_var_sscanf="yes", pf_cv_var_sscanf="no", pf_cv_var_sscanf="no"))
 
 AS_IF([test "$pf_cv_var_sscanf" = "yes"], [AC_DEFINE(SSCANF_HAS_M, 1, [ ])])
 
 dnl ========================================================================
 dnl   bzip2
 dnl ========================================================================
 REQUIRE_HEADER([bzlib.h])
 REQUIRE_LIB([bz2], [BZ2_bzBuffToBuffCompress])
 
 dnl ========================================================================
 dnl sighandler_t is missing from Illumos, Solaris11 systems
 dnl ========================================================================
 
 AC_MSG_CHECKING([for sighandler_t])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <signal.h>]], [[sighandler_t *f;]])],
                   [
                       AC_MSG_RESULT([yes])
                       AC_DEFINE([HAVE_SIGHANDLER_T], [1],
                                 [Define to 1 if sighandler_t is available])
                   ],
                   [AC_MSG_RESULT([no])])
 
 dnl ========================================================================
 dnl   ncurses
 dnl ========================================================================
 dnl
 dnl A few OSes (e.g. Linux) deliver a default "ncurses" alongside "curses".
 dnl Many non-Linux deliver "curses"; sites may add "ncurses".
 dnl
 dnl However, the source-code recommendation for both is to #include "curses.h"
 dnl (i.e. "ncurses" still wants the include to be simple, no-'n', "curses.h").
 dnl
 dnl ncurses takes precedence.
 dnl
 AC_CHECK_HEADERS([curses.h curses/curses.h ncurses.h ncurses/ncurses.h])
 
 dnl Although n-library is preferred, only look for it if the n-header was found.
 CURSESLIBS=''
 PC_NAME_CURSES=""
 PC_LIBS_CURSES=""
 AS_IF([test "$ac_cv_header_ncurses_h" = "yes"], [
     AC_CHECK_LIB(ncurses, printw,
                  [AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
     CURSESLIBS=`$PKG_CONFIG --libs ncurses` || CURSESLIBS='-lncurses'
     PC_NAME_CURSES="ncurses"
 ])
 
 AS_IF([test "$ac_cv_header_ncurses_ncurses_h" = "yes"], [
     AC_CHECK_LIB(ncurses, printw,
                  [AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
     CURSESLIBS=`$PKG_CONFIG --libs ncurses` || CURSESLIBS='-lncurses'
     PC_NAME_CURSES="ncurses"
 ])
 
 dnl Only look for non-n-library if there was no n-library.
 AS_IF([test X"$CURSESLIBS" = X"" && test "$ac_cv_header_curses_h" = "yes"], [
     AC_CHECK_LIB(curses, printw,
                  [CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
     PC_LIBS_CURSES="$CURSESLIBS"
 ])
 
 dnl Only look for non-n-library if there was no n-library.
 AS_IF([test X"$CURSESLIBS" = X"" && test "$ac_cv_header_curses_curses_h" = "yes"], [
     AC_CHECK_LIB(curses, printw,
                  [CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
     PC_LIBS_CURSES="$CURSESLIBS"
 ])
 
 if test "x$CURSESLIBS" != "x"; then
     PCMK_FEATURES="$PCMK_FEATURES ncurses"
 fi
 
 dnl Check for printw() prototype compatibility
 AS_IF([test X"$CURSESLIBS" != X"" && cc_supports_flag -Wcast-qual], [
     ac_save_LIBS=$LIBS
     LIBS="$CURSESLIBS"
     cc_temp_flags "-Wcast-qual $WERROR"
     # avoid broken test because of hardened build environment in Fedora 23+
     # - https://fedoraproject.org/wiki/Changes/Harden_All_Packages
     # - https://bugzilla.redhat.com/1297985
     AS_IF([cc_supports_flag -fPIC], [CFLAGS="$CFLAGS -fPIC"])
 
     AC_MSG_CHECKING([whether curses library is compatible])
     AC_LINK_IFELSE(
         [AC_LANG_PROGRAM([
 #if defined(HAVE_NCURSES_H)
 #  include <ncurses.h>
 #elif defined(HAVE_NCURSES_NCURSES_H)
 #  include <ncurses/ncurses.h>
 #elif defined(HAVE_CURSES_H)
 #  include <curses.h>
 #endif
                          ],
                          [printw((const char *)"Test");]
         )],
         [AC_MSG_RESULT([yes])],
         [
             AC_MSG_RESULT([no])
             AC_MSG_WARN(m4_normalize([Disabling curses because the printw()
                                       function of your (n)curses library is old.
                                       If you wish to enable curses, update to a
                                       newer version (ncurses 5.4 or later is
                                       recommended, available from
                                       https://invisible-island.net/ncurses/)
                                      ]))
             AC_DEFINE([HAVE_INCOMPATIBLE_PRINTW], [1],
                       [Define to 1 if curses library has incompatible printw()])
         ]
     )
 
     LIBS=$ac_save_LIBS
     cc_restore_flags
 ])
 
 AC_SUBST(CURSESLIBS)
 AC_SUBST(PC_NAME_CURSES)
 AC_SUBST(PC_LIBS_CURSES)
 
 dnl ========================================================================
 dnl    Profiling and GProf
 dnl ========================================================================
 
 CFLAGS_ORIG="$CFLAGS"
 AS_IF([test $with_coverage -ne $DISABLED],
       [
         with_profiling=$REQUIRED
         PCMK_FEATURES="$PCMK_FEATURES coverage"
         CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage"
         dnl During linking, make sure to specify -lgcov or -coverage
       ]
 )
 
 AS_IF([test $with_profiling -ne $DISABLED],
       [
           with_profiling=$REQUIRED
           PCMK_FEATURES="$PCMK_FEATURES profile"
 
           dnl Disable various compiler optimizations
           CFLAGS="$CFLAGS -fno-omit-frame-pointer -fno-inline -fno-builtin"
           dnl CFLAGS="$CFLAGS -fno-inline-functions"
           dnl CFLAGS="$CFLAGS -fno-default-inline"
           dnl CFLAGS="$CFLAGS -fno-inline-functions-called-once"
           dnl CFLAGS="$CFLAGS -fno-optimize-sibling-calls"
 
           dnl Turn off optimization so tools can get accurate line numbers
           CFLAGS=`echo $CFLAGS | sed \
                   -e 's/-O.\ //g' \
                   -e 's/-Wp,-D_FORTIFY_SOURCE=.\ //g' \
                   -e 's/-D_FORTIFY_SOURCE=.\ //g'`
           CFLAGS="$CFLAGS -O0 -g3 -gdwarf-2"
 
           AC_MSG_NOTICE([CFLAGS before adding profiling options: $CFLAGS_ORIG])
           AC_MSG_NOTICE([CFLAGS after: $CFLAGS])
       ]
 )
 AC_DEFINE_UNQUOTED([SUPPORT_PROFILING], [$with_profiling], [Support profiling])
 AM_CONDITIONAL([BUILD_PROFILING], [test "$with_profiling" = "$REQUIRED"])
 
 dnl ========================================================================
 dnl    Cluster infrastructure - LibQB
 dnl ========================================================================
 
 PKG_CHECK_MODULES(libqb, libqb >= 0.17)
 CPPFLAGS="$libqb_CFLAGS $CPPFLAGS"
 LIBS="$libqb_LIBS $LIBS"
 
 dnl libqb 2.0.2+ (2020-10)
 AC_CHECK_FUNCS(qb_ipcc_auth_get,
                AC_DEFINE(HAVE_IPCC_AUTH_GET, 1,
                          [Have qb_ipcc_auth_get function]))
 
 dnl libqb 2.0.0+ (2020-05)
 CHECK_ENUM_VALUE([qb/qblog.h],[qb_log_conf],[QB_LOG_CONF_MAX_LINE_LEN])
 CHECK_ENUM_VALUE([qb/qblog.h],[qb_log_conf],[QB_LOG_CONF_ELLIPSIS])
 
 dnl Support Linux-HA fence agents if available
 if test "$cross_compiling" != "yes"; then
     CPPFLAGS="$CPPFLAGS -I${prefix}/include/heartbeat"
 fi
 AC_CHECK_HEADERS([stonith/stonith.h],
                  [
                      AC_CHECK_LIB([pils], [PILLoadPlugin])
                      AC_CHECK_LIB([plumb], [G_main_add_IPC_Channel])
                      PCMK_FEATURES="$PCMK_FEATURES lha"
                  ])
 AM_CONDITIONAL([BUILD_LHA_SUPPORT], [test "$ac_cv_header_stonith_stonith_h" = "yes"])
 
 dnl ===============================================
 dnl Variables needed for substitution
 dnl ===============================================
 CRM_SCHEMA_DIRECTORY="${datadir}/pacemaker"
 AC_DEFINE_UNQUOTED(CRM_SCHEMA_DIRECTORY,"$CRM_SCHEMA_DIRECTORY", Location for the Pacemaker Relax-NG Schema)
 AC_SUBST(CRM_SCHEMA_DIRECTORY)
 
 CRM_CORE_DIR="${localstatedir}/lib/pacemaker/cores"
 AC_DEFINE_UNQUOTED(CRM_CORE_DIR,"$CRM_CORE_DIR", Location to store core files produced by Pacemaker daemons)
 AC_SUBST(CRM_CORE_DIR)
 
 if test x"${CRM_DAEMON_USER}" = x""; then
     CRM_DAEMON_USER="hacluster"
 fi
 AC_DEFINE_UNQUOTED(CRM_DAEMON_USER,"$CRM_DAEMON_USER", User to run Pacemaker daemons as)
 AC_SUBST(CRM_DAEMON_USER)
 
 if test x"${CRM_DAEMON_GROUP}" = x""; then
     CRM_DAEMON_GROUP="haclient"
 fi
 AC_DEFINE_UNQUOTED(CRM_DAEMON_GROUP,"$CRM_DAEMON_GROUP", Group to run Pacemaker daemons as)
 AC_SUBST(CRM_DAEMON_GROUP)
 
 CRM_PACEMAKER_DIR=${localstatedir}/lib/pacemaker
 AC_DEFINE_UNQUOTED(CRM_PACEMAKER_DIR,"$CRM_PACEMAKER_DIR", Location to store directory produced by Pacemaker daemons)
 AC_SUBST(CRM_PACEMAKER_DIR)
 
 CRM_BLACKBOX_DIR=${localstatedir}/lib/pacemaker/blackbox
 AC_DEFINE_UNQUOTED(CRM_BLACKBOX_DIR,"$CRM_BLACKBOX_DIR", Where to keep blackbox dumps)
 AC_SUBST(CRM_BLACKBOX_DIR)
 
 PE_STATE_DIR="${localstatedir}/lib/pacemaker/pengine"
 AC_DEFINE_UNQUOTED(PE_STATE_DIR,"$PE_STATE_DIR", Where to keep scheduler outputs)
 AC_SUBST(PE_STATE_DIR)
 
 CRM_CONFIG_DIR="${localstatedir}/lib/pacemaker/cib"
 AC_DEFINE_UNQUOTED(CRM_CONFIG_DIR,"$CRM_CONFIG_DIR", Where to keep configuration files)
 AC_SUBST(CRM_CONFIG_DIR)
 
 CRM_CONFIG_CTS="${localstatedir}/lib/pacemaker/cts"
 AC_DEFINE_UNQUOTED(CRM_CONFIG_CTS,"$CRM_CONFIG_CTS", Where to keep cts stateful data)
 AC_SUBST(CRM_CONFIG_CTS)
 
 CRM_DAEMON_DIR="${libexecdir}/pacemaker"
 AC_DEFINE_UNQUOTED(CRM_DAEMON_DIR,"$CRM_DAEMON_DIR", Location for Pacemaker daemons)
 AC_SUBST(CRM_DAEMON_DIR)
 
 CRM_STATE_DIR="${runstatedir}/crm"
 AC_DEFINE_UNQUOTED([CRM_STATE_DIR], ["$CRM_STATE_DIR"],
                    [Where to keep state files and sockets])
 AC_SUBST(CRM_STATE_DIR)
 
 CRM_RSCTMP_DIR="${runstatedir}/resource-agents"
 AC_DEFINE_UNQUOTED(CRM_RSCTMP_DIR,"$CRM_RSCTMP_DIR", Where resource agents should keep state files)
 AC_SUBST(CRM_RSCTMP_DIR)
 
 PACEMAKER_CONFIG_DIR="${sysconfdir}/pacemaker"
 AC_DEFINE_UNQUOTED(PACEMAKER_CONFIG_DIR,"$PACEMAKER_CONFIG_DIR", Where to keep configuration files like authkey)
 AC_SUBST(PACEMAKER_CONFIG_DIR)
 
 AC_DEFINE_UNQUOTED(SBIN_DIR,"$sbindir",[Location for system binaries])
 
 AC_PATH_PROGS(GIT, git false)
 
 AC_MSG_CHECKING([build version])
 BUILD_VERSION=$Format:%h$
 if test $BUILD_VERSION != ":%h$"; then
     AC_MSG_RESULT([$BUILD_VERSION (archive hash)])
 elif test -x $GIT && test -d .git; then
     BUILD_VERSION=`$GIT log --pretty="format:%h" -n 1`
     AC_MSG_RESULT([$BUILD_VERSION (git hash)])
 else
     # The current directory name make a reasonable default
     # Most generated archives will include the hash or tag
     BASE=`basename $PWD`
     BUILD_VERSION=`echo $BASE | sed s:.*[[Pp]]acemaker-::`
     AC_MSG_RESULT([$BUILD_VERSION (directory name)])
 fi
 
 AC_DEFINE_UNQUOTED(BUILD_VERSION, "$BUILD_VERSION", Build version)
 AC_SUBST(BUILD_VERSION)
 
 HAVE_dbus=1
 PKG_CHECK_MODULES([DBUS], [dbus-1],
                   [CPPFLAGS="${CPPFLAGS} ${DBUS_CFLAGS}"],
                   [HAVE_dbus=0])
 AC_DEFINE_UNQUOTED(SUPPORT_DBUS, $HAVE_dbus, Support dbus)
 AM_CONDITIONAL(BUILD_DBUS, test $HAVE_dbus = 1)
 AC_CHECK_TYPES([DBusBasicValue],,,[[#include <dbus/dbus.h>]])
 if test $HAVE_dbus = 0; then
         PC_NAME_DBUS=""
 else
         PC_NAME_DBUS="dbus-1"
 fi
 AC_SUBST(PC_NAME_DBUS)
 
 AS_CASE([$enable_systemd],
         [$REQUIRED], [
             AS_IF([test $HAVE_dbus = 0],
                   [AC_MSG_FAILURE([Cannot support systemd resources without DBus])])
             AS_IF([test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
                   [AC_MSG_FAILURE([Cannot support systemd resources without monotonic clock])])
             AS_IF([check_systemdsystemunitdir], [],
                   [AC_MSG_FAILURE([Cannot support systemd resources without systemdsystemunitdir])])
         ],
         [$OPTIONAL], [
             AS_IF([test $HAVE_dbus = 0 \
                    || test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
                   [enable_systemd=$DISABLED],
                   [
                       AC_MSG_CHECKING([for systemd version (using dbus-send)])
                       ret=$({ dbus-send --system --print-reply \
                                   --dest=org.freedesktop.systemd1 \
                                   /org/freedesktop/systemd1 \
                                   org.freedesktop.DBus.Properties.Get \
                                   string:org.freedesktop.systemd1.Manager \
                                   string:Version 2>/dev/null \
                               || echo "version unavailable"; } | tail -n1)
                       # sanitize output a bit (interested just in value, not type),
                       # ret is intentionally unenquoted so as to normalize whitespace
                       ret=$(echo ${ret} | cut -d' ' -f2-)
                       AC_MSG_RESULT([${ret}])
                       AS_IF([test "$ret" != "unavailable" \
                              || systemctl --version 2>/dev/null | grep -q systemd],
                             [
                                 AS_IF([check_systemdsystemunitdir],
                                       [enable_systemd=$REQUIRED],
                                       [enable_systemd=$DISABLED])
                             ],
                             [enable_systemd=$DISABLED]
                       )
                   ])
         ],
 )
 AC_MSG_CHECKING([whether to enable support for managing resources via systemd])
 AS_IF([test $enable_systemd -eq $DISABLED], [AC_MSG_RESULT([no])],
       [
           AC_MSG_RESULT([yes])
           PCMK_FEATURES="$PCMK_FEATURES systemd"
       ]
 )
 AC_SUBST([systemdsystemunitdir])
 AC_DEFINE_UNQUOTED([SUPPORT_SYSTEMD], [$enable_systemd],
                    [Support systemd resources])
 AM_CONDITIONAL([BUILD_SYSTEMD], [test $enable_systemd = $REQUIRED])
 AC_SUBST(SUPPORT_SYSTEMD)
 
 AS_CASE([$enable_upstart],
         [$REQUIRED], [
             AS_IF([test $HAVE_dbus = 0],
                   [AC_MSG_FAILURE([Cannot support Upstart resources without DBus])])
         ],
         [$OPTIONAL], [
             AS_IF([test $HAVE_dbus = 0], [enable_upstart=$DISABLED],
                   [
                       AC_MSG_CHECKING([for Upstart version (using dbus-send)])
                       ret=$({ dbus-send --system --print-reply \
                                   --dest=com.ubuntu.Upstart \
                                   /com/ubuntu/Upstart org.freedesktop.DBus.Properties.Get \
                                   string:com.ubuntu.Upstart0_6 string:version 2>/dev/null \
                               || echo "version unavailable"; } | tail -n1)
                       # sanitize output a bit (interested just in value, not type),
                       # ret is intentionally unenquoted so as to normalize whitespace
                       ret=$(echo ${ret} | cut -d' ' -f2-)
                       AC_MSG_RESULT([${ret}])
                       AS_IF([test "$ret" != "unavailable" \
                              || initctl --version 2>/dev/null | grep -q upstart],
                             [enable_upstart=$REQUIRED],
                             [enable_upstart=$DISABLED]
                       )
                   ])
         ],
 )
 AC_MSG_CHECKING([whether to enable support for managing resources via Upstart])
 AS_IF([test $enable_upstart -eq $DISABLED], [AC_MSG_RESULT([no])],
       [
           AC_MSG_RESULT([yes])
           PCMK_FEATURES="$PCMK_FEATURES upstart"
       ]
 )
 AC_DEFINE_UNQUOTED([SUPPORT_UPSTART], [$enable_upstart],
                    [Support Upstart resources])
 AM_CONDITIONAL([BUILD_UPSTART], [test $enable_upstart -eq $REQUIRED])
 AC_SUBST(SUPPORT_UPSTART)
 
 AS_CASE([$with_nagios],
         [$REQUIRED], [
             AS_IF([test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
                   [AC_MSG_FAILURE([Cannot support nagios resources without monotonic clock])])
         ],
         [$OPTIONAL], [
             AS_IF([test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
                   [with_nagios=$DISABLED], [with_nagios=$REQUIRED])
         ]
 )
 AS_IF([test $with_nagios -eq $REQUIRED], [PCMK_FEATURES="$PCMK_FEATURES nagios"])
 AC_DEFINE_UNQUOTED([SUPPORT_NAGIOS], [$with_nagios], [Support nagios plugins])
 AM_CONDITIONAL([BUILD_NAGIOS], [test $with_nagios -eq $REQUIRED])
 
 if test x"$NAGIOS_PLUGIN_DIR" = x""; then
     NAGIOS_PLUGIN_DIR="${libexecdir}/nagios/plugins"
 fi
 
 AC_DEFINE_UNQUOTED(NAGIOS_PLUGIN_DIR, "$NAGIOS_PLUGIN_DIR", Directory for nagios plugins)
 AC_SUBST(NAGIOS_PLUGIN_DIR)
 
 if test x"$NAGIOS_METADATA_DIR" = x""; then
     NAGIOS_METADATA_DIR="${datadir}/nagios/plugins-metadata"
 fi
 
 AC_DEFINE_UNQUOTED(NAGIOS_METADATA_DIR, "$NAGIOS_METADATA_DIR", Directory for nagios plugins metadata)
 AC_SUBST(NAGIOS_METADATA_DIR)
 
 STACKS=""
 CLUSTERLIBS=""
 PC_NAME_CLUSTER=""
 
 dnl ========================================================================
 dnl    Cluster stack - Corosync
 dnl ========================================================================
 
 COROSYNC_LIBS=""
 
 AS_CASE([$with_corosync],
         [$REQUIRED], [
             # These will be fatal if unavailable
             PKG_CHECK_MODULES([cpg], [libcpg])
             PKG_CHECK_MODULES([cfg], [libcfg])
             PKG_CHECK_MODULES([cmap], [libcmap])
             PKG_CHECK_MODULES([quorum], [libquorum])
             PKG_CHECK_MODULES([libcorosync_common], [libcorosync_common])
         ]
         [$OPTIONAL], [
             PKG_CHECK_MODULES([cpg], [libcpg], [], [with_corosync=$DISABLED])
             PKG_CHECK_MODULES([cfg], [libcfg], [], [with_corosync=$DISABLED])
             PKG_CHECK_MODULES([cmap], [libcmap], [], [with_corosync=$DISABLED])
             PKG_CHECK_MODULES([quorum], [libquorum], [], [with_corosync=$DISABLED])
             PKG_CHECK_MODULES([libcorosync_common], [libcorosync_common], [], [with_corosync=$DISABLED])
             AS_IF([test $with_corosync -ne $DISABLED], [with_corosync=$REQUIRED])
         ]
 )
 AS_IF([test $with_corosync -ne $DISABLED],
       [
           AC_MSG_CHECKING([for Corosync 2 or later])
           AC_MSG_RESULT([yes])
           CFLAGS="$CFLAGS $libqb_CFLAGS $cpg_CFLAGS $cfg_CFLAGS $cmap_CFLAGS $quorum_CFLAGS $libcorosync_common_CFLAGS"
           COROSYNC_LIBS="$COROSYNC_LIBS $cpg_LIBS $cfg_LIBS $cmap_LIBS $quorum_LIBS $libcorosync_common_LIBS"
           CLUSTERLIBS="$CLUSTERLIBS $COROSYNC_LIBS"
           PC_NAME_CLUSTER="$PC_CLUSTER_NAME libcfg libcmap libcorosync_common libcpg libquorum"
           STACKS="$STACKS corosync-ge-2"
 
           dnl Shutdown tracking added (back) to corosync Jan 2021
           saved_LIBS="$LIBS"
           LIBS="$LIBS $COROSYNC_LIBS"
           AC_CHECK_FUNCS(corosync_cfg_trackstart,
           AC_DEFINE(HAVE_COROSYNC_CFG_TRACKSTART, 1,
                          [Have corosync_cfg_trackstart function]))
           LIBS="$saved_LIBS"
       ]
 )
 AC_DEFINE_UNQUOTED([SUPPORT_COROSYNC], [$with_corosync],
                    [Support the Corosync messaging and membership layer])
 AM_CONDITIONAL([BUILD_CS_SUPPORT], [test $with_corosync -eq $REQUIRED])
 AC_SUBST([SUPPORT_COROSYNC])
 
 dnl
 dnl    Cluster stack - Sanity
 dnl
 
 AS_IF([test "x$STACKS" != "x"], [AC_MSG_NOTICE([Supported stacks:${STACKS}])],
       [AC_MSG_FAILURE([At least one cluster stack must be supported])])
 
 PCMK_FEATURES="${PCMK_FEATURES}${STACKS}"
 
 AC_SUBST(CLUSTERLIBS)
 AC_SUBST(PC_NAME_CLUSTER)
 
 dnl ========================================================================
 dnl    CIB secrets
 dnl ========================================================================
 
 AS_IF([test $with_cibsecrets -ne $DISABLED],
       [
           with_cibsecrets=$REQUIRED
           PCMK_FEATURES="$PCMK_FEATURES cibsecrets"
           LRM_CIBSECRETS_DIR="${localstatedir}/lib/pacemaker/lrm/secrets"
           AC_DEFINE_UNQUOTED([LRM_CIBSECRETS_DIR], ["$LRM_CIBSECRETS_DIR"],
                              [Location for CIB secrets])
           AC_SUBST([LRM_CIBSECRETS_DIR])
       ]
 )
 AC_DEFINE_UNQUOTED([SUPPORT_CIBSECRETS], [$with_cibsecrets], [Support CIB secrets])
 AM_CONDITIONAL([BUILD_CIBSECRETS], [test $with_cibsecrets -eq $REQUIRED])
 
 dnl ========================================================================
 dnl    GnuTLS
 dnl ========================================================================
 
 dnl Require GnuTLS >=2.12.0 (2011-03) for Pacemaker Remote support
 PC_NAME_GNUTLS=""
 AS_CASE([$with_gnutls],
         [$REQUIRED], [
             REQUIRE_LIB([gnutls], [gnutls_sec_param_to_pk_bits])
             REQUIRE_HEADER([gnutls/gnutls.h])
         ],
         [$OPTIONAL], [
             AC_CHECK_LIB([gnutls], [gnutls_sec_param_to_pk_bits],
                          [], [with_gnutls=$DISABLED])
             AC_CHECK_HEADERS([gnutls/gnutls.h], [], [with_gnutls=$DISABLED])
         ]
 )
 AS_IF([test $with_gnutls -ne $DISABLED],
       [
           PC_NAME_GNUTLS="gnutls"
           PCMK_FEATURES="$PCMK_FEATURES remote"
       ]
 )
 AC_SUBST([PC_NAME_GNUTLS])
 AM_CONDITIONAL([BUILD_REMOTE], [test $with_gnutls -ne $DISABLED])
 
 dnl ========================================================================
 dnl    System Health
 dnl ========================================================================
 
 dnl Check if servicelog development package is installed
 SERVICELOG=servicelog-1
 SERVICELOG_EXISTS="no"
 AC_MSG_CHECKING([for $SERVICELOG packages])
 if
     $PKG_CONFIG --exists $SERVICELOG
 then
     PKG_CHECK_MODULES([SERVICELOG], [servicelog-1])
     SERVICELOG_EXISTS="yes"
     PCMK_FEATURES="$PCMK_FEATURES servicelog"
 fi
 AC_MSG_RESULT([$SERVICELOG_EXISTS])
 AM_CONDITIONAL(BUILD_SERVICELOG, test "$SERVICELOG_EXISTS" = "yes")
 
 dnl Check if OpenIMPI packages and servicelog are installed
 OPENIPMI="OpenIPMI OpenIPMIposix"
 OPENIPMI_SERVICELOG_EXISTS="no"
 AC_MSG_CHECKING([for $SERVICELOG $OPENIPMI packages])
 if
     $PKG_CONFIG --exists $OPENIPMI $SERVICELOG
 then
     PKG_CHECK_MODULES([OPENIPMI_SERVICELOG],[OpenIPMI OpenIPMIposix])
     REQUIRE_HEADER([malloc.h])
     OPENIPMI_SERVICELOG_EXISTS="yes"
     PCMK_FEATURES="$PCMK_FEATURES ipmiservicelogd"
 fi
 AC_MSG_RESULT([$OPENIPMI_SERVICELOG_EXISTS])
 AM_CONDITIONAL(BUILD_OPENIPMI_SERVICELOG, test "$OPENIPMI_SERVICELOG_EXISTS" = "yes")
 
 # --- ASAN/UBSAN/TSAN (see man gcc) ---
 # when using SANitizers, we need to pass the -fsanitize..
 # to both CFLAGS and LDFLAGS. The CFLAGS/LDFLAGS must be
 # specified as first in the list or there will be runtime
 # issues (for example user has to LD_PRELOAD asan for it to work
 # properly).
 
 AS_IF([test -n "${SANITIZERS}"], [
   SANITIZERS=$(echo $SANITIZERS | sed -e 's/,/ /g')
   for SANITIZER in $SANITIZERS
   do
     AS_CASE([$SANITIZER],
             [asan|ASAN], [
                 SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=address"
                 SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=address -lasan"
                 PCMK_FEATURES="$PCMK_FEATURES asan"
                 REQUIRE_LIB([asan],[main])
             ],
             [ubsan|UBSAN], [
                 SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=undefined"
                 SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=undefined -lubsan"
                 PCMK_FEATURES="$PCMK_FEATURES ubsan"
                 REQUIRE_LIB([ubsan],[main])
             ],
             [tsan|TSAN], [
                 SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=thread"
                 SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=thread -ltsan"
                 PCMK_FEATURES="$PCMK_FEATURES tsan"
                 REQUIRE_LIB([tsan],[main])
             ])
   done
 ])
 
 
 dnl ========================================================================
 dnl Compiler flags
 dnl ========================================================================
 
 dnl Make sure that CFLAGS is not exported. If the user did
 dnl not have CFLAGS in their environment then this should have
 dnl no effect. However if CFLAGS was exported from the user's
 dnl environment, then the new CFLAGS will also be exported
 dnl to sub processes.
 if export | fgrep " CFLAGS=" > /dev/null; then
     SAVED_CFLAGS="$CFLAGS"
     unset CFLAGS
     CFLAGS="$SAVED_CFLAGS"
     unset SAVED_CFLAGS
 fi
 
 AC_ARG_VAR([CFLAGS_HARDENED_LIB], [extra C compiler flags for hardened libraries])
 AC_ARG_VAR([LDFLAGS_HARDENED_LIB], [extra linker flags for hardened libraries])
 
 AC_ARG_VAR([CFLAGS_HARDENED_EXE], [extra C compiler flags for hardened executables])
 AC_ARG_VAR([LDFLAGS_HARDENED_EXE], [extra linker flags for hardened executables])
 
 CC_EXTRAS=""
 
 AS_IF([test "$GCC" != yes], [CFLAGS="$CFLAGS -g"], [
     CFLAGS="$CFLAGS -ggdb"
 
 dnl When we don't have diagnostic push / pull, we can't explicitly disable
 dnl checking for nonliteral formats in the places where they occur on purpose
 dnl thus we disable nonliteral format checking globally as we are aborting
 dnl on warnings. 
 dnl what makes the things really ugly is that nonliteral format checking is 
 dnl obviously available as an extra switch in very modern gcc but for older
 dnl gcc this is part of -Wformat=2 
 dnl so if we have push/pull we can enable -Wformat=2 -Wformat-nonliteral
 dnl if we don't have push/pull but -Wformat-nonliteral we can enable -Wformat=2
 dnl otherwise none of both
 
     gcc_diagnostic_push_pull=no
     cc_temp_flags "$CFLAGS $WERROR"
     AC_MSG_CHECKING([for gcc diagnostic push / pull])
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #pragma GCC diagnostic push
 #pragma GCC diagnostic pop
                       ]])],
                       [
                           AC_MSG_RESULT([yes])
                           gcc_diagnostic_push_pull=yes
                       ], AC_MSG_RESULT([no]))
     cc_restore_flags
 
     AS_IF([cc_supports_flag "-Wformat-nonliteral"],
           [gcc_format_nonliteral=yes],
           [gcc_format_nonliteral=no])
         
     # We had to eliminate -Wnested-externs because of libtool changes
     # Make sure to order options so that the former stand for prerequisites
     # of the latter (e.g., -Wformat-nonliteral requires -Wformat).
     EXTRA_FLAGS="-fgnu89-inline"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wall"
     EXTRA_FLAGS="$EXTRA_FLAGS -Waggregate-return"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wbad-function-cast"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wcast-align"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wdeclaration-after-statement"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wendif-labels"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wfloat-equal"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wformat-security"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wmissing-prototypes"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wmissing-declarations"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wnested-externs"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wno-long-long"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wno-strict-aliasing"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wpointer-arith"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wstrict-prototypes"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wwrite-strings"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wunused-but-set-variable"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wunsigned-char"
 
     AS_IF([test "x$gcc_diagnostic_push_pull" = "xyes"],
           [
               AC_DEFINE([GCC_FORMAT_NONLITERAL_CHECKING_ENABLED], [],
                         [gcc can complain about nonliterals in format])
               EXTRA_FLAGS="$EXTRA_FLAGS -Wformat=2 -Wformat-nonliteral"
           ],
           [test "x$gcc_format_nonliteral" = "xyes"],
           [EXTRA_FLAGS="$EXTRA_FLAGS -Wformat=2"])
 
 # Additional warnings it might be nice to enable one day
 #                -Wshadow
 #                -Wunreachable-code
     for j in $EXTRA_FLAGS
     do
         AS_IF([cc_supports_flag $CC_EXTRAS $j], [CC_EXTRAS="$CC_EXTRAS $j"])
     done
 
     AC_MSG_NOTICE([Using additional gcc flags: ${CC_EXTRAS}])
 ])
 
 dnl
 dnl Hardening flags
 dnl
 dnl The prime control of whether to apply (targeted) hardening build flags and
 dnl which ones is --{enable,disable}-hardening option passed to ./configure:
 dnl
 dnl --enable-hardening=try (default):
 dnl     depending on whether any of CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE,
 dnl     CFLAGS_HARDENED_LIB or LDFLAGS_HARDENED_LIB environment variables
 dnl     (see below) is set and non-null, all these custom flags (even if not
 dnl     set) are used as are, otherwise the best effort is made to offer
 dnl     reasonably strong hardening in several categories (RELRO, PIE,
 dnl     "bind now", stack protector) according to what the selected toolchain
 dnl     can offer
 dnl
 dnl --enable-hardening:
 dnl     same effect as --enable-hardening=try when the environment variables
 dnl     in question are suppressed
 dnl
 dnl --disable-hardening:
 dnl     do not apply any targeted hardening measures at all
 dnl
 dnl The user-injected environment variables that regulate the hardening in
 dnl default case are as follows:
 dnl
 dnl * CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE
 dnl    compiler and linker flags (respectively) for daemon programs
 dnl    (pacemakerd, pacemaker-attrd, pacemaker-controld, pacemaker-execd,
 dnl    pacemaker-based, pacemaker-fenced, pacemaker-remoted,
 dnl    pacemaker-schedulerd)
 dnl
 dnl * CFLAGS_HARDENED_LIB, LDFLAGS_HARDENED_LIB
 dnl    compiler and linker flags (respectively) for libraries linked
 dnl    with the daemon programs
 dnl
 dnl Note that these are purposedly targeted variables (addressing particular
 dnl targets all over the scattered Makefiles) and have no effect outside of
 dnl the predestined scope (e.g., CLI utilities).  For a global reach,
 dnl use CFLAGS, LDFLAGS, etc. as usual.
 dnl
 dnl For guidance on the suitable flags consult, for instance:
 dnl https://fedoraproject.org/wiki/Changes/Harden_All_Packages#Detailed_Harden_Flags_Description
 dnl https://owasp.org/index.php/C-Based_Toolchain_Hardening#GCC.2FBinutils
 dnl
 
 AS_IF([test $enable_hardening -eq $OPTIONAL],
       [
           AS_IF([test "$(env | grep -Ec '^(C|LD)FLAGS_HARDENED_(EXE|LIB)=.')" = 0],
                 [enable_hardening=$REQUIRED],
                 [AC_MSG_NOTICE([Hardening: using custom flags from environment])]
           )
       ],
       [
           unset CFLAGS_HARDENED_EXE
           unset CFLAGS_HARDENED_LIB
           unset LDFLAGS_HARDENED_EXE
           unset LDFLAGS_HARDENED_LIB
       ]
 )
 AS_CASE([$enable_hardening],
         [$DISABLED], [AC_MSG_NOTICE([Hardening: explicitly disabled])],
         [$REQUIRED], [
             CFLAGS_HARDENED_EXE=
             CFLAGS_HARDENED_LIB=
             LDFLAGS_HARDENED_EXE=
             LDFLAGS_HARDENED_LIB=
             relro=0
             pie=0
             bindnow=0
             stackprot="none"
             # daemons incl. libs: partial RELRO
             flag="-Wl,-z,relro"
             CC_CHECK_LDFLAGS(["${flag}"],
                              [
                                  LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
                                  LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"
                                  relro=1
                              ])
             # daemons: PIE for both CFLAGS and LDFLAGS
             AS_IF([cc_supports_flag -fPIE],
                   [
                       flag="-pie"
                       CC_CHECK_LDFLAGS(["${flag}"],
                                        [
                                            CFLAGS_HARDENED_EXE="${CFLAGS_HARDENED_EXE} -fPIE"
                                            LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
                                            pie=1
                                        ])
                   ]
             )
             # daemons incl. libs: full RELRO if sensible + as-needed linking
             #                     so as to possibly mitigate startup performance
             #                     hit caused by excessive linking with unneeded
             #                     libraries
             AS_IF([test "${relro}" = 1 && test "${pie}" = 1],
                   [
                       flag="-Wl,-z,now"
                       CC_CHECK_LDFLAGS(["${flag}"],
                                        [
                                            LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
                                            LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"
                                            bindnow=1
                                        ])
                   ]
             )
             AS_IF([test "${bindnow}" = 1],
                   [
                       flag="-Wl,--as-needed"
                       CC_CHECK_LDFLAGS(["${flag}"],
                                        [
                                            LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
                                            LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"
                                        ])
                   ])
             # universal: prefer strong > all > default stack protector if possible
             flag=
             AS_IF([cc_supports_flag -fstack-protector-strong],
                   [
                       flag="-fstack-protector-strong"
                       stackprot="strong"
                   ],
                   [cc_supports_flag -fstack-protector-all],
                   [
                       flag="-fstack-protector-all"
                       stackprot="all"
                   ],
                   [cc_supports_flag -fstack-protector],
                   [
                       flag="-fstack-protector"
                       stackprot="default"
                   ]
             )
             AS_IF([test -n "${flag}"], [CC_EXTRAS="${CC_EXTRAS} ${flag}"])
             # universal: enable stack clash protection if possible
             AS_IF([cc_supports_flag -fstack-clash-protection],
                   [
                       CC_EXTRAS="${CC_EXTRAS} -fstack-clash-protection"
                       AS_IF([test "${stackprot}" = "none"],
                             [stackprot="clash-only"],
                             [stackprot="${stackprot}+clash"]
                       )
                   ]
             )
             # Log a summary
             AS_IF([test "${relro}" = 1 || test "${pie}" = 1 || test "${stackprot}" != "none"],
                   [AC_MSG_NOTICE(m4_normalize([Hardening:
                         relro=${relro}
                         pie=${pie}
                         bindnow=${bindnow}
                         stackprot=${stackprot}]))
                   ],
                   [AC_MSG_WARN([Hardening: no suitable features in the toolchain detected])]
             )
         ],
 )
 
 CFLAGS="$SANITIZERS_CFLAGS $CFLAGS $CC_EXTRAS"
 LDFLAGS="$SANITIZERS_LDFLAGS $LDFLAGS"
 CFLAGS_HARDENED_EXE="$SANITIZERS_CFLAGS $CFLAGS_HARDENED_EXE"
 LDFLAGS_HARDENED_EXE="$SANITIZERS_LDFLAGS $LDFLAGS_HARDENED_EXE"
 
 NON_FATAL_CFLAGS="$CFLAGS"
 AC_SUBST(NON_FATAL_CFLAGS)
 
 dnl
 dnl We reset CFLAGS to include our warnings *after* all function
 dnl checking goes on, so that our warning flags don't keep the
 dnl AC_*FUNCS() calls above from working.  In particular, -Werror will
 dnl *always* cause us troubles if we set it before here.
 dnl
 dnl
 AS_IF([test $enable_fatal_warnings -ne $DISABLED], [
     AC_MSG_NOTICE([Enabling fatal compiler warnings])
     CFLAGS="$CFLAGS $WERROR"
 ])
 AC_SUBST(CFLAGS)
 
 dnl This is useful for use in Makefiles that need to remove one specific flag
 CFLAGS_COPY="$CFLAGS"
 AC_SUBST(CFLAGS_COPY)
 
 AC_SUBST(LIBADD_DL)        dnl extra flags for dynamic linking libraries
 
 AC_SUBST(LOCALE)
 
 dnl Options for cleaning up the compiler output
 AS_IF([test $enable_quiet -ne $DISABLED],
       [
           AC_MSG_NOTICE([Suppressing make details])
           QUIET_LIBTOOL_OPTS="--silent"
           QUIET_MAKE_OPTS="-s"  # POSIX compliant
       ],
       [
           QUIET_LIBTOOL_OPTS=""
           QUIET_MAKE_OPTS=""
       ]
 )
 
 dnl Put the above variables to use
 LIBTOOL="${LIBTOOL} --tag=CC \$(QUIET_LIBTOOL_OPTS)"
 MAKEFLAGS="${MAKEFLAGS} ${QUIET_MAKE_OPTS}"
 
 # Make features list available (sorted alphabetically, without leading space)
 PCMK_FEATURES=`echo "$PCMK_FEATURES" | sed -e 's/^ //' -e 's/ /\n/g' | sort | xargs`
 AC_DEFINE_UNQUOTED(CRM_FEATURES, "$PCMK_FEATURES", Set of enabled features)
 AC_SUBST(PCMK_FEATURES)
 
 AC_SUBST(CC)
 AC_SUBST(MAKEFLAGS)
 AC_SUBST(LIBTOOL)
 AC_SUBST(QUIET_LIBTOOL_OPTS)
 
 dnl Files we output that need to be executable
 CONFIG_FILES_EXEC([cts/cts-cli],
                   [cts/cts-coverage],
                   [cts/cts-exec],
                   [cts/cts-fencing],
                   [cts/cts-regression],
                   [cts/cts-scheduler],
                   [cts/lxc_autogen.sh],
                   [cts/benchmark/clubench],
                   [cts/lab/CTSlab.py],
                   [cts/lab/OCFIPraTest.py],
                   [cts/lab/cluster_test],
                   [cts/lab/cts],
                   [cts/lab/cts-log-watcher],
                   [cts/support/LSBDummy],
                   [cts/support/cts-support],
                   [cts/support/fence_dummy],
                   [cts/support/pacemaker-cts-dummyd],
                   [daemons/fenced/fence_legacy],
                   [daemons/fenced/fence_watchdog],
                   [doc/abi-check],
                   [extra/resources/ClusterMon],
                   [extra/resources/HealthSMART],
                   [extra/resources/SysInfo],
                   [extra/resources/ifspeed],
                   [extra/resources/o2cb],
                   [maint/bumplibs],
                   [tools/crm_failcount],
                   [tools/crm_master],
                   [tools/crm_report],
                   [tools/crm_standby],
                   [tools/cibsecret],
                   [tools/pcmk_simtimes])
 
 dnl Other files we output
 AC_CONFIG_FILES(Makefile                                            \
                 cts/Makefile                                        \
                 cts/benchmark/Makefile                              \
                 cts/lab/Makefile                                    \
                 cts/lab/CTSvars.py                                  \
                 cts/scheduler/Makefile                              \
                 cts/scheduler/dot/Makefile                          \
                 cts/scheduler/exp/Makefile                          \
                 cts/scheduler/scores/Makefile                       \
                 cts/scheduler/stderr/Makefile                       \
                 cts/scheduler/summary/Makefile                      \
                 cts/scheduler/xml/Makefile                          \
                 cts/support/Makefile                                \
                 cts/support/pacemaker-cts-dummyd@.service           \
                 daemons/Makefile                                    \
                 daemons/attrd/Makefile                              \
                 daemons/based/Makefile                              \
                 daemons/controld/Makefile                           \
                 daemons/execd/Makefile                              \
                 daemons/execd/pacemaker_remote                      \
                 daemons/execd/pacemaker_remote.service              \
                 daemons/fenced/Makefile                             \
                 daemons/pacemakerd/Makefile                         \
                 daemons/pacemakerd/pacemaker.service                \
                 daemons/pacemakerd/pacemaker.upstart                \
                 daemons/pacemakerd/pacemaker.combined.upstart       \
                 daemons/schedulerd/Makefile                         \
                 devel/Makefile                                      \
                 doc/Doxyfile                                        \
                 doc/Makefile                                        \
                 doc/sphinx/Makefile                                 \
                 etc/Makefile                                        \
                 etc/init.d/pacemaker                                \
                 etc/logrotate.d/pacemaker                           \
                 extra/Makefile                                      \
                 extra/alerts/Makefile                               \
                 extra/resources/Makefile                            \
                 include/Makefile                                    \
                 include/crm/Makefile                                \
                 include/crm/cib/Makefile                            \
                 include/crm/common/Makefile                         \
                 include/crm/cluster/Makefile                        \
                 include/crm/fencing/Makefile                        \
                 include/crm/pengine/Makefile                        \
                 include/pcmki/Makefile                              \
                 replace/Makefile                                    \
                 lib/Makefile                                        \
                 lib/libpacemaker.pc                                 \
                 lib/pacemaker.pc                                    \
                 lib/pacemaker-cib.pc                                \
                 lib/pacemaker-lrmd.pc                               \
                 lib/pacemaker-service.pc                            \
                 lib/pacemaker-pe_rules.pc                           \
                 lib/pacemaker-pe_status.pc                          \
                 lib/pacemaker-fencing.pc                            \
                 lib/pacemaker-cluster.pc                            \
                 lib/common/Makefile                                 \
                 lib/common/tests/Makefile                           \
                 lib/common/tests/agents/Makefile                    \
                 lib/common/tests/cmdline/Makefile                   \
                 lib/common/tests/flags/Makefile                     \
                 lib/common/tests/io/Makefile                        \
                 lib/common/tests/iso8601/Makefile                   \
                 lib/common/tests/operations/Makefile                \
                 lib/common/tests/results/Makefile                   \
                 lib/common/tests/strings/Makefile                   \
                 lib/common/tests/utils/Makefile                     \
                 lib/common/tests/xpath/Makefile                     \
                 lib/cluster/Makefile                                \
                 lib/cib/Makefile                                    \
                 lib/gnu/Makefile                                    \
                 lib/pacemaker/Makefile                              \
                 lib/pengine/Makefile                                \
                 lib/pengine/tests/Makefile                          \
                 lib/pengine/tests/rules/Makefile                    \
                 lib/fencing/Makefile                                \
                 lib/lrmd/Makefile                                   \
                 lib/services/Makefile                               \
                 maint/Makefile                                      \
                 tests/Makefile                                      \
                 tools/Makefile                                      \
                 tools/report.collector                              \
                 tools/report.common                                 \
                 tools/crm_mon.service                               \
                 tools/crm_mon.upstart                               \
                 xml/Makefile                                        \
                 xml/pacemaker-schemas.pc                            \
 )
 
 dnl Now process the entire list of files added by previous
 dnl  calls to AC_CONFIG_FILES()
 AC_OUTPUT()
 
 dnl *****************
 dnl Configure summary
 dnl *****************
 
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([$PACKAGE configuration:])
 AC_MSG_NOTICE([  Version                  = ${VERSION} (Build: $BUILD_VERSION)])
 AC_MSG_NOTICE([  Features                 = ${PCMK_FEATURES}])
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([  Prefix                   = ${prefix}])
 AC_MSG_NOTICE([  Executables              = ${sbindir}])
 AC_MSG_NOTICE([  Man pages                = ${mandir}])
 AC_MSG_NOTICE([  Libraries                = ${libdir}])
 AC_MSG_NOTICE([  Header files             = ${includedir}])
 AC_MSG_NOTICE([  Arch-independent files   = ${datadir}])
 AC_MSG_NOTICE([  State information        = ${localstatedir}])
 AC_MSG_NOTICE([  System configuration     = ${sysconfdir}])
 AC_MSG_NOTICE([  OCF agents               = ${OCF_ROOT_DIR}])
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([  HA group name            = ${CRM_DAEMON_GROUP}])
 AC_MSG_NOTICE([  HA user name             = ${CRM_DAEMON_USER}])
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([  CFLAGS                   = ${CFLAGS}])
 AC_MSG_NOTICE([  CFLAGS_HARDENED_EXE      = ${CFLAGS_HARDENED_EXE}])
 AC_MSG_NOTICE([  CFLAGS_HARDENED_LIB      = ${CFLAGS_HARDENED_LIB}])
 AC_MSG_NOTICE([  LDFLAGS_HARDENED_EXE     = ${LDFLAGS_HARDENED_EXE}])
 AC_MSG_NOTICE([  LDFLAGS_HARDENED_LIB     = ${LDFLAGS_HARDENED_LIB}])
 AC_MSG_NOTICE([  Libraries                = ${LIBS}])
 AC_MSG_NOTICE([  Stack Libraries          = ${CLUSTERLIBS}])
 AC_MSG_NOTICE([  Unix socket auth method  = ${us_auth}])
diff --git a/cts/cts-exec.in b/cts/cts-exec.in
index a89f7477a6..96704ece7a 100644
--- a/cts/cts-exec.in
+++ b/cts/cts-exec.in
@@ -1,1268 +1,1274 @@
 #!@PYTHON@
 """ Regression tests for Pacemaker's pacemaker-execd
 """
 
 __copyright__ = "Copyright 2012-2021 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 import io
 import os
 import stat
 import sys
 import subprocess
 import shlex
 import shutil
 import tempfile
 import time
 
 # Where to find test binaries
 # Prefer the source tree if available
 BUILD_DIR = "@abs_top_builddir@"
 TEST_DIR = sys.path[0]
 
 SBIN_DIR = "@sbindir@"
 
 # Check whether Pacemaker Remote support was built
 REMOTE_ENABLED = "@PC_NAME_GNUTLS@" != ""
 
 # File permissions for executable scripts we create
 EXECMODE = stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
 
 # These values must be kept in sync with include/crm/crm.h
 class CrmExit(object):
     OK                   =   0
     ERROR                =   1
     INVALID_PARAM        =   2
     UNIMPLEMENT_FEATURE  =   3
     INSUFFICIENT_PRIV    =   4
     NOT_INSTALLED        =   5
     NOT_CONFIGURED       =   6
     NOT_RUNNING          =   7
     USAGE                =  64
     DATAERR              =  65
     NOINPUT              =  66
     NOUSER               =  67
     NOHOST               =  68
     UNAVAILABLE          =  69
     SOFTWARE             =  70
     OSERR                =  71
     OSFILE               =  72
     CANTCREAT            =  73
     IOERR                =  74
     TEMPFAIL             =  75
     PROTOCOL             =  76
     NOPERM               =  77
     CONFIG               =  78
     FATAL                = 100
     PANIC                = 101
     DISCONNECT           = 102
     SOLO                 = 103
     DIGEST               = 104
     NOSUCH               = 105
     QUORUM               = 106
     UNSAFE               = 107
     EXISTS               = 108
     MULTIPLE             = 109
     OLD                  = 110
     TIMEOUT              = 124
     MAX                  = 255
 
 
 def update_path():
     """ Set the PATH environment variable appropriately for the tests """
 
     new_path = os.environ['PATH']
     if os.path.exists("%s/cts-exec.in" % TEST_DIR):
         print("Running tests from the source tree: %s (%s)" % (BUILD_DIR, TEST_DIR))
         # For pacemaker-execd, cts-exec-helper, and pacemaker-remoted
         new_path = "%s/daemons/execd:%s" % (BUILD_DIR, new_path)
         new_path = "%s/tools:%s" % (BUILD_DIR, new_path)   # For crm_resource
         # For pacemaker-fenced
         new_path = "%s/daemons/fenced:%s" % (BUILD_DIR, new_path)
         # For cts-support
         new_path = "%s/cts/support:%s" % (BUILD_DIR, new_path)
 
     else:
         print("Running tests from the install tree: @CRM_DAEMON_DIR@ (not %s)" % TEST_DIR)
         # For cts-exec-helper, cts-support, pacemaker-execd, pacemaker-fenced,
         # and pacemaker-remoted
         new_path = "@CRM_DAEMON_DIR@:%s" % (new_path)
 
     print('Using PATH="{}"'.format(new_path))
     os.environ['PATH'] = new_path
 
 
 def output_from_command(command):
     """ Run a command, and return its standard output. """
 
     test = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
     test.wait()
     output = test.communicate()[0].decode(sys.stdout.encoding)
     return output.split("\n")
 
 
 class TestError(Exception):
     """ Base class for exceptions in this module """
     pass
 
 
 class ExitCodeError(TestError):
     """ Exception raised when command exit status is unexpected """
 
     def __init__(self, exit_code):
         self.exit_code = exit_code
 
     def __str__(self):
         return repr(self.exit_code)
 
 
 class OutputNotFoundError(TestError):
     """ Exception raised when command output does not contain wanted string """
 
     def __init__(self, output):
         self.output = output
 
     def __str__(self):
         return repr(self.output)
 
 
 class OutputFoundError(TestError):
     """ Exception raised when command output contains unwanted string """
 
     def __init__(self, output):
         self.output = output
 
     def __str__(self):
         return repr(self.output)
 
 
 class Test(object):
     """ Executor for a single pacemaker-execd regression test """
 
     def __init__(self, name, description, verbose=0, tls=0, timeout=2,
                  force_wait=0, logdir="/tmp"):
 
         self.name = name
         self.description = description
         self.cmds = []
 
         if tls:
             self.daemon_location = "pacemaker-remoted"
         else:
             self.daemon_location = "pacemaker-execd"
 
         self.logpath = os.path.join(logdir, self.daemon_location + ".log")
         self.test_tool_location = "cts-exec-helper"
         self.verbose = verbose
         self.tls = tls
         self.timeout = timeout
         self.force_wait = force_wait
 
         self.result_txt = ""
         self.cmd_tool_output = ""
         self.result_exitcode = CrmExit.OK
 
         self.execd_process = None
         self.stonith_process = None
 
         self.executed = 0
 
     def __new_cmd(self, cmd, args, exitcode, stdout_match="", no_wait=0, stdout_negative_match="", kill=None):
         """ Add a command to be executed as part of this test """
 
         if self.verbose and cmd == self.test_tool_location:
             args = args + " -V "
 
         if (cmd == self.test_tool_location) and self.tls:
             args = args + " -S "
 
         self.cmds.append(
             {
                 "cmd" : cmd,
                 "kill" : kill,
                 "args" : args,
                 "expected_exitcode" : exitcode,
                 "stdout_match" : stdout_match,
                 "stdout_negative_match" : stdout_negative_match,
                 "no_wait" : no_wait,
                 "cmd_output" : "",
             }
         )
 
     def start_environment(self):
         """ Prepare the host for running a test """
 
         ### make sure we are in full control here ###
         cmd = shlex.split("killall -q -9 pacemaker-fenced lt-pacemaker-fenced pacemaker-execd lt-pacemaker-execd cts-exec-helper lt-cts-exec-helper pacemaker-remoted")
         test = subprocess.Popen(cmd, stdout=subprocess.PIPE)
         test.wait()
 
         additional_args = ""
 
         if self.tls == 0:
             self.stonith_process = subprocess.Popen(shlex.split("pacemaker-fenced -s"))
 
         if self.verbose:
             additional_args = additional_args + " -V"
 
         self.execd_process = subprocess.Popen(shlex.split("%s %s -l %s"
                                                          % (self.daemon_location, additional_args, self.logpath)))
 
         logfile = None
 
         init_time = time.time()
         update_time = init_time
 
         while True:
             time.sleep(0.1)
 
             if self.force_wait == 0 and logfile == None \
                and os.path.exists(self.logpath):
                 logfile = io.open(self.logpath, 'rt', encoding = "ISO-8859-1")
 
             if self.force_wait == 0 and logfile != None:
                 for line in logfile.readlines():
                     if "successfully started" in line:
                         return
 
             now = time.time()
 
             if self.timeout > 0 and (now - init_time) >= self.timeout:
                 if self.force_wait == 0:
                     print("\tDaemon %s doesn't seem to have been initialized within %fs."
                           "\n\tConsider specifying a longer '--timeout' value."
                           %(self.daemon_location, self.timeout))
                 return
 
             if self.verbose and (now - update_time) >= 5:
                 print("Waiting for %s to be initialized: %fs ..."
                       %(self.daemon_location, now - init_time))
                 update_time = now
 
     def clean_environment(self):
         """ Clean up the host after running a test """
 
         if self.execd_process:
             self.execd_process.terminate()
             self.execd_process.wait()
 
             if self.verbose:
                 print("Daemon output")
                 logfile = io.open(self.logpath, 'rt', errors='replace')
                 for line in logfile:
                     print(line.strip().encode('utf-8', 'replace'))
 
         if self.stonith_process:
             self.stonith_process.terminate()
             self.stonith_process.wait()
 
         self.execd_process = None
         self.stonith_process = None
 
     def add_sys_cmd(self, cmd, args):
         """ Add a simple command to be executed as part of this test """
 
         self.__new_cmd(cmd, args, CrmExit.OK, "")
 
     def add_cmd_check_stdout(self, args, match, no_match=""):
         """ Add a command with expected output to be executed as part of this test """
 
         self.__new_cmd(self.test_tool_location, args, CrmExit.OK, match, 0, no_match)
 
     def add_cmd(self, args):
         """ Add a cts-exec-helper command to be executed as part of this test """
 
         self.__new_cmd(self.test_tool_location, args, CrmExit.OK, "")
 
     def add_cmd_and_kill(self, kill_proc, args):
         """ Add a cts-exec-helper command and system command to be executed as part of this test """
 
         self.__new_cmd(self.test_tool_location, args, CrmExit.OK, "", kill=kill_proc)
 
     def add_expected_fail_cmd(self, args, exitcode=CrmExit.ERROR):
         """ Add a cts-exec-helper command to be executed as part of this test and expected to fail """
 
         self.__new_cmd(self.test_tool_location, args, exitcode, "")
 
     def get_exitcode(self):
         """ Return the exit status of the last test execution """
 
         return self.result_exitcode
 
     def print_result(self, filler):
         """ Print the result of the last test execution """
 
         print("%s%s" % (filler, self.result_txt))
 
     def run_cmd(self, args):
         """ Execute a command as part of this test """
 
         cmd = shlex.split(args['args'])
         cmd.insert(0, args['cmd'])
         if self.verbose:
             print("\n\nRunning: "+" ".join(cmd))
         test = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 
         if args['kill']:
             if self.verbose:
                 print("Also running: "+args['kill'])
             ### Typically, the kill argument is used to detect some sort of
             ### failure. Without yielding for a few seconds here, the process
             ### launched earlier that is listening for the failure may not have
             ### time to connect to pacemaker-execd.
             time.sleep(2)
             subprocess.Popen(shlex.split(args['kill']))
 
         if args['no_wait'] == 0:
             test.wait()
         else:
             return CrmExit.OK
 
         output = test.communicate()[0].decode(sys.stdout.encoding)
         args['cmd_output'] = output
 
         if test.returncode != args['expected_exitcode']:
             raise ExitCodeError(test.returncode)
 
         if args['stdout_match'] != "" and output.count(args['stdout_match']) == 0:
             raise OutputNotFoundError(output)
 
         if args['stdout_negative_match'] != "" and output.count(args['stdout_negative_match']) != 0:
             raise OutputFoundError(output)
 
     def set_error(self, step, cmd):
         """ Record failure of this test """
 
         msg = "FAILURE - '%s' failed at step %d. Command: %s %s"
         self.result_txt = msg % (self.name, step, cmd['cmd'], cmd['args'])
         self.result_exitcode = CrmExit.ERROR
 
     def run(self):
         """ Execute this test. """
 
         res = 0
         i = 1
 
         if self.tls and self.name.count("stonith") != 0:
             self.result_txt = "SKIPPED - '%s' - disabled when testing pacemaker_remote" % (self.name)
             print(self.result_txt)
             return res
 
         self.start_environment()
 
         if self.verbose:
             print("\n--- START TEST - %s" % self.name)
 
         self.result_txt = "SUCCESS - '%s'" % (self.name)
         self.result_exitcode = CrmExit.OK
         for cmd in self.cmds:
             try:
                 self.run_cmd(cmd)
             except ExitCodeError as e:
                 print(cmd['cmd_output'])
                 print("Step %d FAILED - command returned %s, expected %d" % (i, e, cmd['expected_exitcode']))
                 self.set_error(i, cmd);
                 break
             except OutputNotFoundError as e:
                 print("Step %d FAILED - '%s' was not found in command output: %s" % (i, cmd['stdout_match'], e))
                 self.set_error(i, cmd);
                 break
             except OutputFoundError as e:
                 print("Step %d FAILED - '%s' was found in command output: %s" % (i, cmd['stdout_negative_match'], e))
                 self.set_error(i, cmd);
                 break
             if self.verbose:
                 print(cmd['cmd_output'].strip())
                 print("Step %d SUCCESS" % (i))
             i = i + 1
         self.clean_environment()
 
         print(self.result_txt)
         if self.verbose:
             print("--- END TEST - %s\n" % self.name)
 
         self.executed = 1
         return res
 
 class Tests(object):
     """ Collection of all pacemaker-execd regression tests """
 
     def __init__(self, verbose=0, tls=0, timeout=2, force_wait=0,
                  logdir="/tmp"):
 
         self.tests = []
         self.verbose = verbose
         self.tls = tls
         self.timeout = timeout
         self.force_wait = force_wait
         self.logdir = logdir
         self.rsc_classes = output_from_command("crm_resource --list-standards")
         self.rsc_classes = self.rsc_classes[:-1] # Strip trailing empty line
         self.installed_files = []
         self.action_timeout = " -t 9000 "
         if self.tls:
             self.rsc_classes.remove("stonith")
+
+        try:
+            self.rsc_classes.remove("nagios")
+        except ValueError: # Not found
+            pass
+
         if "systemd" in self.rsc_classes:
             try:
                 # This code doesn't need this import, but pacemaker-cts-dummyd
                 # does, so ensure the dependency is available rather than cause
                 # all systemd tests to fail.
                 import systemd.daemon
             except ImportError:
                 print("Python systemd bindings not found.")
                 print("The tests for systemd class are not going to be run.")
                 self.rsc_classes.remove("systemd")
 
         print("Testing resource classes", repr(self.rsc_classes))
 
         self.common_cmds = {
             "ocf_reg_line"      : "-c register_rsc -r ocf_test_rsc "+self.action_timeout+" -C ocf -P pacemaker -T Dummy",
             "ocf_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:ocf_test_rsc action:none rc:ok op_status:complete\"",
             "ocf_unreg_line"    : "-c unregister_rsc -r \"ocf_test_rsc\" "+self.action_timeout,
             "ocf_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:ocf_test_rsc action:none rc:ok op_status:complete\"",
             "ocf_start_line"    : "-c exec -r \"ocf_test_rsc\" -a \"start\" "+self.action_timeout,
             "ocf_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:start rc:ok op_status:complete\" ",
             "ocf_stop_line"     : "-c exec -r \"ocf_test_rsc\" -a \"stop\" "+self.action_timeout,
             "ocf_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:stop rc:ok op_status:complete\" ",
             "ocf_monitor_line"  : '-c exec -r ocf_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "ocf_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout,
             "ocf_cancel_line"   : '-c cancel -r ocf_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "ocf_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
 
             "systemd_reg_line"      : "-c register_rsc -r systemd_test_rsc " +
                                       self.action_timeout +
                                       " -C systemd -T pacemaker-cts-dummyd@3",
             "systemd_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:systemd_test_rsc action:none rc:ok op_status:complete\"",
             "systemd_unreg_line"    : "-c unregister_rsc -r \"systemd_test_rsc\" "+self.action_timeout,
             "systemd_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:systemd_test_rsc action:none rc:ok op_status:complete\"",
             "systemd_start_line"    : "-c exec -r \"systemd_test_rsc\" -a \"start\" "+self.action_timeout,
             "systemd_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:start rc:ok op_status:complete\" ",
             "systemd_stop_line"     : "-c exec -r \"systemd_test_rsc\" -a \"stop\" "+self.action_timeout,
             "systemd_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:stop rc:ok op_status:complete\" ",
             "systemd_monitor_line"  : '-c exec -r systemd_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "systemd_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:monitor rc:ok op_status:complete\" -t 15000 ",
             "systemd_cancel_line"   : '-c cancel -r systemd_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "systemd_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
 
             "upstart_reg_line"      : "-c register_rsc -r upstart_test_rsc "+self.action_timeout+" -C upstart -T pacemaker-cts-dummyd",
             "upstart_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:upstart_test_rsc action:none rc:ok op_status:complete\"",
             "upstart_unreg_line"    : "-c unregister_rsc -r \"upstart_test_rsc\" "+self.action_timeout,
             "upstart_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:upstart_test_rsc action:none rc:ok op_status:complete\"",
             "upstart_start_line"    : "-c exec -r \"upstart_test_rsc\" -a \"start\" "+self.action_timeout,
             "upstart_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:start rc:ok op_status:complete\" ",
             "upstart_stop_line"     : "-c exec -r \"upstart_test_rsc\" -a \"stop\" "+self.action_timeout,
             "upstart_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:stop rc:ok op_status:complete\" ",
             "upstart_monitor_line"  : '-c exec -r upstart_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "upstart_monitor_event" : '-l "NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:monitor rc:ok op_status:complete" -t 15000',
             "upstart_cancel_line"   : '-c cancel -r upstart_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "upstart_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
 
             "service_reg_line"      : "-c register_rsc -r service_test_rsc "+self.action_timeout+" -C service -T LSBDummy",
             "service_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:service_test_rsc action:none rc:ok op_status:complete\"",
             "service_unreg_line"    : "-c unregister_rsc -r \"service_test_rsc\" "+self.action_timeout,
             "service_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:service_test_rsc action:none rc:ok op_status:complete\"",
             "service_start_line"    : "-c exec -r \"service_test_rsc\" -a \"start\" "+self.action_timeout,
             "service_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:start rc:ok op_status:complete\" ",
             "service_stop_line"     : "-c exec -r \"service_test_rsc\" -a \"stop\" "+self.action_timeout,
             "service_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:stop rc:ok op_status:complete\" ",
             "service_monitor_line"  : '-c exec -r service_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "service_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout,
             "service_cancel_line"   : '-c cancel -r service_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "service_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
 
             "lsb_reg_line"      : "-c register_rsc -r lsb_test_rsc "+self.action_timeout+" -C lsb -T LSBDummy",
             "lsb_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:lsb_test_rsc action:none rc:ok op_status:complete\" ",
             "lsb_unreg_line"    : "-c unregister_rsc -r \"lsb_test_rsc\" "+self.action_timeout,
             "lsb_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:lsb_test_rsc action:none rc:ok op_status:complete\"",
             "lsb_start_line"    : "-c exec -r \"lsb_test_rsc\" -a \"start\" "+self.action_timeout,
             "lsb_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:start rc:ok op_status:complete\" ",
             "lsb_stop_line"     : "-c exec -r \"lsb_test_rsc\" -a \"stop\" "+self.action_timeout,
             "lsb_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:stop rc:ok op_status:complete\" ",
             "lsb_monitor_line"  : '-c exec -r lsb_test_rsc -a status -i 2s ' + self.action_timeout,
             "lsb_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:status rc:ok op_status:complete\" "+self.action_timeout,
             "lsb_cancel_line"   : '-c cancel -r lsb_test_rsc -a status -i 2s ' + self.action_timeout,
             "lsb_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:status rc:ok op_status:Cancelled\" ",
 
             "stonith_reg_line"      : "-c register_rsc -r stonith_test_rsc " + self.action_timeout +
 				      " -C stonith -P pacemaker -T fence_dummy",
             "stonith_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:stonith_test_rsc action:none rc:ok op_status:complete\" ",
             "stonith_unreg_line"    : "-c unregister_rsc -r \"stonith_test_rsc\" "+self.action_timeout,
             "stonith_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:stonith_test_rsc action:none rc:ok op_status:complete\"",
             "stonith_start_line"    : '-c exec -r stonith_test_rsc -a start ' + self.action_timeout,
             "stonith_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:start rc:ok op_status:complete\" ",
             "stonith_stop_line"     : "-c exec -r \"stonith_test_rsc\" -a \"stop\" "+self.action_timeout,
             "stonith_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:stop rc:ok op_status:complete\" ",
             "stonith_monitor_line"  : '-c exec -r stonith_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "stonith_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout,
             "stonith_cancel_line"   : '-c cancel -r stonith_test_rsc -a monitor -i 2s ' + self.action_timeout,
             "stonith_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
         }
 
     def new_test(self, name, description):
         """ Create a named test """
 
         test = Test(name, description, self.verbose, self.tls, self.timeout,
                     self.force_wait, self.logdir)
         self.tests.append(test)
         return test
 
     def setup_test_environment(self):
         """ Prepare the host before executing any tests """
 
         if REMOTE_ENABLED:
             os.system("service pacemaker_remote stop")
         self.cleanup_test_environment()
 
         if self.tls and not os.path.isfile("/etc/pacemaker/authkey"):
             print("Installing /etc/pacemaker/authkey ...")
             os.system("mkdir -p /etc/pacemaker")
             os.system("dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1")
             self.installed_files.append("/etc/pacemaker/authkey")
 
         # If we're in build directory, install agents if not already installed
         if os.path.exists("%s/cts/cts-exec.in" % BUILD_DIR):
 
             if not os.path.exists("@OCF_RA_INSTALL_DIR@/pacemaker"):
                 # @TODO remember which components were created and remove them
                 os.makedirs("@OCF_RA_INSTALL_DIR@/pacemaker", 0o755)
 
             for agent in ["Dummy", "Stateful", "ping"]:
                 agent_source = "%s/extra/resources/%s" % (BUILD_DIR, agent)
                 agent_dest = "@OCF_RA_INSTALL_DIR@/pacemaker/%s" % (agent)
                 if not os.path.exists(agent_dest):
                     print("Installing %s ..." % (agent_dest))
                     shutil.copyfile(agent_source, agent_dest)
                     os.chmod(agent_dest, EXECMODE)
                     self.installed_files.append(agent_dest)
 
         subprocess.call(["cts-support", "install"])
 
     def cleanup_test_environment(self):
         """ Clean up the host after executing desired tests """
 
         for installed_file in self.installed_files:
             print("Removing %s ..." % (installed_file))
             os.remove(installed_file)
 
         subprocess.call(["cts-support", "uninstall"])
 
     def build_generic_tests(self):
         """ Register tests that apply to all resource classes """
 
         common_cmds = self.common_cmds
 
         ### register/unregister tests ###
         for rsc in self.rsc_classes:
             test = self.new_test("generic_registration_%s" % (rsc),
                                  "Simple resource registration test for %s standard" % (rsc))
             test.add_cmd(common_cmds["%s_reg_line" % (rsc)] + " " + common_cmds["%s_reg_event" % (rsc)])
             test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
 
         ### start/stop tests  ###
         for rsc in self.rsc_classes:
             test = self.new_test("generic_start_stop_%s" % (rsc), "Simple start and stop test for %s standard" % (rsc))
             test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
             test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
             test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
             test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
 
         ### monitor cancel test ###
         for rsc in self.rsc_classes:
             test = self.new_test("generic_monitor_cancel_%s" % (rsc),
                                  "Simple monitor cancel test for %s standard" % (rsc))
             test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
             test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
             test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
             ### If this fails, that means the monitor may not be getting rescheduled ####
             test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
             ### If this fails, that means the monitor may not be getting rescheduled ####
             test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
             test.add_cmd(common_cmds["%s_cancel_line" % (rsc)] + " " + common_cmds["%s_cancel_event" % (rsc)])
             ### If this happens the monitor did not actually cancel correctly. ###
             test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
             ### If this happens the monitor did not actually cancel correctly. ###
             test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
             test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
             test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
 
         ### monitor duplicate test ###
         for rsc in self.rsc_classes:
             test = self.new_test("generic_monitor_duplicate_%s" % (rsc),
                                  "Test creation and canceling of duplicate monitors for %s standard" % (rsc))
             test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
             test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
             test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
             ### If this fails, that means the monitor may not be getting rescheduled ####
             test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
             ### If this fails, that means the monitor may not be getting rescheduled ####
             test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
 
             # Add the duplicate monitors
             test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
             test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
             test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
             test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
             # verify we still get update events
             ### If this fails, that means the monitor may not be getting rescheduled ####
             test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
 
             # cancel the monitor, if the duplicate merged with the original, we should no longer see monitor updates
             test.add_cmd(common_cmds["%s_cancel_line" % (rsc)] + " " + common_cmds["%s_cancel_event" % (rsc)])
             ### If this happens the monitor did not actually cancel correctly. ###
             test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
             ### If this happens the monitor did not actually cancel correctly. ###
             test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
             test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
             test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
 
         ### stop implies cancel test ###
         for rsc in self.rsc_classes:
             test = self.new_test("generic_stop_implies_cancel_%s" % (rsc),
                                  "Verify stopping a resource implies cancel of recurring ops for %s standard" % (rsc))
             test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
             test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
             test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
             ### If this fails, that means the monitor may not be getting rescheduled ####
             test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
             ### If this fails, that means the monitor may not be getting rescheduled ####
             test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
             test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
             ### If this happens the monitor did not actually cancel correctly. ###
             test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
             ### If this happens the monitor did not actually cancel correctly. ###
             test.add_expected_fail_cmd(common_cmds["%s_monitor_event" % (rsc)], CrmExit.TIMEOUT)
             test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
 
 
     def build_multi_rsc_tests(self):
         """ Register complex tests that involve managing multiple resouces of different types """
 
         common_cmds = self.common_cmds
         # do not use service and systemd at the same time, it is the same resource.
 
         ### register start monitor stop unregister resources of each type at the same time. ###
         test = self.new_test("multi_rsc_start_stop_all",
                              "Start, monitor, and stop resources of multiple types and classes")
         for rsc in self.rsc_classes:
             test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
         for rsc in self.rsc_classes:
             test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
         for rsc in self.rsc_classes:
             test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
         for rsc in self.rsc_classes:
             ### If this fails, that means the monitor is not being rescheduled ####
             test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
         for rsc in self.rsc_classes:
             test.add_cmd(common_cmds["%s_cancel_line" % (rsc)] + " " + common_cmds["%s_cancel_event" % (rsc)])
         for rsc in self.rsc_classes:
             test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
         for rsc in self.rsc_classes:
             test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
 
     def build_negative_tests(self):
         """ Register tests related to how pacemaker-execd handles failures """
 
         ### ocf start timeout test  ###
         test = self.new_test("ocf_start_timeout", "Force start timeout to occur, verify start failure.")
         test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "
                      + self.action_timeout +
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
         # -t must be less than self.action_timeout
         test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" -k \"op_sleep\" -v \"5\" -t 1000 -w")
         test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:error op_status:Timed Out" '
                      + self.action_timeout)
         test.add_cmd("-c exec -r test_rsc -a stop " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
         test.add_cmd("-c unregister_rsc -r test_rsc " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### stonith start timeout test  ###
         test = self.new_test("stonith_start_timeout", "Force start timeout to occur, verify start failure.")
         test.add_cmd('-c register_rsc -r test_rsc ' +
                      '-C stonith -P pacemaker -T fence_dummy ' +
                      self.action_timeout +
                      '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete"')
         test.add_cmd('-c exec -r test_rsc -a start -k monitor_delay -v 30 ' +
                      '-t 1000 -w') # -t must be less than self.action_timeout
         test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:error op_status:Timed Out" '
                      + self.action_timeout)
         test.add_cmd("-c exec -r test_rsc -a stop " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
         test.add_cmd("-c unregister_rsc -r test_rsc " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### stonith component fail ###
         common_cmds = self.common_cmds
         test = self.new_test("stonith_component_fail", "Kill stonith component after pacemaker-execd connects")
         test.add_cmd(common_cmds["stonith_reg_line"]   + " " + common_cmds["stonith_reg_event"])
         test.add_cmd(common_cmds["stonith_start_line"] + " " + common_cmds["stonith_start_event"])
 
         test.add_cmd('-c exec -r stonith_test_rsc -a monitor -i 600s '
                      '-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:complete" '
                      + self.action_timeout)
 
         test.add_cmd_and_kill("killall -9 -q pacemaker-fenced lt-pacemaker-fenced",
                               '-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:error op_status:error" -t 15000')
         test.add_cmd(common_cmds["stonith_unreg_line"] + " " + common_cmds["stonith_unreg_event"])
 
 
         ### monitor fail for ocf resources ###
         test = self.new_test("monitor_fail_ocf", "Force ocf monitor to fail, verify failure is reported.")
         test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "
                      + self.action_timeout +
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
         test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
         test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
         test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
                      + self.action_timeout +
                      '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"')
         test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"'
                      + self.action_timeout)
         test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"'
                      + self.action_timeout)
         test.add_cmd_and_kill("rm -f @localstatedir@/run/Dummy-test_rsc.state",
                               '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete" ' + self.action_timeout)
         test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s ' + self.action_timeout +
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
         test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "
                                    + self.action_timeout, CrmExit.TIMEOUT)
         test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "
                                    + self.action_timeout, CrmExit.TIMEOUT)
         test.add_cmd("-c unregister_rsc -r \"test_rsc\" "
                      + self.action_timeout +
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### verify notify changes only for monitor operation.  ###
         test = self.new_test("monitor_changes_only", "Verify when flag is set, only monitor changes are notified.")
         test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
         test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+" -o "
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
         test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
                      + self.action_timeout +
                      ' -o -l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete" ')
         test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
         test.add_cmd_and_kill('rm -f @localstatedir@/run/Dummy-test_rsc.state', '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete"' + self.action_timeout)
         test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s'
                      + self.action_timeout +
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
         test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
         test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
         test.add_cmd('-c unregister_rsc -r "test_rsc" ' + self.action_timeout +
                      '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete"')
 
         ### monitor fail for systemd resource ###
         if "systemd" in self.rsc_classes:
             test = self.new_test("monitor_fail_systemd", "Force systemd monitor to fail, verify failure is reported..")
             test.add_cmd("-c register_rsc -r \"test_rsc\" -C systemd -T pacemaker-cts-dummyd@3 " +
                          self.action_timeout +
                          "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
             test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
             test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
             test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
                          + self.action_timeout +
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
             test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
             test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
             test.add_cmd_and_kill("pkill -9 -f pacemaker-cts-dummyd",
                                   '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete"' + self.action_timeout)
             test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s' + self.action_timeout +
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
             test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
             test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
             test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### monitor fail for upstart resource ###
         if "upstart" in self.rsc_classes:
             test = self.new_test("monitor_fail_upstart", "Force upstart monitor to fail, verify failure is reported..")
             test.add_cmd("-c register_rsc -r \"test_rsc\" -C upstart -T pacemaker-cts-dummyd "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
             test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
             test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
             test.add_cmd('-c exec -r test_rsc -a monitor -i 1s ' + self.action_timeout +
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
             test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
             test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
             test.add_cmd_and_kill('killall -9 -q dd', '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete"' + self.action_timeout)
             test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s'
                          + self.action_timeout +
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
             test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
             test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
             test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### Cancel non-existent operation on a resource ###
         test = self.new_test("cancel_non_existent_op", "Attempt to cancel the wrong monitor operation, verify expected failure")
         test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
         test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
         test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
         test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
                      + self.action_timeout +
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
         test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout)
         ### interval is wrong, should fail
         test.add_expected_fail_cmd('-c cancel -r test_rsc -a monitor -i 2s' + self.action_timeout +
                                    "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
         ### action name is wrong, should fail
         test.add_expected_fail_cmd('-c cancel -r test_rsc -a stop -i 1s' + self.action_timeout +
                                    "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
         test.add_cmd("-c unregister_rsc -r \"test_rsc\" " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### Attempt to invoke non-existent rsc id ###
         test = self.new_test("invoke_non_existent_rsc", "Attempt to perform operations on a non-existent rsc id.")
         test.add_expected_fail_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                                    "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:error op_status:complete\" ")
         test.add_expected_fail_cmd("-c exec -r test_rsc -a stop "+self.action_timeout+
                                    "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
         test.add_expected_fail_cmd('-c exec -r test_rsc -a monitor -i 6s '
                                    + self.action_timeout +
                                    "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
         test.add_expected_fail_cmd("-c cancel -r test_rsc -a start "+self.action_timeout+
                                    "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Cancelled\" ")
         test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### Register and start a resource that doesn't exist, systemd  ###
         if "systemd" in self.rsc_classes:
             test = self.new_test("start_uninstalled_systemd", "Register uninstalled systemd agent, try to start, verify expected failure")
             test.add_cmd("-c register_rsc -r \"test_rsc\" -C systemd -T this_is_fake1234 "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
             test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
             test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         if "upstart" in self.rsc_classes:
             test = self.new_test("start_uninstalled_upstart", "Register uninstalled upstart agent, try to start, verify expected failure")
             test.add_cmd("-c register_rsc -r \"test_rsc\" -C upstart -T this_is_fake1234 "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
             test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
             test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
                          "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### Register and start a resource that doesn't exist, ocf ###
         test = self.new_test("start_uninstalled_ocf", "Register uninstalled ocf agent, try to start, verify expected failure.")
         test.add_cmd("-c register_rsc -r \"test_rsc\" -C ocf -P pacemaker -T this_is_fake1234 "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
         test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
         test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### Register ocf with non-existent provider  ###
         test = self.new_test("start_ocf_bad_provider", "Register ocf agent with a non-existent provider, verify expected failure.")
         test.add_cmd("-c register_rsc -r \"test_rsc\" -C ocf -P pancakes -T Dummy "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
         test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
         test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### Register ocf with empty provider field  ###
         test = self.new_test("start_ocf_no_provider", "Register ocf agent with a no provider, verify expected failure.")
         test.add_expected_fail_cmd("-c register_rsc -r \"test_rsc\" -C ocf -T Dummy "+self.action_timeout+
                                    "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
         test.add_expected_fail_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                                    "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Error\" ")
         test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
     def build_stress_tests(self):
         """ Register stress tests """
 
         timeout = "-t 20000"
 
         iterations = 25
         test = self.new_test("ocf_stress", "Verify OCF agent handling works under load")
         for i in range(iterations):
             test.add_cmd("-c register_rsc -r rsc_%s %s -C ocf -P heartbeat -T Dummy -l \"NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
             test.add_cmd("-c exec -r rsc_%s -a start %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:complete\"" % (i, timeout, i))
             test.add_cmd('-c exec -r rsc_%s -a monitor %s -i 1s '
                          '-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:complete"' % (i, timeout, i))
         for i in range(iterations):
             test.add_cmd("-c exec -r rsc_%s -a stop %s  -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:complete\"" % (i, timeout, i))
             test.add_cmd("-c unregister_rsc -r rsc_%s %s -l \"NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
 
 
         if "systemd" in self.rsc_classes:
             test = self.new_test("systemd_stress", "Verify systemd dbus connection works under load")
             for i in range(iterations):
                 test.add_cmd("-c register_rsc -r rsc_%s %s -C systemd -T pacemaker-cts-dummyd@3 -l \"NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
                 test.add_cmd("-c exec -r rsc_%s -a start %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:complete\"" % (i, timeout, i))
                 test.add_cmd('-c exec -r rsc_%s -a monitor %s -i 1s '
                              '-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:complete"' % (i, timeout, i))
 
             for i in range(iterations):
                 test.add_cmd("-c exec -r rsc_%s -a stop %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:complete\"" % (i, timeout, i))
                 test.add_cmd("-c unregister_rsc -r rsc_%s %s -l \"NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
 
         iterations = 9
         timeout = "-t 30000"
         ### Verify recurring op in-flight collision is handled in series properly
         test = self.new_test("rsc_inflight_collision", "Verify recurring ops do not collide with other operations for the same rsc.")
         test.add_cmd("-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy "
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self.action_timeout)
         test.add_cmd("-c exec -r test_rsc -a start %s -k op_sleep -v 1 -l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\"" % (timeout))
         for i in range(iterations):
             test.add_cmd('-c exec -r test_rsc -a monitor %s -i 100%dms '
                          '-k op_sleep -v 2 '
                          '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"' % (timeout, i))
 
         test.add_cmd("-c exec -r test_rsc -a stop %s -l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\"" % (timeout))
         test.add_cmd("-c unregister_rsc -r test_rsc %s -l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\"" % (timeout))
 
     def build_custom_tests(self):
         """ Register tests that target specific cases """
 
         ### verify resource temporary folder is created and used by OCF agents.  ###
         test = self.new_test("rsc_tmp_dir", "Verify creation and use of rsc temporary state directory")
         test.add_sys_cmd("ls", "-al @CRM_RSCTMP_DIR@")
         test.add_cmd("-c register_rsc -r test_rsc -P heartbeat -C ocf -T Dummy "
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self.action_timeout)
         test.add_cmd("-c exec -r test_rsc -a start -t 4000")
         test.add_sys_cmd("ls", "-al @CRM_RSCTMP_DIR@")
         test.add_sys_cmd("ls", "@CRM_RSCTMP_DIR@/Dummy-test_rsc.state")
         test.add_cmd("-c exec -r test_rsc -a stop -t 4000")
         test.add_cmd("-c unregister_rsc -r test_rsc "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### start delay then stop test ###
         test = self.new_test("start_delay", "Verify start delay works as expected.")
         test.add_cmd("-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy "
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self.action_timeout)
         test.add_cmd("-c exec -r test_rsc -s 6000 -a start -w -t 6000")
         test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" -t 2000", CrmExit.TIMEOUT)
         test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" -t 6000")
         test.add_cmd("-c exec -r test_rsc -a stop " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
         test.add_cmd("-c unregister_rsc -r test_rsc " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### start delay, but cancel before it gets a chance to start.  ###
         test = self.new_test("start_delay_cancel", "Using start_delay, start a rsc, but cancel the start op before execution.")
         test.add_cmd("-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy "
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self.action_timeout)
         test.add_cmd("-c exec -r test_rsc -s 5000 -a start -w -t 4000")
         test.add_cmd("-c cancel -r test_rsc -a start " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Cancelled\" ")
         test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" -t 5000", CrmExit.TIMEOUT)
         test.add_cmd("-c unregister_rsc -r test_rsc " + self.action_timeout +
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### Register a bunch of resources, verify we can get info on them ###
         test = self.new_test("verify_get_rsc_info", "Register multiple resources, verify retrieval of rsc info.")
         if "systemd" in self.rsc_classes:
             test.add_cmd("-c register_rsc -r rsc1 -C systemd -T pacemaker-cts-dummyd@3 "+self.action_timeout)
             test.add_cmd("-c get_rsc_info -r rsc1 ")
             test.add_cmd("-c unregister_rsc -r rsc1 "+self.action_timeout)
             test.add_expected_fail_cmd("-c get_rsc_info -r rsc1 ")
 
         if "upstart" in self.rsc_classes:
             test.add_cmd("-c register_rsc -r rsc1 -C upstart -T pacemaker-cts-dummyd "+self.action_timeout)
             test.add_cmd("-c get_rsc_info -r rsc1 ")
             test.add_cmd("-c unregister_rsc -r rsc1 "+self.action_timeout)
             test.add_expected_fail_cmd("-c get_rsc_info -r rsc1 ")
 
         test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker "+self.action_timeout)
         test.add_cmd("-c get_rsc_info -r rsc2 ")
         test.add_cmd("-c unregister_rsc -r rsc2 "+self.action_timeout)
         test.add_expected_fail_cmd("-c get_rsc_info -r rsc2 ")
 
         ### Register duplicate, verify only one entry exists and can still be removed.
         test = self.new_test("duplicate_registration", "Register resource multiple times, verify only one entry exists and can be removed.")
         test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker "+self.action_timeout)
         test.add_cmd_check_stdout("-c get_rsc_info -r rsc2 ", "id:rsc2 class:ocf provider:pacemaker type:Dummy")
         test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker "+self.action_timeout)
         test.add_cmd_check_stdout("-c get_rsc_info -r rsc2 ", "id:rsc2 class:ocf provider:pacemaker type:Dummy")
         test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Stateful -P pacemaker "+self.action_timeout)
         test.add_cmd_check_stdout("-c get_rsc_info -r rsc2 ", "id:rsc2 class:ocf provider:pacemaker type:Stateful")
         test.add_cmd("-c unregister_rsc -r rsc2 "+self.action_timeout)
         test.add_expected_fail_cmd("-c get_rsc_info -r rsc2 ")
 
         ### verify the option to only send notification to the original client. ###
         test = self.new_test("notify_orig_client_only", "Verify option to only send notifications to the client originating the action.")
         test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
         test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
         test.add_cmd('-c exec -r \"test_rsc\" -a \"monitor\" -i 1s '
                      + self.action_timeout + ' -n '
                      '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"')
         # this will fail because the monitor notifications should only go to the original caller, which no longer exists.
         test.add_expected_fail_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self.action_timeout, CrmExit.TIMEOUT)
         test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s -t 6000 ')
         test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self.action_timeout+
                      "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
 
         ### get metadata ###
         test = self.new_test("get_ocf_metadata", "Retrieve metadata for a resource")
         test.add_cmd_check_stdout("-c metadata -C \"ocf\" -P \"pacemaker\" -T \"Dummy\"",
                                   "resource-agent name=\"Dummy\"")
         test.add_cmd("-c metadata -C \"ocf\" -P \"pacemaker\" -T \"Stateful\"")
         test.add_expected_fail_cmd("-c metadata -P \"pacemaker\" -T \"Stateful\"")
         test.add_expected_fail_cmd("-c metadata -C \"ocf\" -P \"pacemaker\" -T \"fake_agent\"")
 
         ### get metadata ###
         test = self.new_test("get_lsb_metadata", "Retrieve metadata for a resource")
         test.add_cmd_check_stdout("-c metadata -C \"lsb\" -T \"LSBDummy\"",
                                   "resource-agent name='LSBDummy'")
 
         ### get stonith metadata ###
         test = self.new_test("get_stonith_metadata", "Retrieve stonith metadata for a resource")
         test.add_cmd_check_stdout("-c metadata -C \"stonith\" -P \"pacemaker\" -T \"fence_dummy\"",
                                   "resource-agent name=\"fence_dummy\"")
 
         ### get metadata ###
         if "systemd" in self.rsc_classes:
             test = self.new_test("get_systemd_metadata", "Retrieve metadata for a resource")
             test.add_cmd_check_stdout("-c metadata -C \"systemd\" -T \"pacemaker-cts-dummyd@\"",
                                       "resource-agent name=\"pacemaker-cts-dummyd@\"")
 
         ### get metadata ###
         if "upstart" in self.rsc_classes:
             test = self.new_test("get_upstart_metadata", "Retrieve metadata for a resource")
             test.add_cmd_check_stdout("-c metadata -C \"upstart\" -T \"pacemaker-cts-dummyd\"",
                                       "resource-agent name=\"pacemaker-cts-dummyd\"")
 
         ### get ocf providers  ###
         test = self.new_test("list_ocf_providers",
                              "Retrieve list of available resource providers, verifies pacemaker is a provider.")
         test.add_cmd_check_stdout("-c list_ocf_providers ", "pacemaker")
         test.add_cmd_check_stdout("-c list_ocf_providers -T ping", "pacemaker")
 
         ### Verify agents only exist in their lists ###
         test = self.new_test("verify_agent_lists", "Verify the agent lists contain the right data.")
         test.add_cmd_check_stdout("-c list_agents ", "Stateful")                                  ### ocf ###
         test.add_cmd_check_stdout("-c list_agents -C ocf", "Stateful")
         test.add_cmd_check_stdout("-c list_agents -C lsb", "", "Stateful")                        ### should not exist
         test.add_cmd_check_stdout("-c list_agents -C service", "", "Stateful")                    ### should not exist
         test.add_cmd_check_stdout("-c list_agents ", "LSBDummy")                                  ### init.d ###
         test.add_cmd_check_stdout("-c list_agents -C lsb", "LSBDummy")
         test.add_cmd_check_stdout("-c list_agents -C service", "LSBDummy")
         test.add_cmd_check_stdout("-c list_agents -C ocf", "", "pacemaker-cts-dummyd@")           ### should not exist
 
         test.add_cmd_check_stdout("-c list_agents -C ocf", "", "pacemaker-cts-dummyd@")           ### should not exist
         test.add_cmd_check_stdout("-c list_agents -C lsb", "", "fence_dummy")                     ### should not exist
         test.add_cmd_check_stdout("-c list_agents -C service", "", "fence_dummy")                 ### should not exist
         test.add_cmd_check_stdout("-c list_agents -C ocf", "", "fence_dummy")                     ### should not exist
 
         if "systemd" in self.rsc_classes:
             test.add_cmd_check_stdout("-c list_agents ", "pacemaker-cts-dummyd@")             ### systemd ###
             test.add_cmd_check_stdout("-c list_agents -C service", "LSBDummy")
             test.add_cmd_check_stdout("-c list_agents -C systemd", "", "Stateful")            ### should not exist
             test.add_cmd_check_stdout("-c list_agents -C systemd", "pacemaker-cts-dummyd@")
             test.add_cmd_check_stdout("-c list_agents -C systemd", "", "fence_dummy")         ### should not exist
 
         if "upstart" in self.rsc_classes:
             test.add_cmd_check_stdout("-c list_agents ", "pacemaker-cts-dummyd")              ### upstart ###
             test.add_cmd_check_stdout("-c list_agents -C service", "LSBDummy")
             test.add_cmd_check_stdout("-c list_agents -C upstart", "", "Stateful")            ### should not exist
             test.add_cmd_check_stdout("-c list_agents -C upstart", "pacemaker-cts-dummyd")
             test.add_cmd_check_stdout("-c list_agents -C upstart", "", "fence_dummy")         ### should not exist
 
         if "stonith" in self.rsc_classes:
             test.add_cmd_check_stdout("-c list_agents -C stonith", "fence_dummy")             ### stonith ###
             test.add_cmd_check_stdout("-c list_agents -C stonith", "", "pacemaker-cts-dummyd@") ### should not exist
             test.add_cmd_check_stdout("-c list_agents -C stonith", "", "Stateful")            ### should not exist
             test.add_cmd_check_stdout("-c list_agents ", "fence_dummy")
 
     def print_list(self):
         """ List all registered tests """
 
         print("\n==== %d TESTS FOUND ====" % (len(self.tests)))
         print("%35s - %s" % ("TEST NAME", "TEST DESCRIPTION"))
         print("%35s - %s" % ("--------------------", "--------------------"))
         for test in self.tests:
             print("%35s - %s" % (test.name, test.description))
         print("==== END OF LIST ====\n")
 
     def run_single(self, name):
         """ Run a single named test """
 
         for test in self.tests:
             if test.name == name:
                 test.run()
                 break
 
     def run_tests_matching(self, pattern):
         """ Run all tests whose name matches a pattern """
 
         for test in self.tests:
             if test.name.count(pattern) != 0:
                 test.run()
 
     def run_tests(self):
         """ Run all tests """
 
         for test in self.tests:
             test.run()
 
     def exit(self):
         """ Exit (with error status code if any test failed) """
 
         for test in self.tests:
             if test.executed == 0:
                 continue
 
             if test.get_exitcode() != CrmExit.OK:
                 sys.exit(CrmExit.ERROR)
 
         sys.exit(CrmExit.OK)
 
     def print_results(self):
         """ Print summary of results of executed tests """
 
         failures = 0
         success = 0
         print("\n\n======= FINAL RESULTS ==========")
         print("\n--- FAILURE RESULTS:")
         for test in self.tests:
             if test.executed == 0:
                 continue
 
             if test.get_exitcode() != CrmExit.OK:
                 failures = failures + 1
                 test.print_result("    ")
             else:
                 success = success + 1
 
         if failures == 0:
             print("    None")
 
         print("\n--- TOTALS\n    Pass:%d\n    Fail:%d\n" % (success, failures))
 
 
 class TestOptions(object):
     """ Option handler """
 
     def __init__(self):
         self.options = {}
         self.options['list-tests'] = 0
         self.options['run-all'] = 1
         self.options['run-only'] = ""
         self.options['run-only-pattern'] = ""
         self.options['verbose'] = 0
         self.options['timeout'] = 2
         self.options['force-wait'] = 0
         self.options['invalid-arg'] = ""
         self.options['show-usage'] = 0
         self.options['pacemaker-remote'] = 0
 
     def build_options(self, argv):
         """ Set options based on command-line arguments """
 
         args = argv[1:]
         skip = 0
         for i in range(0, len(args)):
             if skip:
                 skip = 0
                 continue
             elif args[i] == "-h" or args[i] == "--help":
                 self.options['show-usage'] = 1
             elif args[i] == "-l" or args[i] == "--list-tests":
                 self.options['list-tests'] = 1
             elif args[i] == "-V" or args[i] == "--verbose":
                 self.options['verbose'] = 1
             elif args[i] == "-t" or args[i] == "--timeout":
                 self.options['timeout'] = float(args[i+1])
             elif args[i] == "-w" or args[i] == "--force-wait":
                 self.options['force-wait'] = 1
             elif args[i] == "-R" or args[i] == "--pacemaker-remote":
                 if REMOTE_ENABLED:
                     self.options['pacemaker-remote'] = 1
                 else:
                     print("ERROR: This build does not support Pacemaker Remote")
                     sys.exit(CrmExit.USAGE)
             elif args[i] == "-r" or args[i] == "--run-only":
                 self.options['run-only'] = args[i+1]
                 skip = 1
             elif args[i] == "-p" or args[i] == "--run-only-pattern":
                 self.options['run-only-pattern'] = args[i+1]
                 skip = 1
 
 def show_usage():
     """ Show command usage """
 
     print("usage: " + sys.argv[0] + " [options]")
     print("If no options are provided, all tests will run")
     print("Options:")
     print("\t [--help | -h]                        Show usage")
     print("\t [--list-tests | -l]                  Print out all registered tests.")
     print("\t [--run-only | -r 'testname']         Run a specific test")
     print("\t [--verbose | -V]                     Verbose output")
     print("\t [--timeout | -t 'floating point number']"
             "\n\t\tUp to how many seconds each test case waits for the daemon to be initialized."
             "\n\t\tDefaults to 2. The value 0 means no limit.")
     print("\t [--force-wait | -w]"
             "\n\t\tEach test case waits the default/specified --timeout for the daemon without tracking the log.")
     if REMOTE_ENABLED:
         print("\t [--pacemaker-remote | -R             Test pacemaker-remoted binary instead of pacemaker-execd")
     print("\t [--run-only-pattern | -p 'string']   Run only tests containing the string value")
     print("\n\tExample: Run only the test 'start_stop'")
     print("\t\t " + sys.argv[0] + " --run-only start_stop")
     print("\n\tExample: Run only the tests with the string 'systemd' present in them")
     print("\t\t " + sys.argv[0] + " --run-only-pattern systemd")
 
 
 def main(argv):
     """ Run pacemaker-execd regression tests as specified by arguments """
 
     update_path()
 
     opts = TestOptions()
     opts.build_options(argv)
 
     if opts.options['show-usage']:
         show_usage()
         sys.exit(CrmExit.OK)
 
     # Create a temporary directory for log files (the directory will
     # automatically be erased when done)
     with tempfile.TemporaryDirectory(prefix="cts-exec-") as logdir:
         tests = Tests(opts.options['verbose'], opts.options['pacemaker-remote'],
                       opts.options['timeout'], opts.options['force-wait'],
                       logdir)
 
         tests.build_generic_tests()
         tests.build_multi_rsc_tests()
         tests.build_negative_tests()
         tests.build_custom_tests()
         tests.build_stress_tests()
 
         if opts.options['list-tests']:
             tests.print_list()
             sys.exit(CrmExit.OK)
 
         tests.setup_test_environment()
 
         print("Starting ...")
 
         if opts.options['run-only-pattern'] != "":
             tests.run_tests_matching(opts.options['run-only-pattern'])
             tests.print_results()
         elif opts.options['run-only'] != "":
             tests.run_single(opts.options['run-only'])
             tests.print_results()
         else:
             tests.run_tests()
             tests.print_results()
 
         tests.cleanup_test_environment()
         tests.exit()
 
 
 if __name__ == "__main__":
     main(sys.argv)
diff --git a/daemons/based/based_io.c b/daemons/based/based_io.c
index 7815b38074..6d069ac91d 100644
--- a/daemons/based/based_io.c
+++ b/daemons/based/based_io.c
@@ -1,467 +1,472 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <string.h>
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <dirent.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 
 #include <crm/crm.h>
 
 #include <crm/cib.h>
 #include <crm/common/util.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/cib/internal.h>
 #include <crm/cluster.h>
 
 #include <pacemaker-based.h>
 
 crm_trigger_t *cib_writer = NULL;
 
 int write_cib_contents(gpointer p);
 
 static void
 cib_rename(const char *old)
 {
     int new_fd;
     char *new = crm_strdup_printf("%s/cib.auto.XXXXXX", cib_root);
 
     umask(S_IWGRP | S_IWOTH | S_IROTH);
     new_fd = mkstemp(new);
     crm_err("Archiving unusable file %s as %s", old, new);
     if ((new_fd < 0) || (rename(old, new) < 0)) {
         crm_perror(LOG_ERR, "Couldn't rename %s as %s", old, new);
         crm_err("Disabling disk writes and continuing");
         cib_writes_enabled = FALSE;
     }
     if (new_fd > 0) {
         close(new_fd);
     }
     free(new);
 }
 
 /*
  * It is the callers responsibility to free the output of this function
  */
 
 static xmlNode *
 retrieveCib(const char *filename, const char *sigfile)
 {
     xmlNode *root = NULL;
 
     crm_info("Reading cluster configuration file %s (digest: %s)",
              filename, sigfile);
     switch (cib_file_read_and_verify(filename, sigfile, &root)) {
         case -pcmk_err_cib_corrupt:
             crm_warn("Continuing but %s will NOT be used.", filename);
             break;
 
         case -pcmk_err_cib_modified:
             /* Archive the original files so the contents are not lost */
             crm_warn("Continuing but %s will NOT be used.", filename);
             cib_rename(filename);
             cib_rename(sigfile);
             break;
     }
     return root;
 }
 
 /*
  * for OSs without support for direntry->d_type, like Solaris
  */
 #ifndef DT_UNKNOWN
 # define DT_UNKNOWN     0
 # define DT_FIFO        1
 # define DT_CHR         2
 # define DT_DIR         4
 # define DT_BLK         6
 # define DT_REG         8
 # define DT_LNK         10
 # define DT_SOCK        12
 # define DT_WHT         14
 #endif /*DT_UNKNOWN*/
 
 static int cib_archive_filter(const struct dirent * a)
 {
     int rc = 0;
     /* Looking for regular files (d_type = 8) starting with 'cib-' and not ending in .sig */
     struct stat s;
     char *a_path = crm_strdup_printf("%s/%s", cib_root, a->d_name);
 
     if(stat(a_path, &s) != 0) {
         rc = errno;
         crm_trace("%s - stat failed: %s (%d)", a->d_name, pcmk_strerror(rc), rc);
         rc = 0;
 
     } else if ((s.st_mode & S_IFREG) != S_IFREG) {
         unsigned char dtype;
 #ifdef HAVE_STRUCT_DIRENT_D_TYPE
         dtype = a->d_type;
 #else
         switch (s.st_mode & S_IFMT) {
             case S_IFREG:  dtype = DT_REG;      break;
             case S_IFDIR:  dtype = DT_DIR;      break;
             case S_IFCHR:  dtype = DT_CHR;      break;
             case S_IFBLK:  dtype = DT_BLK;      break;
             case S_IFLNK:  dtype = DT_LNK;      break;
             case S_IFIFO:  dtype = DT_FIFO;     break;
             case S_IFSOCK: dtype = DT_SOCK;     break;
             default:       dtype = DT_UNKNOWN;  break;
         }
 #endif
          crm_trace("%s - wrong type (%d)", a->d_name, dtype);
 
     } else if(strstr(a->d_name, "cib-") != a->d_name) {
         crm_trace("%s - wrong prefix", a->d_name);
 
     } else if (pcmk__ends_with_ext(a->d_name, ".sig")) {
         crm_trace("%s - wrong suffix", a->d_name);
 
     } else {
         crm_debug("%s - candidate", a->d_name);
         rc = 1;
     }
 
     free(a_path);
     return rc;
 }
 
 static int cib_archive_sort(const struct dirent ** a, const struct dirent **b)
 {
     /* Order by creation date - most recently created file first */
     int rc = 0;
     struct stat buf;
 
     time_t a_age = 0;
     time_t b_age = 0;
 
     char *a_path = crm_strdup_printf("%s/%s", cib_root, a[0]->d_name);
     char *b_path = crm_strdup_printf("%s/%s", cib_root, b[0]->d_name);
 
     if(stat(a_path, &buf) == 0) {
         a_age = buf.st_ctime;
     }
     if(stat(b_path, &buf) == 0) {
         b_age = buf.st_ctime;
     }
 
     free(a_path);
     free(b_path);
 
     if(a_age > b_age) {
         rc = 1;
     } else if(a_age < b_age) {
         rc = -1;
     }
 
     crm_trace("%s (%lu) vs. %s (%lu) : %d",
 	a[0]->d_name, (unsigned long)a_age,
 	b[0]->d_name, (unsigned long)b_age, rc);
     return rc;
 }
 
 xmlNode *
 readCibXmlFile(const char *dir, const char *file, gboolean discard_status)
 {
     struct dirent **namelist = NULL;
 
     int lpc = 0;
     char *sigfile = NULL;
     char *sigfilepath = NULL;
     char *filename = NULL;
     const char *name = NULL;
     const char *value = NULL;
     const char *validation = NULL;
     const char *use_valgrind = getenv("PCMK_valgrind_enabled");
 
     xmlNode *root = NULL;
     xmlNode *status = NULL;
 
     sigfile = crm_strdup_printf("%s.sig", file);
     if (pcmk__daemon_can_write(dir, file) == FALSE
             || pcmk__daemon_can_write(dir, sigfile) == FALSE) {
         cib_status = -EACCES;
         return NULL;
     }
 
     filename = crm_strdup_printf("%s/%s", dir, file);
     sigfilepath = crm_strdup_printf("%s/%s", dir, sigfile);
     free(sigfile);
 
     cib_status = pcmk_ok;
     root = retrieveCib(filename, sigfilepath);
     free(filename);
     free(sigfilepath);
 
     if (root == NULL) {
         crm_warn("Primary configuration corrupt or unusable, trying backups in %s", cib_root);
         lpc = scandir(cib_root, &namelist, cib_archive_filter, cib_archive_sort);
         if (lpc < 0) {
             crm_perror(LOG_NOTICE, "scandir(%s) failed", cib_root);
         }
     }
 
     while (root == NULL && lpc > 1) {
         crm_debug("Testing %d candidates", lpc);
 
         lpc--;
 
         filename = crm_strdup_printf("%s/%s", cib_root, namelist[lpc]->d_name);
         sigfile = crm_strdup_printf("%s.sig", filename);
 
         crm_info("Reading cluster configuration file %s (digest: %s)",
                  filename, sigfile);
         if (cib_file_read_and_verify(filename, sigfile, &root) < 0) {
             crm_warn("Continuing but %s will NOT be used.", filename);
         } else {
             crm_notice("Continuing with last valid configuration archive: %s", filename);
         }
 
         free(namelist[lpc]);
         free(filename);
         free(sigfile);
     }
     free(namelist);
 
     if (root == NULL) {
         root = createEmptyCib(0);
         crm_warn("Continuing with an empty configuration.");
     }
 
     if (cib_writes_enabled && use_valgrind &&
         (crm_is_true(use_valgrind) || strstr(use_valgrind, "pacemaker-based"))) {
 
         cib_writes_enabled = FALSE;
         crm_err("*** Disabling disk writes to avoid confusing Valgrind ***");
     }
 
     status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
     if (discard_status && status != NULL) {
         /* strip out the status section if there is one */
         free_xml(status);
         status = NULL;
     }
     if (status == NULL) {
         create_xml_node(root, XML_CIB_TAG_STATUS);
     }
 
     /* Do this before schema validation happens */
 
     /* fill in some defaults */
     name = XML_ATTR_GENERATION_ADMIN;
     value = crm_element_value(root, name);
     if (value == NULL) {
         crm_warn("No value for %s was specified in the configuration.", name);
         crm_warn("The recommended course of action is to shutdown,"
                  " run crm_verify and fix any errors it reports.");
         crm_warn("We will default to zero and continue but may get"
                  " confused about which configuration to use if"
                  " multiple nodes are powered up at the same time.");
         crm_xml_add_int(root, name, 0);
     }
 
     name = XML_ATTR_GENERATION;
     value = crm_element_value(root, name);
     if (value == NULL) {
         crm_xml_add_int(root, name, 0);
     }
 
     name = XML_ATTR_NUMUPDATES;
     value = crm_element_value(root, name);
     if (value == NULL) {
         crm_xml_add_int(root, name, 0);
     }
 
     // Unset (DC should set appropriate value)
     xml_remove_prop(root, XML_ATTR_DC_UUID);
 
     if (discard_status) {
         crm_log_xml_trace(root, "[on-disk]");
     }
 
     validation = crm_element_value(root, XML_ATTR_VALIDATION);
     if (validate_xml(root, NULL, TRUE) == FALSE) {
         crm_err("CIB does not validate with %s", crm_str(validation));
         cib_status = -pcmk_err_schema_validation;
 
     } else if (validation == NULL) {
         int version = 0;
 
         update_validation(&root, &version, 0, FALSE, FALSE);
         if (version > 0) {
             crm_notice("Enabling %s validation on"
                        " the existing (sane) configuration", get_schema_name(version));
         } else {
             crm_err("CIB does not validate with any known schema");
             cib_status = -pcmk_err_schema_validation;
         }
     }
 
     return root;
 }
 
 gboolean
 uninitializeCib(void)
 {
     xmlNode *tmp_cib = the_cib;
 
     if (tmp_cib == NULL) {
         crm_debug("The CIB has already been deallocated.");
         return FALSE;
     }
 
     the_cib = NULL;
 
     crm_debug("Deallocating the CIB.");
 
     free_xml(tmp_cib);
 
     crm_debug("The CIB has been deallocated.");
 
     return TRUE;
 }
 
 /*
  * This method will free the old CIB pointer on success and the new one
  * on failure.
  */
 int
 activateCibXml(xmlNode * new_cib, gboolean to_disk, const char *op)
 {
     if (new_cib) {
         xmlNode *saved_cib = the_cib;
 
         CRM_ASSERT(new_cib != saved_cib);
         the_cib = new_cib;
         free_xml(saved_cib);
         if (cib_writes_enabled && cib_status == pcmk_ok && to_disk) {
             crm_debug("Triggering CIB write for %s op", op);
             mainloop_set_trigger(cib_writer);
         }
         return pcmk_ok;
     }
 
     crm_err("Ignoring invalid CIB");
     if (the_cib) {
         crm_warn("Reverting to last known CIB");
     } else {
         crm_crit("Could not write out new CIB and no saved version to revert to");
     }
     return -ENODATA;
 }
 
 static void
 cib_diskwrite_complete(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
 {
-    if (signo) {
-        crm_notice("Disk write process terminated with signal %d (pid=%d, core=%d)", signo, pid,
-                   core);
+    const char *errmsg = "Could not write CIB to disk";
 
-    } else  {
-        do_crm_log(exitcode == 0 ? LOG_TRACE : LOG_ERR, "Disk write process exited (pid=%d, rc=%d)",
-                   pid, exitcode);
+    if ((exitcode != 0) && cib_writes_enabled) {
+        cib_writes_enabled = FALSE;
+        errmsg = "Disabling CIB disk writes after failure";
     }
 
-    if (exitcode != 0 && cib_writes_enabled) {
-        crm_err("Disabling disk writes after write failure");
-        cib_writes_enabled = FALSE;
+    if ((signo == 0) && (exitcode == 0)) {
+        crm_trace("Disk write [%d] succeeded", (int) pid);
+
+    } else if (signo == 0) {
+        crm_err("%s: process %d exited %d", errmsg, (int) pid, exitcode);
+
+    } else {
+        crm_err("%s: process %d terminated with signal %d (%s)%s",
+                errmsg, (int) pid, signo, strsignal(signo),
+                (core? " and dumped core" : ""));
     }
 
     mainloop_trigger_complete(cib_writer);
 }
 
 int
 write_cib_contents(gpointer p)
 {
     int exit_rc = pcmk_ok;
     xmlNode *cib_local = NULL;
 
     /* Make a copy of the CIB to write (possibly in a forked child) */
     if (p) {
         /* Synchronous write out */
         cib_local = copy_xml(p);
 
     } else {
         int pid = 0;
         int bb_state = qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0);
 
         /* Turn it off before the fork() to avoid:
          * - 2 processes writing to the same shared mem
          * - the child needing to disable it
          *   (which would close it from underneath the parent)
          * This way, the shared mem files are already closed
          */
         qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
 
         pid = fork();
         if (pid < 0) {
             crm_perror(LOG_ERR, "Disabling disk writes after fork failure");
             cib_writes_enabled = FALSE;
             return FALSE;
         }
 
         if (pid) {
             /* Parent */
             mainloop_child_add(pid, 0, "disk-writer", NULL, cib_diskwrite_complete);
             if (bb_state == QB_LOG_STATE_ENABLED) {
                 /* Re-enable now that it it safe */
                 qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE);
             }
 
             return -1;          /* -1 means 'still work to do' */
         }
 
         /* Asynchronous write-out after a fork() */
 
         /* In theory, we can scribble on the_cib here and not affect the parent,
          * but let's be safe anyway.
          */
         cib_local = copy_xml(the_cib);
     }
 
     /* Write the CIB */
     exit_rc = cib_file_write_with_digest(cib_local, cib_root, "cib.xml");
 
     /* A nonzero exit code will cause further writes to be disabled */
     free_xml(cib_local);
     if (p == NULL) {
         crm_exit_t exit_code = CRM_EX_OK;
 
         switch (exit_rc) {
             case pcmk_ok:
                 exit_code = CRM_EX_OK;
                 break;
             case pcmk_err_cib_modified:
                 exit_code = CRM_EX_DIGEST; // Existing CIB doesn't match digest
                 break;
             case pcmk_err_cib_backup: // Existing CIB couldn't be backed up
             case pcmk_err_cib_save:   // New CIB couldn't be saved
                 exit_code = CRM_EX_CANTCREAT;
                 break;
             default:
                 exit_code = CRM_EX_ERROR;
                 break;
         }
 
         /* Use _exit() because exit() could affect the parent adversely */
         _exit(exit_code);
     }
     return exit_rc;
 }
diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c
index d4df9f290c..84bd93727a 100644
--- a/daemons/execd/execd_commands.c
+++ b/daemons/execd/execd_commands.c
@@ -1,1956 +1,1903 @@
 /*
  * Copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <glib.h>
 
 // Check whether we have a high-resolution monotonic clock
 #undef PCMK__TIME_USE_CGT
 #if HAVE_DECL_CLOCK_MONOTONIC && defined(CLOCK_MONOTONIC)
 #  define PCMK__TIME_USE_CGT
 #  include <time.h>  /* clock_gettime */
 #endif
 
 #include <unistd.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/services_internal.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/msg_xml.h>
 
 #include "pacemaker-execd.h"
 
 GHashTable *rsc_list = NULL;
 
 typedef struct lrmd_cmd_s {
     int timeout;
     guint interval_ms;
     int start_delay;
     int timeout_orig;
 
     int call_id;
 
     int call_opts;
     /* Timer ids, must be removed on cmd destruction. */
     int delay_id;
     int stonith_recurring_id;
 
     int rsc_deleted;
 
     int service_flags;
 
     char *client_id;
     char *origin;
     char *rsc_id;
     char *action;
     char *real_action;
     char *userdata_str;
 
     pcmk__action_result_t result;
 
     /* We can track operation queue time and run time, to be saved with the CIB
      * resource history (and displayed in cluster status). We need
      * high-resolution monotonic time for this purpose, so we use
      * clock_gettime(CLOCK_MONOTONIC, ...) (if available, otherwise this feature
      * is disabled).
      *
      * However, we also need epoch timestamps for recording the time the command
      * last ran and the time its return value last changed, for use in time
      * displays (as opposed to interval calculations). We keep time_t values for
      * this purpose.
      *
      * The last run time is used for both purposes, so we keep redundant
      * monotonic and epoch values for this. Technically the two could represent
      * different times, but since time_t has only second resolution and the
      * values are used for distinct purposes, that is not significant.
      */
 #ifdef PCMK__TIME_USE_CGT
     /* Recurring and systemd operations may involve more than one executor
      * command per operation, so they need info about the original and the most
      * recent.
      */
     struct timespec t_first_run;    // When op first ran
     struct timespec t_run;          // When op most recently ran
     struct timespec t_first_queue;  // When op was first queued
     struct timespec t_queue;        // When op was most recently queued
 #endif
     time_t epoch_last_run;          // Epoch timestamp of when op last ran
     time_t epoch_rcchange;          // Epoch timestamp of when rc last changed
 
     bool first_notify_sent;
     int last_notify_rc;
     int last_notify_op_status;
     int last_pid;
 
     GHashTable *params;
 } lrmd_cmd_t;
 
 static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc);
 static gboolean lrmd_rsc_dispatch(gpointer user_data);
 static void cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id);
 
 #ifdef PCMK__TIME_USE_CGT
 
 /*!
  * \internal
  * \brief Check whether a struct timespec has been set
  *
  * \param[in] timespec  Time to check
  *
  * \return true if timespec has been set (i.e. is nonzero), false otherwise
  */
 static inline bool
 time_is_set(struct timespec *timespec)
 {
     return (timespec != NULL) &&
            ((timespec->tv_sec != 0) || (timespec->tv_nsec != 0));
 }
 
 /*
  * \internal
  * \brief Set a timespec (and its original if unset) to the current time
  *
  * \param[out] t_current  Where to store current time
  * \param[out] t_orig     Where to copy t_current if unset
  */
 static void
 get_current_time(struct timespec *t_current, struct timespec *t_orig)
 {
     clock_gettime(CLOCK_MONOTONIC, t_current);
     if ((t_orig != NULL) && !time_is_set(t_orig)) {
         *t_orig = *t_current;
     }
 }
 
 /*!
  * \internal
  * \brief Return difference between two times in milliseconds
  *
  * \param[in] now  More recent time (or NULL to use current time)
  * \param[in] old  Earlier time
  *
  * \return milliseconds difference (or 0 if old is NULL or unset)
  *
  * \note Can overflow on 32bit machines when the differences is around
  *       24 days or more.
  */
 static int
 time_diff_ms(struct timespec *now, struct timespec *old)
 {
     int diff_ms = 0;
 
     if (time_is_set(old)) {
         struct timespec local_now = { 0, };
 
         if (now == NULL) {
             clock_gettime(CLOCK_MONOTONIC, &local_now);
             now = &local_now;
         }
         diff_ms = (now->tv_sec - old->tv_sec) * 1000
                   + (now->tv_nsec - old->tv_nsec) / 1000000;
     }
     return diff_ms;
 }
 
 /*!
  * \internal
  * \brief Reset a command's operation times to their original values.
  *
  * Reset a command's run and queued timestamps to the timestamps of the original
  * command, so we report the entire time since then and not just the time since
  * the most recent command (for recurring and systemd operations).
  *
  * \param[in] cmd  Executor command object to reset
  *
  * \note It's not obvious what the queued time should be for a systemd
  *       start/stop operation, which might go like this:
  *         initial command queued 5ms, runs 3s
  *         monitor command queued 10ms, runs 10s
  *         monitor command queued 10ms, runs 10s
  *       Is the queued time for that operation 5ms, 10ms or 25ms? The current
  *       implementation will report 5ms. If it's 25ms, then we need to
  *       subtract 20ms from the total exec time so as not to count it twice.
  *       We can implement that later if it matters to anyone ...
  */
 static void
 cmd_original_times(lrmd_cmd_t * cmd)
 {
     cmd->t_run = cmd->t_first_run;
     cmd->t_queue = cmd->t_first_queue;
 }
 #endif
 
 static inline bool
 action_matches(lrmd_cmd_t *cmd, const char *action, guint interval_ms)
 {
     return (cmd->interval_ms == interval_ms)
            && pcmk__str_eq(cmd->action, action, pcmk__str_casei);
 }
 
 static void
 log_finished(lrmd_cmd_t * cmd, int exec_time, int queue_time)
 {
     char pid_str[32] = { 0, };
     int log_level = LOG_INFO;
 
     if (cmd->last_pid) {
         snprintf(pid_str, 32, "%d", cmd->last_pid);
     }
 
     if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
         log_level = LOG_DEBUG;
     }
 #ifdef PCMK__TIME_USE_CGT
     do_crm_log(log_level, "%s %s (call %d%s%s) exited with status %d"
                " (execution time %dms, queue time %dms)",
                cmd->rsc_id, cmd->action, cmd->call_id,
                (cmd->last_pid? ", PID " : ""), pid_str,
                cmd->result.exit_status, exec_time, queue_time);
 #else
     do_crm_log(log_level, "%s %s (call %d%s%s) exited with status %d",
                cmd->rsc_id, cmd->action, cmd->call_id,
                (cmd->last_pid? ", PID " : ""), pid_str,
                cmd->result.exit_status);
 #endif
 }
 
 static void
 log_execute(lrmd_cmd_t * cmd)
 {
     int log_level = LOG_INFO;
 
     if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
         log_level = LOG_DEBUG;
     }
 
     do_crm_log(log_level, "executing - rsc:%s action:%s call_id:%d",
                cmd->rsc_id, cmd->action, cmd->call_id);
 }
 
 static const char *
 normalize_action_name(lrmd_rsc_t * rsc, const char *action)
 {
     if (pcmk__str_eq(action, "monitor", pcmk__str_casei) &&
         pcmk_is_set(pcmk_get_ra_caps(rsc->class), pcmk_ra_cap_status)) {
         return "status";
     }
     return action;
 }
 
 static lrmd_rsc_t *
 build_rsc_from_xml(xmlNode * msg)
 {
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
     lrmd_rsc_t *rsc = NULL;
 
     rsc = calloc(1, sizeof(lrmd_rsc_t));
 
     crm_element_value_int(msg, F_LRMD_CALLOPTS, &rsc->call_opts);
 
     rsc->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
     rsc->class = crm_element_value_copy(rsc_xml, F_LRMD_CLASS);
     rsc->provider = crm_element_value_copy(rsc_xml, F_LRMD_PROVIDER);
     rsc->type = crm_element_value_copy(rsc_xml, F_LRMD_TYPE);
     rsc->work = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_rsc_dispatch, rsc);
     rsc->st_probe_rc = -ENODEV; // if stonith, initialize to "not running"
     return rsc;
 }
 
 static lrmd_cmd_t *
 create_lrmd_cmd(xmlNode *msg, pcmk__client_t *client)
 {
     int call_options = 0;
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
     lrmd_cmd_t *cmd = NULL;
 
     cmd = calloc(1, sizeof(lrmd_cmd_t));
 
     crm_element_value_int(msg, F_LRMD_CALLOPTS, &call_options);
     cmd->call_opts = call_options;
     cmd->client_id = strdup(client->id);
 
     crm_element_value_int(msg, F_LRMD_CALLID, &cmd->call_id);
     crm_element_value_ms(rsc_xml, F_LRMD_RSC_INTERVAL, &cmd->interval_ms);
     crm_element_value_int(rsc_xml, F_LRMD_TIMEOUT, &cmd->timeout);
     crm_element_value_int(rsc_xml, F_LRMD_RSC_START_DELAY, &cmd->start_delay);
     cmd->timeout_orig = cmd->timeout;
 
     cmd->origin = crm_element_value_copy(rsc_xml, F_LRMD_ORIGIN);
     cmd->action = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ACTION);
     cmd->userdata_str = crm_element_value_copy(rsc_xml, F_LRMD_RSC_USERDATA_STR);
     cmd->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
 
     cmd->params = xml2list(rsc_xml);
 
     if (pcmk__str_eq(g_hash_table_lookup(cmd->params, "CRM_meta_on_fail"), "block", pcmk__str_casei)) {
         crm_debug("Setting flag to leave pid group on timeout and "
                   "only kill action pid for " PCMK__OP_FMT,
                   cmd->rsc_id, cmd->action, cmd->interval_ms);
         cmd->service_flags = pcmk__set_flags_as(__func__, __LINE__,
                                                 LOG_TRACE, "Action",
                                                 cmd->action, 0,
                                                 SVC_ACTION_LEAVE_GROUP,
                                                 "SVC_ACTION_LEAVE_GROUP");
     }
     return cmd;
 }
 
 static void
 stop_recurring_timer(lrmd_cmd_t *cmd)
 {
     if (cmd) {
         if (cmd->stonith_recurring_id) {
             g_source_remove(cmd->stonith_recurring_id);
         }
         cmd->stonith_recurring_id = 0;
     }
 }
 
 static void
 free_lrmd_cmd(lrmd_cmd_t * cmd)
 {
     stop_recurring_timer(cmd);
     if (cmd->delay_id) {
         g_source_remove(cmd->delay_id);
     }
     if (cmd->params) {
         g_hash_table_destroy(cmd->params);
     }
     pcmk__reset_result(&(cmd->result));
     free(cmd->origin);
     free(cmd->action);
     free(cmd->real_action);
     free(cmd->userdata_str);
     free(cmd->rsc_id);
     free(cmd->client_id);
     free(cmd);
 }
 
 static gboolean
 stonith_recurring_op_helper(gpointer data)
 {
     lrmd_cmd_t *cmd = data;
     lrmd_rsc_t *rsc;
 
     cmd->stonith_recurring_id = 0;
 
     if (!cmd->rsc_id) {
         return FALSE;
     }
 
     rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);
 
     CRM_ASSERT(rsc != NULL);
     /* take it out of recurring_ops list, and put it in the pending ops
      * to be executed */
     rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
     rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
 #ifdef PCMK__TIME_USE_CGT
     get_current_time(&(cmd->t_queue), &(cmd->t_first_queue));
 #endif
     mainloop_set_trigger(rsc->work);
 
     return FALSE;
 }
 
 static inline void
 start_recurring_timer(lrmd_cmd_t *cmd)
 {
     if (cmd && (cmd->interval_ms > 0)) {
         cmd->stonith_recurring_id = g_timeout_add(cmd->interval_ms,
                                                   stonith_recurring_op_helper,
                                                   cmd);
     }
 }
 
 static gboolean
 start_delay_helper(gpointer data)
 {
     lrmd_cmd_t *cmd = data;
     lrmd_rsc_t *rsc = NULL;
 
     cmd->delay_id = 0;
     rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;
 
     if (rsc) {
         mainloop_set_trigger(rsc->work);
     }
 
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Check whether a list already contains the equivalent of a given action
  */
 static lrmd_cmd_t *
 find_duplicate_action(GList *action_list, lrmd_cmd_t *cmd)
 {
     for (GList *item = action_list; item != NULL; item = item->next) {
         lrmd_cmd_t *dup = item->data;
 
         if (action_matches(cmd, dup->action, dup->interval_ms)) {
             return dup;
         }
     }
     return NULL;
 }
 
 static bool
 merge_recurring_duplicate(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
 {
     lrmd_cmd_t * dup = NULL;
     bool dup_pending = true;
 
     if (cmd->interval_ms == 0) {
         return false;
     }
 
     // Search for a duplicate of this action (in-flight or not)
     dup = find_duplicate_action(rsc->pending_ops, cmd);
     if (dup == NULL) {
         dup_pending = false;
         dup = find_duplicate_action(rsc->recurring_ops, cmd);
         if (dup == NULL) {
             return false;
         }
     }
 
     /* Do not merge fencing monitors marked for cancellation, so we can reply to
      * the cancellation separately.
      */
     if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
                      pcmk__str_casei)
         && (dup->result.execution_status == PCMK_EXEC_CANCELLED)) {
         return false;
     }
 
     /* This should not occur. If it does, we need to investigate how something
      * like this is possible in the controller.
      */
     crm_warn("Duplicate recurring op entry detected (" PCMK__OP_FMT
              "), merging with previous op entry",
              rsc->rsc_id, normalize_action_name(rsc, dup->action),
              dup->interval_ms);
 
     // Merge new action's call ID and user data into existing action
     dup->first_notify_sent = false;
     free(dup->userdata_str);
     dup->userdata_str = cmd->userdata_str;
     cmd->userdata_str = NULL;
     dup->call_id = cmd->call_id;
     free_lrmd_cmd(cmd);
     cmd = NULL;
 
     /* If dup is not pending, that means it has already executed at least once
      * and is waiting in the interval. In that case, stop waiting and initiate
      * a new instance now.
      */
     if (!dup_pending) {
         if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
                          pcmk__str_casei)) {
             stop_recurring_timer(dup);
             stonith_recurring_op_helper(dup);
         } else {
             services_action_kick(rsc->rsc_id,
                                  normalize_action_name(rsc, dup->action),
                                  dup->interval_ms);
         }
     }
     return true;
 }
 
 static void
 schedule_lrmd_cmd(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
 {
     CRM_CHECK(cmd != NULL, return);
     CRM_CHECK(rsc != NULL, return);
 
     crm_trace("Scheduling %s on %s", cmd->action, rsc->rsc_id);
 
     if (merge_recurring_duplicate(rsc, cmd)) {
         // Equivalent of cmd has already been scheduled
         return;
     }
 
     /* The controller expects the executor to automatically cancel
      * recurring operations before a resource stops.
      */
     if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
         cancel_all_recurring(rsc, NULL);
     }
 
     rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
 #ifdef PCMK__TIME_USE_CGT
     get_current_time(&(cmd->t_queue), &(cmd->t_first_queue));
 #endif
     mainloop_set_trigger(rsc->work);
 
     if (cmd->start_delay) {
         cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
     }
 }
 
 static xmlNode *
 create_lrmd_reply(const char *origin, int rc, int call_id)
 {
     xmlNode *reply = create_xml_node(NULL, T_LRMD_REPLY);
 
     crm_xml_add(reply, F_LRMD_ORIGIN, origin);
     crm_xml_add_int(reply, F_LRMD_RC, rc);
     crm_xml_add_int(reply, F_LRMD_CALLID, call_id);
     return reply;
 }
 
 static void
 send_client_notify(gpointer key, gpointer value, gpointer user_data)
 {
     xmlNode *update_msg = user_data;
     pcmk__client_t *client = value;
     int rc;
     int log_level = LOG_WARNING;
     const char *msg = NULL;
 
     CRM_CHECK(client != NULL, return);
     if (client->name == NULL) {
         crm_trace("Skipping notification to client without name");
         return;
     }
     if (pcmk_is_set(client->flags, pcmk__client_to_proxy)) {
         /* We only want to notify clients of the executor IPC API. If we are
          * running as Pacemaker Remote, we may have clients proxied to other
          * IPC services in the cluster, so skip those.
          */
         crm_trace("Skipping executor API notification to client %s",
                   pcmk__client_name(client));
         return;
     }
 
     rc = lrmd_server_send_notify(client, update_msg);
     if (rc == pcmk_rc_ok) {
         return;
     }
 
     switch (rc) {
         case ENOTCONN:
         case EPIPE: // Client exited without waiting for notification
             log_level = LOG_INFO;
             msg = "Disconnected";
             break;
 
         default:
             msg = pcmk_rc_str(rc);
             break;
     }
     do_crm_log(log_level, "Could not notify client %s: %s " CRM_XS " rc=%d",
                pcmk__client_name(client), msg, rc);
 }
 
 static void
 send_cmd_complete_notify(lrmd_cmd_t * cmd)
 {
     xmlNode *notify = NULL;
 
 #ifdef PCMK__TIME_USE_CGT
     int exec_time = time_diff_ms(NULL, &(cmd->t_run));
     int queue_time = time_diff_ms(&cmd->t_run, &(cmd->t_queue));
 
     log_finished(cmd, exec_time, queue_time);
 #else
     log_finished(cmd, 0, 0);
 #endif
 
     /* if the first notify result for a cmd has already been sent earlier, and the
      * the option to only send notifies on result changes is set. Check to see
      * if the last result is the same as the new one. If so, suppress this update */
     if (cmd->first_notify_sent && (cmd->call_opts & lrmd_opt_notify_changes_only)) {
         if ((cmd->last_notify_rc == cmd->result.exit_status) &&
             (cmd->last_notify_op_status == cmd->result.execution_status)) {
 
             /* only send changes */
             return;
         }
 
     }
 
     cmd->first_notify_sent = true;
     cmd->last_notify_rc = cmd->result.exit_status;
     cmd->last_notify_op_status = cmd->result.execution_status;
 
     notify = create_xml_node(NULL, T_LRMD_NOTIFY);
 
     crm_xml_add(notify, F_LRMD_ORIGIN, __func__);
     crm_xml_add_int(notify, F_LRMD_TIMEOUT, cmd->timeout);
     crm_xml_add_ms(notify, F_LRMD_RSC_INTERVAL, cmd->interval_ms);
     crm_xml_add_int(notify, F_LRMD_RSC_START_DELAY, cmd->start_delay);
     crm_xml_add_int(notify, F_LRMD_EXEC_RC, cmd->result.exit_status);
     crm_xml_add_int(notify, F_LRMD_OP_STATUS, cmd->result.execution_status);
     crm_xml_add_int(notify, F_LRMD_CALLID, cmd->call_id);
     crm_xml_add_int(notify, F_LRMD_RSC_DELETED, cmd->rsc_deleted);
 
     crm_xml_add_ll(notify, F_LRMD_RSC_RUN_TIME,
                    (long long) cmd->epoch_last_run);
     crm_xml_add_ll(notify, F_LRMD_RSC_RCCHANGE_TIME,
                    (long long) cmd->epoch_rcchange);
 #ifdef PCMK__TIME_USE_CGT
     crm_xml_add_int(notify, F_LRMD_RSC_EXEC_TIME, exec_time);
     crm_xml_add_int(notify, F_LRMD_RSC_QUEUE_TIME, queue_time);
 #endif
 
     crm_xml_add(notify, F_LRMD_OPERATION, LRMD_OP_RSC_EXEC);
     crm_xml_add(notify, F_LRMD_RSC_ID, cmd->rsc_id);
     if(cmd->real_action) {
         crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->real_action);
     } else {
         crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->action);
     }
     crm_xml_add(notify, F_LRMD_RSC_USERDATA_STR, cmd->userdata_str);
     crm_xml_add(notify, F_LRMD_RSC_EXIT_REASON, cmd->result.exit_reason);
 
     if (cmd->result.action_stderr != NULL) {
         crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->result.action_stderr);
 
     } else if (cmd->result.action_stdout != NULL) {
         crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->result.action_stdout);
     }
 
     if (cmd->params) {
         char *key = NULL;
         char *value = NULL;
         GHashTableIter iter;
 
         xmlNode *args = create_xml_node(notify, XML_TAG_ATTRS);
 
         g_hash_table_iter_init(&iter, cmd->params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             hash2smartfield((gpointer) key, (gpointer) value, args);
         }
     }
     if (cmd->client_id && (cmd->call_opts & lrmd_opt_notify_orig_only)) {
         pcmk__client_t *client = pcmk__find_client_by_id(cmd->client_id);
 
         if (client) {
             send_client_notify(client->id, client, notify);
         }
     } else {
         pcmk__foreach_ipc_client(send_client_notify, notify);
     }
 
     free_xml(notify);
 }
 
 static void
 send_generic_notify(int rc, xmlNode * request)
 {
     if (pcmk__ipc_client_count() != 0) {
         int call_id = 0;
         xmlNode *notify = NULL;
         xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
         const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
         const char *op = crm_element_value(request, F_LRMD_OPERATION);
 
         crm_element_value_int(request, F_LRMD_CALLID, &call_id);
 
         notify = create_xml_node(NULL, T_LRMD_NOTIFY);
         crm_xml_add(notify, F_LRMD_ORIGIN, __func__);
         crm_xml_add_int(notify, F_LRMD_RC, rc);
         crm_xml_add_int(notify, F_LRMD_CALLID, call_id);
         crm_xml_add(notify, F_LRMD_OPERATION, op);
         crm_xml_add(notify, F_LRMD_RSC_ID, rsc_id);
 
         pcmk__foreach_ipc_client(send_client_notify, notify);
 
         free_xml(notify);
     }
 }
 
 static void
 cmd_reset(lrmd_cmd_t * cmd)
 {
     cmd->last_pid = 0;
 #ifdef PCMK__TIME_USE_CGT
     memset(&cmd->t_run, 0, sizeof(cmd->t_run));
     memset(&cmd->t_queue, 0, sizeof(cmd->t_queue));
 #endif
     cmd->epoch_last_run = 0;
 
     pcmk__reset_result(&(cmd->result));
     cmd->result.execution_status = PCMK_EXEC_DONE;
 }
 
 static void
 cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc)
 {
     crm_trace("Resource operation rsc:%s action:%s completed (%p %p)", cmd->rsc_id, cmd->action,
               rsc ? rsc->active : NULL, cmd);
 
     if (rsc && (rsc->active == cmd)) {
         rsc->active = NULL;
         mainloop_set_trigger(rsc->work);
     }
 
     if (!rsc) {
         cmd->rsc_deleted = 1;
     }
 
     /* reset original timeout so client notification has correct information */
     cmd->timeout = cmd->timeout_orig;
 
     send_cmd_complete_notify(cmd);
 
     if ((cmd->interval_ms != 0)
         && (cmd->result.execution_status == PCMK_EXEC_CANCELLED)) {
 
         if (rsc) {
             rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
             rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
         }
         free_lrmd_cmd(cmd);
     } else if (cmd->interval_ms == 0) {
         if (rsc) {
             rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
         }
         free_lrmd_cmd(cmd);
     } else {
         /* Clear all the values pertaining just to the last iteration of a recurring op. */
         cmd_reset(cmd);
     }
 }
 
-static int
-ocf2uniform_rc(int rc)
-{
-    switch (rc) {
-        case PCMK_OCF_DEGRADED:
-        case PCMK_OCF_DEGRADED_PROMOTED:
-            break;
-        default:
-            if (rc < 0 || rc > PCMK_OCF_FAILED_PROMOTED) {
-                return PCMK_OCF_UNKNOWN_ERROR;
-            }
-    }
-
-    return rc;
-}
-
 static int
 stonith2uniform_rc(const char *action, int rc)
 {
     switch (rc) {
         case pcmk_ok:
             rc = PCMK_OCF_OK;
             break;
 
         case -ENODEV:
             /* This should be possible only for probes in practice, but
              * interpret for all actions to be safe.
              */
             if (pcmk__str_eq(action, "monitor", pcmk__str_casei)) {
                 rc = PCMK_OCF_NOT_RUNNING;
             } else if (pcmk__str_eq(action, "stop", pcmk__str_casei)) {
                 rc = PCMK_OCF_OK;
             } else {
                 rc = PCMK_OCF_NOT_INSTALLED;
             }
             break;
 
         case -EOPNOTSUPP:
             rc = PCMK_OCF_UNIMPLEMENT_FEATURE;
             break;
 
         default:
             rc = PCMK_OCF_UNKNOWN_ERROR;
             break;
     }
     return rc;
 }
 
-#if SUPPORT_NAGIOS
 static int
-nagios2uniform_rc(const char *action, int rc)
+action_get_uniform_rc(svc_action_t *action)
 {
-    if (rc < 0) {
-        return PCMK_OCF_UNKNOWN_ERROR;
-    }
-
-    switch (rc) {
-        case NAGIOS_STATE_OK:
-            return PCMK_OCF_OK;
-        case NAGIOS_INSUFFICIENT_PRIV:
-            return PCMK_OCF_INSUFFICIENT_PRIV;
-        case NAGIOS_NOT_INSTALLED:
-            return PCMK_OCF_NOT_INSTALLED;
-        case NAGIOS_STATE_WARNING:
-        case NAGIOS_STATE_CRITICAL:
-        case NAGIOS_STATE_UNKNOWN:
-        case NAGIOS_STATE_DEPENDENT:
-        default:
-            return PCMK_OCF_UNKNOWN_ERROR;
-    }
-
-    return PCMK_OCF_UNKNOWN_ERROR;
-}
-#endif
+    lrmd_cmd_t *cmd = action->cb_data;
 
-static int
-get_uniform_rc(const char *standard, const char *action, int rc)
-{
-    if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
-        return ocf2uniform_rc(rc);
-    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
-        return stonith2uniform_rc(action, rc);
-    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
-        return rc;
-    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_casei)) {
-        return rc;
-#if SUPPORT_NAGIOS
-    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
-        return nagios2uniform_rc(action, rc);
-#endif
+    if (pcmk__str_eq(action->standard, PCMK_RESOURCE_CLASS_STONITH,
+                            pcmk__str_casei)) {
+        return stonith2uniform_rc(cmd->action, action->rc);
     } else {
-        return services_get_ocf_exitcode(action, rc);
-    }
-}
+        enum ocf_exitcode code = services_result2ocf(action->standard,
+                                                     cmd->action, action->rc);
 
-static int
-action_get_uniform_rc(svc_action_t * action)
-{
-    lrmd_cmd_t *cmd = action->cb_data;
-    return get_uniform_rc(action->standard, cmd->action, action->rc);
+        // Cast variable instead of function return to keep compilers happy
+        return (int) code;
+    }
 }
 
 struct notify_new_client_data {
     xmlNode *notify;
     pcmk__client_t *new_client;
 };
 
 static void
 notify_one_client(gpointer key, gpointer value, gpointer user_data)
 {
     pcmk__client_t *client = value;
     struct notify_new_client_data *data = user_data;
 
     if (!pcmk__str_eq(client->id, data->new_client->id, pcmk__str_casei)) {
         send_client_notify(key, (gpointer) client, (gpointer) data->notify);
     }
 }
 
 void
 notify_of_new_client(pcmk__client_t *new_client)
 {
     struct notify_new_client_data data;
 
     data.new_client = new_client;
     data.notify = create_xml_node(NULL, T_LRMD_NOTIFY);
     crm_xml_add(data.notify, F_LRMD_ORIGIN, __func__);
     crm_xml_add(data.notify, F_LRMD_OPERATION, LRMD_OP_NEW_CLIENT);
     pcmk__foreach_ipc_client(notify_one_client, &data);
     free_xml(data.notify);
 }
 
 void
 client_disconnect_cleanup(const char *client_id)
 {
     GHashTableIter iter;
     lrmd_rsc_t *rsc = NULL;
     char *key = NULL;
 
     g_hash_table_iter_init(&iter, rsc_list);
     while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) {
         if (rsc->call_opts & lrmd_opt_drop_recurring) {
             /* This client is disconnecting, drop any recurring operations
              * it may have initiated on the resource */
             cancel_all_recurring(rsc, client_id);
         }
     }
 }
 
 static void
 action_complete(svc_action_t * action)
 {
     lrmd_rsc_t *rsc;
     lrmd_cmd_t *cmd = action->cb_data;
 
 #ifdef PCMK__TIME_USE_CGT
     const char *rclass = NULL;
     bool goagain = false;
 #endif
 
     if (!cmd) {
         crm_err("Completed executor action (%s) does not match any known operations",
                 action->id);
         return;
     }
 
 #ifdef PCMK__TIME_USE_CGT
     if (cmd->result.exit_status != action->rc) {
         cmd->epoch_rcchange = time(NULL);
     }
 #endif
 
     cmd->last_pid = action->pid;
     pcmk__set_result(&(cmd->result), action_get_uniform_rc(action),
                      action->status, services__exit_reason(action));
     rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;
 
 #ifdef PCMK__TIME_USE_CGT
     if (rsc && pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
         rclass = resources_find_service_class(rsc->type);
     } else if(rsc) {
         rclass = rsc->class;
     }
 
     if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
         if ((cmd->result.exit_status == PCMK_OCF_OK)
             && pcmk__strcase_any_of(cmd->action, "start", "stop", NULL)) {
             /* systemd returns from start and stop actions after the action
              * begins, not after it completes. We have to jump through a few
              * hoops so that we don't report 'complete' to the rest of pacemaker
              * until it's actually done.
              */
             goagain = true;
             cmd->real_action = cmd->action;
             cmd->action = strdup("monitor");
 
         } else if (cmd->real_action != NULL) {
             // This is follow-up monitor to check whether start/stop completed
             if (cmd->result.execution_status == PCMK_EXEC_PENDING) {
                 goagain = true;
 
             } else if ((cmd->result.exit_status == PCMK_OCF_OK)
                        && pcmk__str_eq(cmd->real_action, "stop", pcmk__str_casei)) {
                 goagain = true;
 
             } else {
                 int time_sum = time_diff_ms(NULL, &(cmd->t_first_run));
                 int timeout_left = cmd->timeout_orig - time_sum;
 
                 crm_debug("%s systemd %s is now complete (elapsed=%dms, "
                           "remaining=%dms): %s (%d)",
                           cmd->rsc_id, cmd->real_action, time_sum, timeout_left,
                           services_ocf_exitcode_str(cmd->result.exit_status),
                           cmd->result.exit_status);
                 cmd_original_times(cmd);
 
                 // Monitors may return "not running", but start/stop shouldn't
                 if ((cmd->result.execution_status == PCMK_EXEC_DONE)
                     && (cmd->result.exit_status == PCMK_OCF_NOT_RUNNING)) {
 
                     if (pcmk__str_eq(cmd->real_action, "start", pcmk__str_casei)) {
                         cmd->result.exit_status = PCMK_OCF_UNKNOWN_ERROR;
                     } else if (pcmk__str_eq(cmd->real_action, "stop", pcmk__str_casei)) {
                         cmd->result.exit_status = PCMK_OCF_OK;
                     }
                 }
             }
         }
     }
 #endif
 
 #if SUPPORT_NAGIOS
     if (rsc && pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
         if (action_matches(cmd, "monitor", 0)
             && (cmd->result.exit_status == PCMK_OCF_OK)) {
             /* Successfully executed --version for the nagios plugin */
             cmd->result.exit_status = PCMK_OCF_NOT_RUNNING;
 
         } else if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)
                    && (cmd->result.exit_status != PCMK_OCF_OK)) {
 #ifdef PCMK__TIME_USE_CGT
             goagain = true;
 #endif
         }
     }
 #endif
 
 #ifdef PCMK__TIME_USE_CGT
     if (goagain) {
         int time_sum = time_diff_ms(NULL, &(cmd->t_first_run));
         int timeout_left = cmd->timeout_orig - time_sum;
         int delay = cmd->timeout_orig / 10;
 
         if(delay >= timeout_left && timeout_left > 20) {
             delay = timeout_left/2;
         }
 
         delay = QB_MIN(2000, delay);
         if (delay < timeout_left) {
             cmd->start_delay = delay;
             cmd->timeout = timeout_left;
 
             if (cmd->result.exit_status == PCMK_OCF_OK) {
                 crm_debug("%s %s may still be in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
                           cmd->rsc_id, cmd->real_action, time_sum, timeout_left, delay);
 
             } else if (cmd->result.execution_status == PCMK_EXEC_PENDING) {
                 crm_info("%s %s is still in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
                          cmd->rsc_id, cmd->action, time_sum, timeout_left, delay);
 
             } else {
                 crm_notice("%s %s failed '%s' (%d): re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
                            cmd->rsc_id, cmd->action,
                            services_ocf_exitcode_str(cmd->result.exit_status),
                            cmd->result.exit_status, time_sum, timeout_left,
                            delay);
             }
 
             cmd_reset(cmd);
             if(rsc) {
                 rsc->active = NULL;
             }
             schedule_lrmd_cmd(rsc, cmd);
 
             /* Don't finalize cmd, we're not done with it yet */
             return;
 
         } else {
             crm_notice("Giving up on %s %s (rc=%d): timeout (elapsed=%dms, remaining=%dms)",
                        cmd->rsc_id,
                        (cmd->real_action? cmd->real_action : cmd->action),
                        cmd->result.exit_status, time_sum, timeout_left);
             pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
                              PCMK_EXEC_TIMEOUT,
                              "Investigate reason for timeout, and adjust "
                              "configured operation timeout if necessary");
             cmd_original_times(cmd);
         }
     }
 #endif
 
     pcmk__set_result_output(&(cmd->result),
                             action->stdout_data, action->stderr_data);
     cmd_finalize(cmd, rsc);
 }
 
 /*!
  * \internal
  * \brief Determine operation status of a stonith operation
  *
  * Non-stonith resource operations get their operation status directly from the
  * service library, but the fencer does not have an equivalent, so we must infer
  * an operation status from the fencer API's return code.
  *
  * \param[in] action       Name of action performed on stonith resource
  * \param[in] interval_ms  Action interval
  * \param[in] rc           Action result from fencer
  *
  * \return Operation status corresponding to fencer API return code
  */
 static int
 stonith_rc2status(const char *action, guint interval_ms, int rc)
 {
     int status = PCMK_EXEC_DONE;
 
     switch (rc) {
         case pcmk_ok:
             break;
 
         case -EOPNOTSUPP:
         case -EPROTONOSUPPORT:
             status = PCMK_EXEC_NOT_SUPPORTED;
             break;
 
         case -ETIME:
         case -ETIMEDOUT:
             status = PCMK_EXEC_TIMEOUT;
             break;
 
         case -ENOTCONN:
         case -ECOMM:
             // Couldn't talk to fencer
             status = PCMK_EXEC_ERROR;
             break;
 
         case -ENODEV:
             // The device is not registered with the fencer
             status = PCMK_EXEC_ERROR;
             break;
 
         default:
             break;
     }
     return status;
 }
 
 static void
 stonith_action_complete(lrmd_cmd_t * cmd, int rc)
 {
     // This can be NULL if resource was removed before command completed
     lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);
 
     cmd->result.exit_status = stonith2uniform_rc(cmd->action, rc);
 
     /* This function may be called with status already set to cancelled, if a
      * pending action was aborted. Otherwise, we need to determine status from
      * the fencer return code.
      */
     if (cmd->result.execution_status != PCMK_EXEC_CANCELLED) {
         cmd->result.execution_status = stonith_rc2status(cmd->action,
                                                          cmd->interval_ms, rc);
 
         // Certain successful actions change the known state of the resource
         if ((rsc != NULL) && (cmd->result.exit_status == PCMK_OCF_OK)) {
             if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
                 rsc->st_probe_rc = pcmk_ok; // maps to PCMK_OCF_OK
             } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
                 rsc->st_probe_rc = -ENODEV; // maps to PCMK_OCF_NOT_RUNNING
             }
         }
     }
 
     // Give the user more detail than an OCF code
     if (rc != -pcmk_err_generic) {
         cmd->result.exit_reason = strdup(pcmk_strerror(rc));
     }
 
     /* The recurring timer should not be running at this point in any case, but
      * as a failsafe, stop it if it is.
      */
     stop_recurring_timer(cmd);
 
     /* Reschedule this command if appropriate. If a recurring command is *not*
      * rescheduled, its status must be PCMK_EXEC_CANCELLED, otherwise it will
      * not be removed from recurring_ops by cmd_finalize().
      */
     if (rsc && (cmd->interval_ms > 0)
         && (cmd->result.execution_status != PCMK_EXEC_CANCELLED)) {
         start_recurring_timer(cmd);
     }
 
     cmd_finalize(cmd, rsc);
 }
 
 static void
 lrmd_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data)
 {
     stonith_action_complete(data->userdata, data->rc);
 }
 
 void
 stonith_connection_failed(void)
 {
     GHashTableIter iter;
     GList *cmd_list = NULL;
     GList *cmd_iter = NULL;
     lrmd_rsc_t *rsc = NULL;
     char *key = NULL;
 
     g_hash_table_iter_init(&iter, rsc_list);
     while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) {
         if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
             /* If we registered this fence device, we don't know whether the
              * fencer still has the registration or not. Cause future probes to
              * return PCMK_OCF_UNKNOWN_ERROR until the resource is stopped or
              * started successfully. This is especially important if the
              * controller also went away (possibly due to a cluster layer
              * restart) and won't receive our client notification of any
              * monitors finalized below.
              */
             if (rsc->st_probe_rc == pcmk_ok) {
                 rsc->st_probe_rc = pcmk_err_generic;
             }
 
             if (rsc->active) {
                 cmd_list = g_list_append(cmd_list, rsc->active);
             }
             if (rsc->recurring_ops) {
                 cmd_list = g_list_concat(cmd_list, rsc->recurring_ops);
             }
             if (rsc->pending_ops) {
                 cmd_list = g_list_concat(cmd_list, rsc->pending_ops);
             }
             rsc->pending_ops = rsc->recurring_ops = NULL;
         }
     }
 
     if (!cmd_list) {
         return;
     }
 
     crm_err("Connection to fencer failed, finalizing %d pending operations",
             g_list_length(cmd_list));
     for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
         stonith_action_complete(cmd_iter->data, -ENOTCONN);
     }
     g_list_free(cmd_list);
 }
 
 /*!
  * \internal
  * \brief Execute a stonith resource "start" action
  *
  * Start a stonith resource by registering it with the fencer.
  * (Stonith agents don't have a start command.)
  *
  * \param[in] stonith_api  Connection to fencer
  * \param[in] rsc          Stonith resource to start
  * \param[in] cmd          Start command to execute
  *
  * \return pcmk_ok on success, -errno otherwise
  */
 static int
 execd_stonith_start(stonith_t *stonith_api, lrmd_rsc_t *rsc, lrmd_cmd_t *cmd)
 {
     char *key = NULL;
     char *value = NULL;
     stonith_key_value_t *device_params = NULL;
     int rc = pcmk_ok;
 
     // Convert command parameters to stonith API key/values
     if (cmd->params) {
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, cmd->params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             device_params = stonith_key_value_add(device_params, key, value);
         }
     }
 
     /* The fencer will automatically register devices via CIB notifications
      * when the CIB changes, but to avoid a possible race condition between
      * the fencer receiving the notification and the executor requesting that
      * resource, the executor registers the device as well. The fencer knows how
      * to handle duplicate registrations.
      */
     rc = stonith_api->cmds->register_device(stonith_api, st_opt_sync_call,
                                             cmd->rsc_id, rsc->provider,
                                             rsc->type, device_params);
 
     stonith_key_value_freeall(device_params, 1, 1);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Execute a stonith resource "stop" action
  *
  * Stop a stonith resource by unregistering it with the fencer.
  * (Stonith agents don't have a stop command.)
  *
  * \param[in] stonith_api  Connection to fencer
  * \param[in] rsc          Stonith resource to stop
  *
  * \return pcmk_ok on success, -errno otherwise
  */
 static inline int
 execd_stonith_stop(stonith_t *stonith_api, const lrmd_rsc_t *rsc)
 {
     /* @TODO Failure would indicate a problem communicating with fencer;
      * perhaps we should try reconnecting and retrying a few times?
      */
     return stonith_api->cmds->remove_device(stonith_api, st_opt_sync_call,
                                             rsc->rsc_id);
 }
 
 /*!
  * \internal
  * \brief Initiate a stonith resource agent recurring "monitor" action
  *
  * \param[in] stonith_api  Connection to fencer
  * \param[in] rsc          Stonith resource to monitor
  * \param[in] cmd          Monitor command being executed
  *
  * \return pcmk_ok if monitor was successfully initiated, -errno otherwise
  */
 static inline int
 execd_stonith_monitor(stonith_t *stonith_api, lrmd_rsc_t *rsc, lrmd_cmd_t *cmd)
 {
     int rc = stonith_api->cmds->monitor(stonith_api, 0, cmd->rsc_id,
                                         cmd->timeout / 1000);
 
     rc = stonith_api->cmds->register_callback(stonith_api, rc, 0, 0, cmd,
                                               "lrmd_stonith_callback",
                                               lrmd_stonith_callback);
     if (rc == TRUE) {
         rsc->active = cmd;
         rc = pcmk_ok;
     } else {
         rc = -pcmk_err_generic;
     }
     return rc;
 }
 
 static void
 lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
 {
     int rc = 0;
     bool do_monitor = FALSE;
 
     stonith_t *stonith_api = get_stonith_connection();
 
     if (!stonith_api) {
         rc = -ENOTCONN;
 
     } else if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
         rc = execd_stonith_start(stonith_api, rsc, cmd);
         if (rc == 0) {
             do_monitor = TRUE;
         }
 
     } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
         rc = execd_stonith_stop(stonith_api, rsc);
 
     } else if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
         if (cmd->interval_ms > 0) {
             do_monitor = TRUE;
         } else {
             rc = rsc->st_probe_rc;
         }
     }
 
     if (do_monitor) {
         rc = execd_stonith_monitor(stonith_api, rsc, cmd);
         if (rc == pcmk_ok) {
             // Don't clean up yet, we will find out result of the monitor later
             return;
         }
     }
 
     stonith_action_complete(cmd, rc);
 }
 
 static int
 lrmd_rsc_execute_service_lib(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
 {
     svc_action_t *action = NULL;
     GHashTable *params_copy = NULL;
 
     CRM_ASSERT(rsc);
     CRM_ASSERT(cmd);
 
     crm_trace("Creating action, resource:%s action:%s class:%s provider:%s agent:%s",
               rsc->rsc_id, cmd->action, rsc->class, rsc->provider, rsc->type);
 
 #if SUPPORT_NAGIOS
     /* Recurring operations are cancelled anyway for a stop operation */
     if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)
         && pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
 
         cmd->result.exit_status = PCMK_OCF_OK;
         goto exec_done;
     }
 #endif
 
     params_copy = pcmk__str_table_dup(cmd->params);
 
     action = services__create_resource_action(rsc->rsc_id, rsc->class, rsc->provider,
                                      rsc->type,
                                      normalize_action_name(rsc, cmd->action),
                                      cmd->interval_ms, cmd->timeout,
                                      params_copy, cmd->service_flags);
 
     if (action == NULL) {
         pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
                          PCMK_EXEC_ERROR, strerror(ENOMEM));
         goto exec_done;
     }
 
     if (action->rc != PCMK_OCF_UNKNOWN) {
         pcmk__set_result(&(cmd->result), action->rc, action->status,
                          services__exit_reason(action));
         services_action_free(action);
         goto exec_done;
     }
 
     action->cb_data = cmd;
 
     if (services_action_async(action, action_complete)) {
         /* When services_action_async() returns TRUE, the callback might have
          * been called -- in this case action_complete(), which might free cmd,
          * so cmd cannot be used here.
          */
         return TRUE;
     }
 
     pcmk__set_result(&(cmd->result), action->rc, action->status,
                      services__exit_reason(action));
     services_action_free(action);
     action = NULL;
 
   exec_done:
     cmd_finalize(cmd, rsc);
     return TRUE;
 }
 
 static gboolean
 lrmd_rsc_execute(lrmd_rsc_t * rsc)
 {
     lrmd_cmd_t *cmd = NULL;
 
     CRM_CHECK(rsc != NULL, return FALSE);
 
     if (rsc->active) {
         crm_trace("%s is still active", rsc->rsc_id);
         return TRUE;
     }
 
     if (rsc->pending_ops) {
         GList *first = rsc->pending_ops;
 
         cmd = first->data;
         if (cmd->delay_id) {
             crm_trace
                 ("Command %s %s was asked to run too early, waiting for start_delay timeout of %dms",
                  cmd->rsc_id, cmd->action, cmd->start_delay);
             return TRUE;
         }
         rsc->pending_ops = g_list_remove_link(rsc->pending_ops, first);
         g_list_free_1(first);
 
 #ifdef PCMK__TIME_USE_CGT
         get_current_time(&(cmd->t_run), &(cmd->t_first_run));
 #endif
         cmd->epoch_last_run = time(NULL);
     }
 
     if (!cmd) {
         crm_trace("Nothing further to do for %s", rsc->rsc_id);
         return TRUE;
     }
 
     rsc->active = cmd;          /* only one op at a time for a rsc */
     if (cmd->interval_ms) {
         rsc->recurring_ops = g_list_append(rsc->recurring_ops, cmd);
     }
 
     log_execute(cmd);
 
     if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         lrmd_rsc_execute_stonith(rsc, cmd);
     } else {
         lrmd_rsc_execute_service_lib(rsc, cmd);
     }
 
     return TRUE;
 }
 
 static gboolean
 lrmd_rsc_dispatch(gpointer user_data)
 {
     return lrmd_rsc_execute(user_data);
 }
 
 void
 free_rsc(gpointer data)
 {
     GList *gIter = NULL;
     lrmd_rsc_t *rsc = data;
     int is_stonith = pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
                                   pcmk__str_casei);
 
     gIter = rsc->pending_ops;
     while (gIter != NULL) {
         GList *next = gIter->next;
         lrmd_cmd_t *cmd = gIter->data;
 
         /* command was never executed */
         cmd->result.execution_status = PCMK_EXEC_CANCELLED;
         cmd_finalize(cmd, NULL);
 
         gIter = next;
     }
     /* frees list, but not list elements. */
     g_list_free(rsc->pending_ops);
 
     gIter = rsc->recurring_ops;
     while (gIter != NULL) {
         GList *next = gIter->next;
         lrmd_cmd_t *cmd = gIter->data;
 
         if (is_stonith) {
             cmd->result.execution_status = PCMK_EXEC_CANCELLED;
             /* If a stonith command is in-flight, just mark it as cancelled;
              * it is not safe to finalize/free the cmd until the stonith api
              * says it has either completed or timed out.
              */
             if (rsc->active != cmd) {
                 cmd_finalize(cmd, NULL);
             }
         } else {
             /* This command is already handed off to service library,
              * let service library cancel it and tell us via the callback
              * when it is cancelled. The rsc can be safely destroyed
              * even if we are waiting for the cancel result */
             services_action_cancel(rsc->rsc_id,
                                    normalize_action_name(rsc, cmd->action),
                                    cmd->interval_ms);
         }
 
         gIter = next;
     }
     /* frees list, but not list elements. */
     g_list_free(rsc->recurring_ops);
 
     free(rsc->rsc_id);
     free(rsc->class);
     free(rsc->provider);
     free(rsc->type);
     mainloop_destroy_trigger(rsc->work);
 
     free(rsc);
 }
 
 static int
 process_lrmd_signon(pcmk__client_t *client, xmlNode *request, int call_id,
                     xmlNode **reply)
 {
     int rc = pcmk_ok;
     const char *is_ipc_provider = crm_element_value(request, F_LRMD_IS_IPC_PROVIDER);
     const char *protocol_version = crm_element_value(request, F_LRMD_PROTOCOL_VERSION);
 
     if (compare_version(protocol_version, LRMD_MIN_PROTOCOL_VERSION) < 0) {
         crm_err("Cluster API version must be greater than or equal to %s, not %s",
                 LRMD_MIN_PROTOCOL_VERSION, protocol_version);
         rc = -EPROTO;
     }
 
     if (crm_is_true(is_ipc_provider)) {
 #ifdef PCMK__COMPILE_REMOTE
         if ((client->remote != NULL) && client->remote->tls_handshake_complete) {
             // This is a remote connection from a cluster node's controller
             ipc_proxy_add_provider(client);
         } else {
             rc = -EACCES;
         }
 #else
         rc = -EPROTONOSUPPORT;
 #endif
     }
 
     *reply = create_lrmd_reply(__func__, rc, call_id);
     crm_xml_add(*reply, F_LRMD_OPERATION, CRM_OP_REGISTER);
     crm_xml_add(*reply, F_LRMD_CLIENTID, client->id);
     crm_xml_add(*reply, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 
     return rc;
 }
 
 static int
 process_lrmd_rsc_register(pcmk__client_t *client, uint32_t id, xmlNode *request)
 {
     int rc = pcmk_ok;
     lrmd_rsc_t *rsc = build_rsc_from_xml(request);
     lrmd_rsc_t *dup = g_hash_table_lookup(rsc_list, rsc->rsc_id);
 
     if (dup &&
         pcmk__str_eq(rsc->class, dup->class, pcmk__str_casei) &&
         pcmk__str_eq(rsc->provider, dup->provider, pcmk__str_casei) && pcmk__str_eq(rsc->type, dup->type, pcmk__str_casei)) {
 
         crm_notice("Ignoring duplicate registration of '%s'", rsc->rsc_id);
         free_rsc(rsc);
         return rc;
     }
 
     g_hash_table_replace(rsc_list, rsc->rsc_id, rsc);
     crm_info("Cached agent information for '%s'", rsc->rsc_id);
     return rc;
 }
 
 static xmlNode *
 process_lrmd_get_rsc_info(xmlNode *request, int call_id)
 {
     int rc = pcmk_ok;
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
     xmlNode *reply = NULL;
     lrmd_rsc_t *rsc = NULL;
 
     if (rsc_id == NULL) {
         rc = -ENODEV;
     } else {
         rsc = g_hash_table_lookup(rsc_list, rsc_id);
         if (rsc == NULL) {
             crm_info("Agent information for '%s' not in cache", rsc_id);
             rc = -ENODEV;
         }
     }
 
     reply = create_lrmd_reply(__func__, rc, call_id);
     if (rsc) {
         crm_xml_add(reply, F_LRMD_RSC_ID, rsc->rsc_id);
         crm_xml_add(reply, F_LRMD_CLASS, rsc->class);
         crm_xml_add(reply, F_LRMD_PROVIDER, rsc->provider);
         crm_xml_add(reply, F_LRMD_TYPE, rsc->type);
     }
     return reply;
 }
 
 static int
 process_lrmd_rsc_unregister(pcmk__client_t *client, uint32_t id,
                             xmlNode *request)
 {
     int rc = pcmk_ok;
     lrmd_rsc_t *rsc = NULL;
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
 
     if (!rsc_id) {
         return -ENODEV;
     }
 
     rsc = g_hash_table_lookup(rsc_list, rsc_id);
     if (rsc == NULL) {
         crm_info("Ignoring unregistration of resource '%s', which is not registered",
                  rsc_id);
         return pcmk_ok;
     }
 
     if (rsc->active) {
         /* let the caller know there are still active ops on this rsc to watch for */
         crm_trace("Operation (0x%p) still in progress for unregistered resource %s",
                   rsc->active, rsc_id);
         rc = -EINPROGRESS;
     }
 
     g_hash_table_remove(rsc_list, rsc_id);
 
     return rc;
 }
 
 static int
 process_lrmd_rsc_exec(pcmk__client_t *client, uint32_t id, xmlNode *request)
 {
     lrmd_rsc_t *rsc = NULL;
     lrmd_cmd_t *cmd = NULL;
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
     int call_id;
 
     if (!rsc_id) {
         return -EINVAL;
     }
     if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
         crm_info("Resource '%s' not found (%d active resources)",
                  rsc_id, g_hash_table_size(rsc_list));
         return -ENODEV;
     }
 
     cmd = create_lrmd_cmd(request, client);
     call_id = cmd->call_id;
 
     /* Don't reference cmd after handing it off to be scheduled.
      * The cmd could get merged and freed. */
     schedule_lrmd_cmd(rsc, cmd);
 
     return call_id;
 }
 
 static int
 cancel_op(const char *rsc_id, const char *action, guint interval_ms)
 {
     GList *gIter = NULL;
     lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, rsc_id);
 
     /* How to cancel an action.
      * 1. Check pending ops list, if it hasn't been handed off
      *    to the service library or stonith recurring list remove
      *    it there and that will stop it.
      * 2. If it isn't in the pending ops list, then it's either a
      *    recurring op in the stonith recurring list, or the service
      *    library's recurring list.  Stop it there
      * 3. If not found in any lists, then this operation has either
      *    been executed already and is not a recurring operation, or
      *    never existed.
      */
     if (!rsc) {
         return -ENODEV;
     }
 
     for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
         lrmd_cmd_t *cmd = gIter->data;
 
         if (action_matches(cmd, action, interval_ms)) {
             cmd->result.execution_status = PCMK_EXEC_CANCELLED;
             cmd_finalize(cmd, rsc);
             return pcmk_ok;
         }
     }
 
     if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         /* The service library does not handle stonith operations.
          * We have to handle recurring stonith operations ourselves. */
         for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
             lrmd_cmd_t *cmd = gIter->data;
 
             if (action_matches(cmd, action, interval_ms)) {
                 cmd->result.execution_status = PCMK_EXEC_CANCELLED;
                 if (rsc->active != cmd) {
                     cmd_finalize(cmd, rsc);
                 }
                 return pcmk_ok;
             }
         }
     } else if (services_action_cancel(rsc_id,
                                       normalize_action_name(rsc, action),
                                       interval_ms) == TRUE) {
         /* The service library will tell the action_complete callback function
          * this action was cancelled, which will destroy the cmd and remove
          * it from the recurring_op list. Do not do that in this function
          * if the service library says it cancelled it. */
         return pcmk_ok;
     }
 
     return -EOPNOTSUPP;
 }
 
 static void
 cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id)
 {
     GList *cmd_list = NULL;
     GList *cmd_iter = NULL;
 
     /* Notice a copy of each list is created when concat is called.
      * This prevents odd behavior from occurring when the cmd_list
      * is iterated through later on.  It is possible the cancel_op
      * function may end up modifying the recurring_ops and pending_ops
      * lists.  If we did not copy those lists, our cmd_list iteration
      * could get messed up.*/
     if (rsc->recurring_ops) {
         cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->recurring_ops));
     }
     if (rsc->pending_ops) {
         cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->pending_ops));
     }
     if (!cmd_list) {
         return;
     }
 
     for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
         lrmd_cmd_t *cmd = cmd_iter->data;
 
         if (cmd->interval_ms == 0) {
             continue;
         }
 
         if (client_id && !pcmk__str_eq(cmd->client_id, client_id, pcmk__str_casei)) {
             continue;
         }
 
         cancel_op(rsc->rsc_id, cmd->action, cmd->interval_ms);
     }
     /* frees only the copied list data, not the cmds */
     g_list_free(cmd_list);
 }
 
 static int
 process_lrmd_rsc_cancel(pcmk__client_t *client, uint32_t id, xmlNode *request)
 {
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
     const char *action = crm_element_value(rsc_xml, F_LRMD_RSC_ACTION);
     guint interval_ms = 0;
 
     crm_element_value_ms(rsc_xml, F_LRMD_RSC_INTERVAL, &interval_ms);
 
     if (!rsc_id || !action) {
         return -EINVAL;
     }
 
     return cancel_op(rsc_id, action, interval_ms);
 }
 
 static void
 add_recurring_op_xml(xmlNode *reply, lrmd_rsc_t *rsc)
 {
     xmlNode *rsc_xml = create_xml_node(reply, F_LRMD_RSC);
 
     crm_xml_add(rsc_xml, F_LRMD_RSC_ID, rsc->rsc_id);
     for (GList *item = rsc->recurring_ops; item != NULL; item = item->next) {
         lrmd_cmd_t *cmd = item->data;
         xmlNode *op_xml = create_xml_node(rsc_xml, T_LRMD_RSC_OP);
 
         crm_xml_add(op_xml, F_LRMD_RSC_ACTION,
                     (cmd->real_action? cmd->real_action : cmd->action));
         crm_xml_add_ms(op_xml, F_LRMD_RSC_INTERVAL, cmd->interval_ms);
         crm_xml_add_int(op_xml, F_LRMD_TIMEOUT, cmd->timeout_orig);
     }
 }
 
 static xmlNode *
 process_lrmd_get_recurring(xmlNode *request, int call_id)
 {
     int rc = pcmk_ok;
     const char *rsc_id = NULL;
     lrmd_rsc_t *rsc = NULL;
     xmlNode *reply = NULL;
     xmlNode *rsc_xml = NULL;
 
     // Resource ID is optional
     rsc_xml = first_named_child(request, F_LRMD_CALLDATA);
     if (rsc_xml) {
         rsc_xml = first_named_child(rsc_xml, F_LRMD_RSC);
     }
     if (rsc_xml) {
         rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
     }
 
     // If resource ID is specified, resource must exist
     if (rsc_id != NULL) {
         rsc = g_hash_table_lookup(rsc_list, rsc_id);
         if (rsc == NULL) {
             crm_info("Resource '%s' not found (%d active resources)",
                      rsc_id, g_hash_table_size(rsc_list));
             rc = -ENODEV;
         }
     }
 
     reply = create_lrmd_reply(__func__, rc, call_id);
 
     // If resource ID is not specified, check all resources
     if (rsc_id == NULL) {
         GHashTableIter iter;
         char *key = NULL;
 
         g_hash_table_iter_init(&iter, rsc_list);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key,
                                       (gpointer *) &rsc)) {
             add_recurring_op_xml(reply, rsc);
         }
     } else if (rsc) {
         add_recurring_op_xml(reply, rsc);
     }
     return reply;
 }
 
 void
 process_lrmd_message(pcmk__client_t *client, uint32_t id, xmlNode *request)
 {
     int rc = pcmk_ok;
     int call_id = 0;
     const char *op = crm_element_value(request, F_LRMD_OPERATION);
     int do_reply = 0;
     int do_notify = 0;
     xmlNode *reply = NULL;
 
     /* Certain IPC commands may be done only by privileged users (i.e. root or
      * hacluster), because they would otherwise provide a means of bypassing
      * ACLs.
      */
     bool allowed = pcmk_is_set(client->flags, pcmk__client_privileged);
 
     crm_trace("Processing %s operation from %s", op, client->id);
     crm_element_value_int(request, F_LRMD_CALLID, &call_id);
 
     if (pcmk__str_eq(op, CRM_OP_IPC_FWD, pcmk__str_none)) {
 #ifdef PCMK__COMPILE_REMOTE
         if (allowed) {
             ipc_proxy_forward_client(client, request);
         } else {
             rc = -EACCES;
         }
 #else
         rc = -EPROTONOSUPPORT;
 #endif
         do_reply = 1;
     } else if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
         rc = process_lrmd_signon(client, request, call_id, &reply);
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_REG, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_rsc_register(client, id, request);
             do_notify = 1;
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_INFO, pcmk__str_none)) {
         if (allowed) {
             reply = process_lrmd_get_rsc_info(request, call_id);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_rsc_unregister(client, id, request);
             /* don't notify anyone about failed un-registers */
             if (rc == pcmk_ok || rc == -EINPROGRESS) {
                 do_notify = 1;
             }
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_rsc_exec(client, id, request);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_CANCEL, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_rsc_cancel(client, id, request);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_POKE, pcmk__str_none)) {
         do_notify = 1;
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_CHECK, pcmk__str_none)) {
         if (allowed) {
             xmlNode *data = get_message_xml(request, F_LRMD_CALLDATA);
 
             CRM_LOG_ASSERT(data != NULL);
             pcmk__valid_sbd_timeout(crm_element_value(data, F_LRMD_WATCHDOG));
         } else {
             rc = -EACCES;
         }
     } else if (pcmk__str_eq(op, LRMD_OP_ALERT_EXEC, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_alert_exec(client, id, request);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_GET_RECURRING, pcmk__str_none)) {
         if (allowed) {
             reply = process_lrmd_get_recurring(request, call_id);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else {
         rc = -EOPNOTSUPP;
         do_reply = 1;
         crm_err("Unknown IPC request '%s' from client %s",
                 op, pcmk__client_name(client));
     }
 
     if (rc == -EACCES) {
         crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
                  op, pcmk__client_name(client));
     }
 
     crm_debug("Processed %s operation from %s: rc=%d, reply=%d, notify=%d",
               op, client->id, rc, do_reply, do_notify);
 
     if (do_reply) {
         int send_rc = pcmk_rc_ok;
 
         if (reply == NULL) {
             reply = create_lrmd_reply(__func__, rc, call_id);
         }
         send_rc = lrmd_server_send_reply(client, id, reply);
         free_xml(reply);
         if (send_rc != pcmk_rc_ok) {
             crm_warn("Reply to client %s failed: %s " CRM_XS " rc=%d",
                      pcmk__client_name(client), pcmk_rc_str(send_rc), send_rc);
         }
     }
 
     if (do_notify) {
         send_generic_notify(rc, request);
     }
 }
diff --git a/daemons/pacemakerd/pcmkd_subdaemons.c b/daemons/pacemakerd/pcmkd_subdaemons.c
index b1aea780f2..9556d7409c 100644
--- a/daemons/pacemakerd/pcmkd_subdaemons.c
+++ b/daemons/pacemakerd/pcmkd_subdaemons.c
@@ -1,840 +1,841 @@
 /*
  * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include "pacemakerd.h"
 
 #include <errno.h>
 #include <grp.h>
 #include <signal.h>
 #include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
 
 #include <crm/cluster.h>
 #include <crm/msg_xml.h>
 
 typedef struct pcmk_child_s {
     pid_t pid;
     int respawn_count;
     bool respawn;
     const char *name;
     const char *uid;
     const char *command;
     const char *endpoint;  /* IPC server name */
     bool needs_cluster;
 
     /* Anything below here will be dynamically initialized */
     bool needs_retry;
     bool active_before_startup;
 } pcmk_child_t;
 
 #define PCMK_PROCESS_CHECK_INTERVAL 5
 #define SHUTDOWN_ESCALATION_PERIOD 180000  /* 3m */
 
 /* Index into the array below */
 #define PCMK_CHILD_CONTROLD  5
 
 static pcmk_child_t pcmk_children[] = {
     {
         0, 0, true,  "pacemaker-based", CRM_DAEMON_USER,
         CRM_DAEMON_DIR "/pacemaker-based", PCMK__SERVER_BASED_RO,
         true
     },
     {
         0, 0, true, "pacemaker-fenced", NULL,
         CRM_DAEMON_DIR "/pacemaker-fenced", "stonith-ng",
         true
     },
     {
         0, 0, true,  "pacemaker-execd", NULL,
         CRM_DAEMON_DIR "/pacemaker-execd", CRM_SYSTEM_LRMD,
         false
     },
     {
         0, 0, true, "pacemaker-attrd", CRM_DAEMON_USER,
         CRM_DAEMON_DIR "/pacemaker-attrd", T_ATTRD,
         true
     },
     {
         0, 0, true, "pacemaker-schedulerd", CRM_DAEMON_USER,
         CRM_DAEMON_DIR "/pacemaker-schedulerd", CRM_SYSTEM_PENGINE,
         false
     },
     {
         0, 0, true, "pacemaker-controld", CRM_DAEMON_USER,
         CRM_DAEMON_DIR "/pacemaker-controld", CRM_SYSTEM_CRMD,
         true
     },
 };
 
 static char *opts_default[] = { NULL, NULL };
 static char *opts_vgrind[] = { NULL, NULL, NULL, NULL, NULL };
 
 crm_trigger_t *shutdown_trigger = NULL;
 crm_trigger_t *startup_trigger = NULL;
 
 /* When contacted via pacemakerd-api by a client having sbd in
  * the name we assume it is sbd-daemon which wants to know
  * if pacemakerd shutdown gracefully.
  * Thus when everything is shutdown properly pacemakerd
  * waits till it has reported the graceful completion of
  * shutdown to sbd and just when sbd-client closes the
  * connection we can assume that the report has arrived
  * properly so that pacemakerd can finally exit.
  * Following two variables are used to track that handshake.
  */
 unsigned int shutdown_complete_state_reported_to = 0;
 gboolean shutdown_complete_state_reported_client_closed = FALSE;
 
 /* state we report when asked via pacemakerd-api status-ping */
 const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
 gboolean running_with_sbd = FALSE; /* local copy */
 
 GMainLoop *mainloop = NULL;
 
 static gboolean fatal_error = FALSE;
 static bool global_keep_tracking = false;
 
 static gboolean check_active_before_startup_processes(gpointer user_data);
 static int child_liveness(pcmk_child_t *child);
 static gboolean escalate_shutdown(gpointer data);
 static int start_child(pcmk_child_t * child);
 static void pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode);
 static void pcmk_process_exit(pcmk_child_t * child);
 static gboolean pcmk_shutdown_worker(gpointer user_data);
 static gboolean stop_child(pcmk_child_t * child, int signal);
 
 static bool
 pcmkd_cluster_connected(void)
 {
 #if SUPPORT_COROSYNC
     return pcmkd_corosync_connected();
 #else
     return true;
 #endif
 }
 
 static gboolean
 check_active_before_startup_processes(gpointer user_data)
 {
     gboolean keep_tracking = FALSE;
 
     for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
         if (!pcmk_children[i].active_before_startup) {
             /* we are already tracking it as a child process. */
             continue;
         } else {
             int rc = child_liveness(&pcmk_children[i]);
 
             switch (rc) {
                 case pcmk_rc_ok:
                     break;
                 case pcmk_rc_ipc_unresponsive:
                 case pcmk_rc_ipc_pid_only: // This case: it was previously OK
                     if (pcmk_children[i].respawn) {
                         crm_err("%s[%lld] terminated%s", pcmk_children[i].name,
                                 (long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[i].pid),
                                 (rc == pcmk_rc_ipc_pid_only)? " as IPC server" : "");
                     } else {
                         /* orderly shutdown */
                         crm_notice("%s[%lld] terminated%s", pcmk_children[i].name,
                                    (long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[i].pid),
                                    (rc == pcmk_rc_ipc_pid_only)? " as IPC server" : "");
                     }
                     pcmk_process_exit(&(pcmk_children[i]));
                     continue;
                 default:
                     crm_exit(CRM_EX_FATAL);
                     break;  /* static analysis/noreturn */
             }
         }
         /* at least one of the processes found at startup
          * is still going, so keep this recurring timer around */
         keep_tracking = TRUE;
     }
 
     global_keep_tracking = keep_tracking;
     return keep_tracking;
 }
 
 static gboolean
 escalate_shutdown(gpointer data)
 {
     pcmk_child_t *child = data;
 
     if (child->pid == PCMK__SPECIAL_PID) {
         pcmk_process_exit(child);
 
     } else if (child->pid != 0) {
         /* Use SIGSEGV instead of SIGKILL to create a core so we can see what it was up to */
         crm_err("Child %s not terminating in a timely manner, forcing", child->name);
         stop_child(child, SIGSEGV);
     }
     return FALSE;
 }
 
 static void
 pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
 {
     pcmk_child_t *child = mainloop_child_userdata(p);
     const char *name = mainloop_child_name(p);
 
     if (signo) {
         do_crm_log(((signo == SIGKILL)? LOG_WARNING : LOG_ERR),
-                   "%s[%d] terminated with signal %d (core=%d)",
-                   name, pid, signo, core);
+                   "%s[%d] terminated with signal %d (%s)%s",
+                   name, pid, signo, strsignal(signo),
+                   (core? " and dumped core" : ""));
 
     } else {
         switch(exitcode) {
             case CRM_EX_OK:
                 crm_info("%s[%d] exited with status %d (%s)",
                          name, pid, exitcode, crm_exit_str(exitcode));
                 break;
 
             case CRM_EX_FATAL:
                 crm_warn("Shutting cluster down because %s[%d] had fatal failure",
                          name, pid);
                 child->respawn = false;
                 fatal_error = TRUE;
                 pcmk_shutdown(SIGTERM);
                 break;
 
             case CRM_EX_PANIC:
                 crm_emerg("%s[%d] instructed the machine to reset", name, pid);
                 child->respawn = false;
                 fatal_error = TRUE;
                 pcmk__panic(__func__);
                 pcmk_shutdown(SIGTERM);
                 break;
 
             default:
                 crm_err("%s[%d] exited with status %d (%s)",
                         name, pid, exitcode, crm_exit_str(exitcode));
                 break;
         }
     }
 
     pcmk_process_exit(child);
 }
 
 static void
 pcmk_process_exit(pcmk_child_t * child)
 {
     child->pid = 0;
     child->active_before_startup = false;
 
     child->respawn_count += 1;
     if (child->respawn_count > MAX_RESPAWN) {
         crm_err("Child respawn count exceeded by %s", child->name);
         child->respawn = false;
     }
 
     if (shutdown_trigger) {
         /* resume step-wise shutdown (returned TRUE yields no parallelizing) */
         mainloop_set_trigger(shutdown_trigger);
 
     } else if (!child->respawn) {
         /* nothing to do */
 
     } else if (crm_is_true(getenv("PCMK_fail_fast"))) {
         crm_err("Rebooting system because of %s", child->name);
         pcmk__panic(__func__);
 
     } else if (child_liveness(child) == pcmk_rc_ok) {
         crm_warn("One-off suppressing strict respawning of a child process %s,"
                  " appears alright per %s IPC end-point",
                  child->name, child->endpoint);
         /* need to monitor how it evolves, and start new process if badly */
         child->active_before_startup = true;
         if (!global_keep_tracking) {
             global_keep_tracking = true;
             g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL,
                                   check_active_before_startup_processes, NULL);
         }
 
     } else {
         if (child->needs_cluster && !pcmkd_cluster_connected()) {
             crm_notice("Skipping cluster-based subdaemon %s until cluster returns",
                        child->name);
             child->needs_retry = true;
             return;
         }
 
         crm_notice("Respawning failed child process: %s", child->name);
         start_child(child);
     }
 }
 
 static gboolean
 pcmk_shutdown_worker(gpointer user_data)
 {
     static int phase = PCMK__NELEM(pcmk_children) - 1;
     static time_t next_log = 0;
 
     if (phase == PCMK__NELEM(pcmk_children) - 1) {
         crm_notice("Shutting down Pacemaker");
         pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN;
     }
 
     for (; phase >= 0; phase--) {
         pcmk_child_t *child = &(pcmk_children[phase]);
 
         if (child->pid != 0) {
             time_t now = time(NULL);
 
             if (child->respawn) {
                 if (child->pid == PCMK__SPECIAL_PID) {
                     crm_warn("The process behind %s IPC cannot be"
                              " terminated, so either wait the graceful"
                              " period of %ld s for its native termination"
                              " if it vitally depends on some other daemons"
                              " going down in a controlled way already,"
                              " or locate and kill the correct %s process"
                              " on your own; set PCMK_fail_fast=1 to avoid"
                              " this altogether next time around",
                              child->name, (long) SHUTDOWN_ESCALATION_PERIOD,
                              child->command);
                 }
                 next_log = now + 30;
                 child->respawn = false;
                 stop_child(child, SIGTERM);
                 if (phase < PCMK_CHILD_CONTROLD) {
                     g_timeout_add(SHUTDOWN_ESCALATION_PERIOD,
                                   escalate_shutdown, child);
                 }
 
             } else if (now >= next_log) {
                 next_log = now + 30;
                 crm_notice("Still waiting for %s to terminate "
                            CRM_XS " pid=%lld",
                            child->name, (long long) child->pid);
             }
             return TRUE;
         }
 
         /* cleanup */
         crm_debug("%s confirmed stopped", child->name);
         child->pid = 0;
     }
 
     crm_notice("Shutdown complete");
     pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE;
     if (!fatal_error && running_with_sbd &&
         pcmk__get_sbd_sync_resource_startup() &&
         !shutdown_complete_state_reported_client_closed) {
         crm_notice("Waiting for SBD to pick up shutdown-complete-state.");
         return TRUE;
     }
 
     {
         const char *delay = pcmk__env_option(PCMK__ENV_SHUTDOWN_DELAY);
         if(delay) {
             sync();
             pcmk__sleep_ms(crm_get_msec(delay));
         }
     }
 
     g_main_loop_quit(mainloop);
 
     if (fatal_error) {
         crm_notice("Shutting down and staying down after fatal error");
 #ifdef SUPPORT_COROSYNC
         pcmkd_shutdown_corosync();
 #endif
         crm_exit(CRM_EX_FATAL);
     }
 
     return TRUE;
 }
 
 /* TODO once libqb is taught to juggle with IPC end-points carried over as
         bare file descriptor (https://github.com/ClusterLabs/libqb/issues/325)
         it shall hand over these descriptors here if/once they are successfully
         pre-opened in (presumably) child_liveness(), to avoid any remaining
         room for races */
  // \return Standard Pacemaker return code
 static int
 start_child(pcmk_child_t * child)
 {
     uid_t uid = 0;
     gid_t gid = 0;
     gboolean use_valgrind = FALSE;
     gboolean use_callgrind = FALSE;
     const char *env_valgrind = getenv("PCMK_valgrind_enabled");
     const char *env_callgrind = getenv("PCMK_callgrind_enabled");
 
     child->active_before_startup = false;
 
     if (child->command == NULL) {
         crm_info("Nothing to do for child \"%s\"", child->name);
         return pcmk_rc_ok;
     }
 
     if (env_callgrind != NULL && crm_is_true(env_callgrind)) {
         use_callgrind = TRUE;
         use_valgrind = TRUE;
 
     } else if (env_callgrind != NULL && strstr(env_callgrind, child->name)) {
         use_callgrind = TRUE;
         use_valgrind = TRUE;
 
     } else if (env_valgrind != NULL && crm_is_true(env_valgrind)) {
         use_valgrind = TRUE;
 
     } else if (env_valgrind != NULL && strstr(env_valgrind, child->name)) {
         use_valgrind = TRUE;
     }
 
     if (use_valgrind && strlen(VALGRIND_BIN) == 0) {
         crm_warn("Cannot enable valgrind for %s:"
                  " The location of the valgrind binary is unknown", child->name);
         use_valgrind = FALSE;
     }
 
     if (child->uid) {
         if (crm_user_lookup(child->uid, &uid, &gid) < 0) {
             crm_err("Invalid user (%s) for %s: not found", child->uid, child->name);
             return EACCES;
         }
         crm_info("Using uid=%u and group=%u for process %s", uid, gid, child->name);
     }
 
     child->pid = fork();
     CRM_ASSERT(child->pid != -1);
 
     if (child->pid > 0) {
         /* parent */
         mainloop_child_add(child->pid, 0, child->name, child, pcmk_child_exit);
 
         crm_info("Forked child %lld for process %s%s",
                  (long long) child->pid, child->name,
                  use_valgrind ? " (valgrind enabled: " VALGRIND_BIN ")" : "");
         return pcmk_rc_ok;
 
     } else {
         /* Start a new session */
         (void)setsid();
 
         /* Setup the two alternate arg arrays */
         opts_vgrind[0] = strdup(VALGRIND_BIN);
         if (use_callgrind) {
             opts_vgrind[1] = strdup("--tool=callgrind");
             opts_vgrind[2] = strdup("--callgrind-out-file=" CRM_STATE_DIR "/callgrind.out.%p");
             opts_vgrind[3] = strdup(child->command);
             opts_vgrind[4] = NULL;
         } else {
             opts_vgrind[1] = strdup(child->command);
             opts_vgrind[2] = NULL;
             opts_vgrind[3] = NULL;
             opts_vgrind[4] = NULL;
         }
         opts_default[0] = strdup(child->command);
 
         if(gid) {
             // Whether we need root group access to talk to cluster layer
             bool need_root_group = TRUE;
 
             if (is_corosync_cluster()) {
                 /* Corosync clusters can drop root group access, because we set
                  * uidgid.gid.${gid}=1 via CMAP, which allows these processes to
                  * connect to corosync.
                  */
                 need_root_group = FALSE;
             }
 
             // Drop root group access if not needed
             if (!need_root_group && (setgid(gid) < 0)) {
                 crm_warn("Could not set group to %d: %s", gid, strerror(errno));
             }
 
             /* Initialize supplementary groups to only those always granted to
              * the user, plus haclient (so we can access IPC).
              */
             if (initgroups(child->uid, gid) < 0) {
                 crm_err("Cannot initialize groups for %s: %s (%d)", child->uid, pcmk_strerror(errno), errno);
             }
         }
 
         if (uid && setuid(uid) < 0) {
             crm_warn("Could not set user to %s (id %d): %s",
                      child->uid, uid, strerror(errno));
         }
 
         pcmk__close_fds_in_child(true);
 
         pcmk__open_devnull(O_RDONLY);   // stdin (fd 0)
         pcmk__open_devnull(O_WRONLY);   // stdout (fd 1)
         pcmk__open_devnull(O_WRONLY);   // stderr (fd 2)
 
         if (use_valgrind) {
             (void)execvp(VALGRIND_BIN, opts_vgrind);
         } else {
             (void)execvp(child->command, opts_default);
         }
         crm_crit("Could not execute %s: %s", child->command, strerror(errno));
         crm_exit(CRM_EX_FATAL);
     }
     return pcmk_rc_ok;          /* never reached */
 }
 
 /*!
  * \internal
  * \brief Check the liveness of the child based on IPC name and PID if tracked
  *
  * \param[inout] child  Child tracked data
  *
  * \return Standard Pacemaker return code
  *
  * \note Return codes of particular interest include pcmk_rc_ipc_unresponsive
  *       indicating that no trace of IPC liveness was detected,
  *       pcmk_rc_ipc_unauthorized indicating that the IPC endpoint is blocked by
  *       an unauthorized process, and pcmk_rc_ipc_pid_only indicating that
  *       the child is up by PID but not IPC end-point (possibly starting).
  * \note This function doesn't modify any of \p child members but \c pid,
  *       and is not actively toying with processes as such but invoking
  *       \c stop_child in one particular case (there's for some reason
  *       a different authentic holder of the IPC end-point).
  */
 static int
 child_liveness(pcmk_child_t *child)
 {
     uid_t cl_uid = 0;
     gid_t cl_gid = 0;
     const uid_t root_uid = 0;
     const gid_t root_gid = 0;
     const uid_t *ref_uid;
     const gid_t *ref_gid;
     int rc = pcmk_rc_ipc_unresponsive;
     pid_t ipc_pid = 0;
 
     if (child->endpoint == NULL
             && (child->pid <= 0 || child->pid == PCMK__SPECIAL_PID)) {
         crm_err("Cannot track child %s for missing both API end-point and PID",
                 child->name);
         rc = EINVAL; // Misuse of function when child is not trackable
 
     } else if (child->endpoint != NULL) {
         int legacy_rc = pcmk_ok;
 
         if (child->uid == NULL) {
             ref_uid = &root_uid;
             ref_gid = &root_gid;
         } else {
             ref_uid = &cl_uid;
             ref_gid = &cl_gid;
             legacy_rc = pcmk_daemon_user(&cl_uid, &cl_gid);
         }
 
         if (legacy_rc < 0) {
             rc = pcmk_legacy2rc(legacy_rc);
             crm_err("Could not find user and group IDs for user %s: %s "
                     CRM_XS " rc=%d", CRM_DAEMON_USER, pcmk_rc_str(rc), rc);
         } else {
             rc = pcmk__ipc_is_authentic_process_active(child->endpoint,
                                                        *ref_uid, *ref_gid,
                                                        &ipc_pid);
             if ((rc == pcmk_rc_ok) || (rc == pcmk_rc_ipc_unresponsive)) {
                 if (child->pid <= 0) {
                     /* If rc is pcmk_rc_ok, ipc_pid is nonzero and this
                      * initializes a new child. If rc is
                      * pcmk_rc_ipc_unresponsive, ipc_pid is zero, and we will
                      * investigate further.
                      */
                     child->pid = ipc_pid;
                 } else if ((ipc_pid != 0) && (child->pid != ipc_pid)) {
                     /* An unexpected (but authorized) process is responding to
                      * IPC. Investigate further.
                      */
                     rc = pcmk_rc_ipc_unresponsive;
                 }
             }
         }
     }
 
     if (rc == pcmk_rc_ipc_unresponsive) {
         /* If we get here, a child without IPC is being tracked, no IPC liveness
          * has been detected, or IPC liveness has been detected with an
          * unexpected (but authorized) process. This is safe on FreeBSD since
          * the only change possible from a proper child's PID into "special" PID
          * of 1 behind more loosely related process.
          */
         int ret = pcmk__pid_active(child->pid, child->name);
 
         if (ipc_pid && ((ret != pcmk_rc_ok)
                         || ipc_pid == PCMK__SPECIAL_PID
                         || (pcmk__pid_active(ipc_pid,
                                              child->name) == pcmk_rc_ok))) {
             /* An unexpected (but authorized) process was detected at the IPC
              * endpoint, and either it is active, or the child we're tracking is
              * not.
              */
 
             if (ret == pcmk_rc_ok) {
                 /* The child we're tracking is active. Kill it, and adopt the
                  * detected process. This assumes that our children don't fork
                  * (thus getting a different PID owning the IPC), but rather the
                  * tracking got out of sync because of some means external to
                  * Pacemaker, and adopting the detected process is better than
                  * killing it and possibly having to spawn a new child.
                  */
                 /* not possessing IPC, afterall (what about corosync CPG?) */
                 stop_child(child, SIGKILL);
             }
             rc = pcmk_rc_ok;
             child->pid = ipc_pid;
         } else if (ret == pcmk_rc_ok) {
             // Our tracked child's PID was found active, but not its IPC
             rc = pcmk_rc_ipc_pid_only;
         } else if ((child->pid == 0) && (ret == EINVAL)) {
             // FreeBSD can return EINVAL
             rc = pcmk_rc_ipc_unresponsive;
         } else {
             switch (ret) {
                 case EACCES:
                     rc = pcmk_rc_ipc_unauthorized;
                     break;
                 case ESRCH:
                     rc = pcmk_rc_ipc_unresponsive;
                     break;
                 default:
                     rc = ret;
                     break;
             }
         }
     }
     return rc;
 }
 
 /*!
  * \internal
  * \brief Initial one-off check of the pre-existing "child" processes
  *
  * With "child" process, we mean the subdaemon that defines an API end-point
  * (all of them do as of the comment) -- the possible complement is skipped
  * as it is deemed it has no such shared resources to cause conflicts about,
  * hence it can presumably be started anew without hesitation.
  * If that won't hold true in the future, the concept of a shared resource
  * will have to be generalized beyond the API end-point.
  *
  * For boundary cases that the "child" is still starting (IPC end-point is yet
  * to be witnessed), or more rarely (practically FreeBSD only), when there's
  * a pre-existing "untrackable" authentic process, we give the situation some
  * time to possibly unfold in the right direction, meaning that said socket
  * will appear or the unattainable process will disappear per the observable
  * IPC, respectively.
  *
  * \return Standard Pacemaker return code
  *
  * \note Since this gets run at the very start, \c respawn_count fields
  *       for particular children get temporarily overloaded with "rounds
  *       of waiting" tracking, restored once we are about to finish with
  *       success (i.e. returning value >=0) and will remain unrestored
  *       otherwise.  One way to suppress liveness detection logic for
  *       particular child is to set the said value to a negative number.
  */
 #define WAIT_TRIES 4  /* together with interleaved sleeps, worst case ~ 1s */
 int
 find_and_track_existing_processes(void)
 {
     bool tracking = false;
     bool wait_in_progress;
     int rc;
     size_t i, rounds;
 
     for (rounds = 1; rounds <= WAIT_TRIES; rounds++) {
         wait_in_progress = false;
         for (i = 0; i < PCMK__NELEM(pcmk_children); i++) {
 
             if ((pcmk_children[i].endpoint == NULL)
                 || (pcmk_children[i].respawn_count < 0)) {
                 continue;
             }
 
             rc = child_liveness(&pcmk_children[i]);
             if (rc == pcmk_rc_ipc_unresponsive) {
                 /* As a speculation, don't give up if there are more rounds to
                  * come for other reasons, but don't artificially wait just
                  * because of this, since we would preferably start ASAP.
                  */
                 continue;
             }
 
             pcmk_children[i].respawn_count = rounds;
             switch (rc) {
                 case pcmk_rc_ok:
                     if (pcmk_children[i].pid == PCMK__SPECIAL_PID) {
                         if (crm_is_true(getenv("PCMK_fail_fast"))) {
                             crm_crit("Cannot reliably track pre-existing"
                                      " authentic process behind %s IPC on this"
                                      " platform and PCMK_fail_fast requested",
                                      pcmk_children[i].endpoint);
                             return EOPNOTSUPP;
                         } else if (pcmk_children[i].respawn_count == WAIT_TRIES) {
                             crm_notice("Assuming pre-existing authentic, though"
                                        " on this platform untrackable, process"
                                        " behind %s IPC is stable (was in %d"
                                        " previous samples) so rather than"
                                        " bailing out (PCMK_fail_fast not"
                                        " requested), we just switch to a less"
                                        " optimal IPC liveness monitoring"
                                        " (not very suitable for heavy load)",
                                        pcmk_children[i].name, WAIT_TRIES - 1);
                             crm_warn("The process behind %s IPC cannot be"
                                      " terminated, so the overall shutdown"
                                      " will get delayed implicitly (%ld s),"
                                      " which serves as a graceful period for"
                                      " its native termination if it vitally"
                                      " depends on some other daemons going"
                                      " down in a controlled way already",
                                      pcmk_children[i].name,
                                      (long) SHUTDOWN_ESCALATION_PERIOD);
                         } else {
                             wait_in_progress = true;
                             crm_warn("Cannot reliably track pre-existing"
                                      " authentic process behind %s IPC on this"
                                      " platform, can still disappear in %d"
                                      " attempt(s)", pcmk_children[i].endpoint,
                                      WAIT_TRIES - pcmk_children[i].respawn_count);
                             continue;
                         }
                     }
                     crm_notice("Tracking existing %s process (pid=%lld)",
                                pcmk_children[i].name,
                                (long long) PCMK__SPECIAL_PID_AS_0(
                                                pcmk_children[i].pid));
                     pcmk_children[i].respawn_count = -1;  /* 0~keep watching */
                     pcmk_children[i].active_before_startup = true;
                     tracking = true;
                     break;
                 case pcmk_rc_ipc_pid_only:
                     if (pcmk_children[i].respawn_count == WAIT_TRIES) {
                         crm_crit("%s IPC end-point for existing authentic"
                                  " process %lld did not (re)appear",
                                  pcmk_children[i].endpoint,
                                  (long long) PCMK__SPECIAL_PID_AS_0(
                                                  pcmk_children[i].pid));
                         return rc;
                     }
                     wait_in_progress = true;
                     crm_warn("Cannot find %s IPC end-point for existing"
                              " authentic process %lld, can still (re)appear"
                              " in %d attempts (?)",
                              pcmk_children[i].endpoint,
                              (long long) PCMK__SPECIAL_PID_AS_0(
                                              pcmk_children[i].pid),
                              WAIT_TRIES - pcmk_children[i].respawn_count);
                     continue;
                 default:
                     crm_crit("Checked liveness of %s: %s " CRM_XS " rc=%d",
                              pcmk_children[i].name, pcmk_rc_str(rc), rc);
                     return rc;
             }
         }
         if (!wait_in_progress) {
             break;
         }
         pcmk__sleep_ms(250); // Wait a bit for changes to possibly happen
     }
     for (i = 0; i < PCMK__NELEM(pcmk_children); i++) {
         pcmk_children[i].respawn_count = 0;  /* restore pristine state */
     }
 
     if (tracking) {
         g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL,
                               check_active_before_startup_processes, NULL);
     }
     return pcmk_rc_ok;
 }
 
 gboolean
 init_children_processes(void *user_data)
 {
     /* start any children that have not been detected */
     for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
         if (pcmk_children[i].pid != 0) {
             /* we are already tracking it */
             continue;
         }
 
         start_child(&(pcmk_children[i]));
     }
 
     /* From this point on, any daemons being started will be due to
      * respawning rather than node start.
      *
      * This may be useful for the daemons to know
      */
     setenv("PCMK_respawned", "true", 1);
     pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING;
     return TRUE;
 }
 
 void
 pcmk_shutdown(int nsig)
 {
     if (shutdown_trigger == NULL) {
         shutdown_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, pcmk_shutdown_worker, NULL);
     }
     mainloop_set_trigger(shutdown_trigger);
 }
 
 void
 restart_cluster_subdaemons(void)
 {
     for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
         if (!pcmk_children[i].needs_retry || pcmk_children[i].pid != 0) {
             continue;
         }
 
         crm_notice("Respawning cluster-based subdaemon: %s", pcmk_children[i].name);
         if (start_child(&pcmk_children[i])) {
             pcmk_children[i].needs_retry = false;
         }
     }
 }
 
 static gboolean
 stop_child(pcmk_child_t * child, int signal)
 {
     if (signal == 0) {
         signal = SIGTERM;
     }
 
     /* why to skip PID of 1?
        - FreeBSD ~ how untrackable process behind IPC is masqueraded as
        - elsewhere: how "init" task is designated; in particular, in systemd
          arrangement of socket-based activation, this is pretty real */
     if (child->command == NULL || child->pid == PCMK__SPECIAL_PID) {
         crm_debug("Nothing to do for child \"%s\" (process %lld)",
                   child->name, (long long) PCMK__SPECIAL_PID_AS_0(child->pid));
         return TRUE;
     }
 
     if (child->pid <= 0) {
         crm_trace("Client %s not running", child->name);
         return TRUE;
     }
 
     errno = 0;
     if (kill(child->pid, signal) == 0) {
         crm_notice("Stopping %s "CRM_XS" sent signal %d to process %lld",
                    child->name, signal, (long long) child->pid);
 
     } else {
         crm_err("Could not stop %s (process %lld) with signal %d: %s",
                 child->name, (long long) child->pid, signal, strerror(errno));
     }
 
     return TRUE;
 }
 
diff --git a/doc/sphinx/Pacemaker_Explained/resources.rst b/doc/sphinx/Pacemaker_Explained/resources.rst
index 773188c7cc..2737354bc7 100644
--- a/doc/sphinx/Pacemaker_Explained/resources.rst
+++ b/doc/sphinx/Pacemaker_Explained/resources.rst
@@ -1,1036 +1,1063 @@
 .. _resource:
 
 Cluster Resources
 -----------------
 
 .. _s-resource-primitive:
 
 What is a Cluster Resource?
 ###########################
 
 .. index::
    single: resource
 
 A resource is a service made highly available by a cluster.
 The simplest type of resource, a *primitive* resource, is described
 in this chapter. More complex forms, such as groups and clones,
 are described in later chapters.
 
 Every primitive resource has a *resource agent*. A resource agent is an
 external program that abstracts the service it provides and present a
 consistent view to the cluster.
 
 This allows the cluster to be agnostic about the resources it manages.
 The cluster doesn't need to understand how the resource works because
 it relies on the resource agent to do the right thing when given a
 **start**, **stop** or **monitor** command. For this reason, it is crucial
 that resource agents are well-tested.
 
 Typically, resource agents come in the form of shell scripts. However,
 they can be written using any technology (such as C, Python or Perl)
 that the author is comfortable with.
 
 .. _s-resource-supported:
 
 .. index::
    single: resource; class
  
 Resource Classes
 ################
 
 Pacemaker supports several classes of agents:
 
 * OCF
 * LSB
 * Systemd
 * Upstart (deprecated)
 * Service
 * Fencing
 * Nagios Plugins
 
 .. index::
    single: resource; OCF
    single: OCF; resources
    single: Open Cluster Framework; resources
 
 Open Cluster Framework
 ______________________
 
 The OCF standard [#]_ is basically an extension of the Linux Standard
 Base conventions for init scripts to:
 
 * support parameters,
 * make them self-describing, and
 * make them extensible
 
 OCF specs have strict definitions of the exit codes that actions must return [#]_.
 
 The cluster follows these specifications exactly, and giving the wrong
 exit code will cause the cluster to behave in ways you will likely
 find puzzling and annoying.  In particular, the cluster needs to
 distinguish a completely stopped resource from one which is in some
 erroneous and indeterminate state.
 
 Parameters are passed to the resource agent as environment variables, with the
 special prefix ``OCF_RESKEY_``.  So, a parameter which the user thinks
 of as ``ip`` will be passed to the resource agent as ``OCF_RESKEY_ip``.  The
 number and purpose of the parameters is left to the resource agent; however,
 the resource agent should use the **meta-data** command to advertise any that it
 supports.
 
 The OCF class is the most preferred as it is an industry standard,
 highly flexible (allowing parameters to be passed to agents in a
 non-positional manner) and self-describing.
 
 For more information, see the
 `reference <http://www.linux-ha.org/wiki/OCF_Resource_Agents>`_ and
 the *Resource Agents* chapter of *Pacemaker Administration*.
 
 .. index::
    single: resource; LSB
    single: LSB; resources
    single: Linux Standard Base; resources
 
 Linux Standard Base
 ___________________
 
 *LSB* resource agents are more commonly known as *init scripts*. If a full path
 is not given, they are assumed to be located in ``/etc/init.d``.
 
 Commonly, they are provided by the OS distribution. In order to be used
 with a Pacemaker cluster, they must conform to the LSB specification [#]_.
 
 .. warning::
 
    Many distributions or particular software packages claim LSB compliance
    but ship with broken init scripts.  For details on how to check whether
    your init script is LSB-compatible, see the `Resource Agents` chapter of
    `Pacemaker Administration`. Common problematic violations of the LSB
    standard include:
 
    * Not implementing the ``status`` operation at all
    * Not observing the correct exit status codes for
      ``start``/``stop``/``status`` actions
    * Starting a started resource returns an error
    * Stopping a stopped resource returns an error
 
 .. important::
 
    Remember to make sure the computer is `not` configured to start any
    services at boot time -- that should be controlled by the cluster.
 
 .. _s-resource-supported-systemd:
 
 .. index::
    single: Resource; Systemd
    single: Systemd; resources
 
 Systemd
 _______
 
 Most Linux distributions have replaced the old
 `SysV <http://en.wikipedia.org/wiki/Init#SysV-style>`_ style of
 initialization daemons and scripts with
 `Systemd <http://www.freedesktop.org/wiki/Software/systemd>`_.
 
 Pacemaker is able to manage these services `if they are present`.
 
 Instead of init scripts, systemd has `unit files`.  Generally, the
 services (unit files) are provided by the OS distribution, but there
 are online guides for converting from init scripts [#]_.
 
 .. important::
 
    Remember to make sure the computer is `not` configured to start any
    services at boot time -- that should be controlled by the cluster.
 
 .. index::
    single: Resource; Upstart
    single: Upstart; resources
 
 Upstart
 _______
 
 Some distributions replaced the old
 `SysV <http://en.wikipedia.org/wiki/Init#SysV-style>`_ style of
 initialization daemons (and scripts) with
 `Upstart <http://upstart.ubuntu.com/>`_.
 
 Pacemaker is able to manage these services `if they are present`.
 
 Instead of init scripts, Upstart has `jobs`.  Generally, the
 services (jobs) are provided by the OS distribution.
 
 .. important::
 
    Remember to make sure the computer is `not` configured to start any
    services at boot time -- that should be controlled by the cluster.
 
 .. warning::
 
    Upstart support is deprecated in Pacemaker. Upstart is no longer an actively
    maintained project, and test platforms for it are no longer readily usable.
    Support will likely be dropped entirely at the next major release of
    Pacemaker.
 
 
 .. index::
    single: Resource; System Services
    single: System Service; resources
 
 System Services
 _______________
 
 Since there are various types of system services (``systemd``,
 ``upstart``, and ``lsb``), Pacemaker supports a special ``service`` alias which
 intelligently figures out which one applies to a given cluster node.
 
 This is particularly useful when the cluster contains a mix of
 ``systemd``, ``upstart``, and ``lsb``.
 
 In order, Pacemaker will try to find the named service as:
 
 * an LSB init script
 
 * a Systemd unit file
 
 * an Upstart job
 
 .. index::
    single: Resource; STONITH
    single: STONITH; resources
 
 STONITH
 _______
 
 The STONITH class is used exclusively for fencing-related resources.  This is
 discussed later in :ref:`fencing`.
 
 .. index::
    single: Resource; Nagios Plugins
    single: Nagios Plugins; resources
 
 Nagios Plugins
 ______________
 
-Nagios Plugins [#]_ allow us to monitor services on remote hosts.
+Nagios Plugins [#]_ are a way to monitor services. Pacemaker can use these as
+resources, to react to a change in the service's status.
+
+To use plugins as resources, Pacemaker must have been built with support, and
+OCF-style meta-data for the plugins must be installed on nodes that can run
+them. Meta-data for several common plugins is provided by the
+`nagios-agents-metadata <https://github.com/ClusterLabs/nagios-agents-metadata>`_
+project.
+
+The supported parameters for such a resource are same as the long options of
+the plugin.
+
+Start and monitor actions for plugin resources are implemented as invoking the
+plugin. A plugin result of "OK" (0) is treated as success, a result of "WARN"
+(1) is treated as a successful but degraded service, and any other result is
+considered a failure.
+
+A plugin resource is not going to change its status after recovery by
+restarting the plugin, so using them alone does not make sense with ``on-fail``
+set (or left to default) to ``restart``. Another value could make sense, for
+example, if you want to fence or standby nodes that cannot reach some external
+service.
+
+A more common use case for plugin resources is to configure them with a
+``container`` meta-attribute set to the name of another resource that actually
+makes the service available, such as a virtual machine or container.
+
+With ``container`` set, the plugin resource will automatically be colocated
+with the containing resource and ordered after it, and the containing resource
+will be considered failed if the plugin resource fails. This allows monitoring
+of a service inside a virtual machine or container, with recovery of the
+virtual machine or container if the service fails.
+
+Configuring a virtual machine as a guest node, or a container as a
+:ref:`bundle <s-resource-bundle>`, is the preferred way of monitoring a service
+inside, but plugin resources can be useful when it is not practical to modify
+the virtual machine or container image for this purpose.
 
-Pacemaker is able to do remote monitoring with the plugins `if they are
-present`.
-
-A common use case is to configure them as resources belonging to a resource
-container (usually a virtual machine), and the container will be restarted
-if any of them has failed. Another use is to configure them as ordinary
-resources to be used for monitoring hosts or services via the network.
-
-The supported parameters are same as the long options of the plugin.
 
 .. _primitive-resource:
 
 Resource Properties
 ###################
 
 These values tell the cluster which resource agent to use for the resource,
 where to find that resource agent and what standards it conforms to.
 
 .. table:: **Properties of a Primitive Resource**
 
    +----------+------------------------------------------------------------------+
    | Field    | Description                                                      |
    +==========+==================================================================+
    | id       | .. index::                                                       |
    |          |    single: id; resource                                          |
    |          |    single: resource; property, id                                |
    |          |                                                                  |
    |          | Your name for the resource                                       |
    +----------+------------------------------------------------------------------+
    | class    | .. index::                                                       |
    |          |    single: class; resource                                       |
    |          |    single: resource; property, class                             |
    |          |                                                                  |
    |          | The standard the resource agent conforms to. Allowed values:     |
    |          | ``lsb``, ``nagios``, ``ocf``, ``service``, ``stonith``,          |
    |          | ``systemd``, ``upstart``                                         |
    +----------+------------------------------------------------------------------+
    | type     | .. index::                                                       |
    |          |    single: type; resource                                        |
    |          |    single: resource; property, type                              |
    |          |                                                                  |
    |          | The name of the Resource Agent you wish to use. E.g.             |
    |          | ``IPaddr`` or ``Filesystem``                                     |
    +----------+------------------------------------------------------------------+
    | provider | .. index::                                                       |
    |          |    single: provider; resource                                    |
    |          |    single: resource; property, provider                          |
    |          |                                                                  |
    |          | The OCF spec allows multiple vendors to supply the same resource |
    |          | agent. To use the OCF resource agents supplied by the Heartbeat  |
    |          | project, you would specify ``heartbeat`` here.                   |
    +----------+------------------------------------------------------------------+
 
 The XML definition of a resource can be queried with the **crm_resource** tool.
 For example:
 
 .. code-block:: none
 
    # crm_resource --resource Email --query-xml
 
 might produce:
 
 .. topic:: A system resource definition
 
    .. code-block:: xml
 
       <primitive id="Email" class="service" type="exim"/>
 
 .. note::
 
    One of the main drawbacks to system services (LSB, systemd or
    Upstart) resources is that they do not allow any parameters!
 
 .. topic:: An OCF resource definition
 
    .. code-block:: xml
 
       <primitive id="Public-IP" class="ocf" type="IPaddr" provider="heartbeat">
          <instance_attributes id="Public-IP-params">
             <nvpair id="Public-IP-ip" name="ip" value="192.0.2.2"/>
          </instance_attributes>
       </primitive>
 
 .. _resource_options:
 
 Resource Options
 ################
 
 Resources have two types of options: *meta-attributes* and *instance attributes*.
 Meta-attributes apply to any type of resource, while instance attributes
 are specific to each resource agent.
 
 Resource Meta-Attributes
 ________________________
 
 Meta-attributes are used by the cluster to decide how a resource should
 behave and can be easily set using the ``--meta`` option of the
 **crm_resource** command.
 
 .. table:: **Meta-attributes of a Primitive Resource**
 
    +----------------------------+----------------------------------+------------------------------------------------------+
    | Field                      | Default                          | Description                                          |
    +============================+==================================+======================================================+
    | priority                   | 0                                | .. index::                                           |
    |                            |                                  |    single: priority; resource option                 |
    |                            |                                  |    single: resource; option, priority                |
    |                            |                                  |                                                      |
    |                            |                                  | If not all resources can be active, the cluster      |
    |                            |                                  | will stop lower priority resources in order to       |
    |                            |                                  | keep higher priority ones active.                    |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | critical                   | true                             | .. index::                                           |
    |                            |                                  |    single: critical; resource option                 |
    |                            |                                  |    single: resource; option, critical                |
    |                            |                                  |                                                      |
    |                            |                                  | Use this value as the default for ``influence`` in   |
    |                            |                                  | all :ref:`colocation constraints                     |
    |                            |                                  | <s-resource-colocation>` involving this resource,    |
    |                            |                                  | as well as the implicit colocation constraints       |
    |                            |                                  | created if this resource is in a :ref:`group         |
    |                            |                                  | <group-resources>`. For details, see                 |
    |                            |                                  | :ref:`s-coloc-influence`. *(since 2.1.0)*            |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | target-role                | Started                          | .. index::                                           |
    |                            |                                  |    single: target-role; resource option              |
    |                            |                                  |    single: resource; option, target-role             |
    |                            |                                  |                                                      |
    |                            |                                  | What state should the cluster attempt to keep this   |
    |                            |                                  | resource in? Allowed values:                         |
    |                            |                                  |                                                      |
    |                            |                                  | * ``Stopped:`` Force the resource to be stopped      |
    |                            |                                  | * ``Started:`` Allow the resource to be started      |
    |                            |                                  |   (and in the case of :ref:`promotable clone         |
    |                            |                                  |   resources <s-resource-promotable>`, promoted       |
    |                            |                                  |   if appropriate)                                    |
    |                            |                                  | * ``Unpromoted:`` Allow the resource to be started,  |
    |                            |                                  |   but only in the unpromoted role if the resource is |
    |                            |                                  |   :ref:`promotable <s-resource-promotable>`          |
    |                            |                                  | * ``Promoted:`` Equivalent to ``Started``            |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | is-managed                 | TRUE                             | .. index::                                           |
    |                            |                                  |    single: is-managed; resource option               |
    |                            |                                  |    single: resource; option, is-managed              |
    |                            |                                  |                                                      |
    |                            |                                  | Is the cluster allowed to start and stop             |
    |                            |                                  | the resource?  Allowed values: ``true``, ``false``   |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | maintenance                | FALSE                            | .. index::                                           |
    |                            |                                  |    single: maintenance; resource option              |
    |                            |                                  |    single: resource; option, maintenance             |
    |                            |                                  |                                                      |
    |                            |                                  | Similar to the ``maintenance-mode``                  |
    |                            |                                  | :ref:`cluster option <cluster_options>`, but for     |
    |                            |                                  | a single resource. If true, the resource will not    |
    |                            |                                  | be started, stopped, or monitored on any node. This  |
    |                            |                                  | differs from ``is-managed`` in that monitors will    |
    |                            |                                  | not be run. Allowed values: ``true``, ``false``      |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | resource-stickiness        | 1 for individual clone           | .. _resource-stickiness:                             |
    |                            | instances, 0 for all             |                                                      |
    |                            | other resources                  | .. index::                                           |
    |                            |                                  |    single: resource-stickiness; resource option      |
    |                            |                                  |    single: resource; option, resource-stickiness     |
    |                            |                                  |                                                      |
    |                            |                                  | A score that will be added to the current node when  |
    |                            |                                  | a resource is already active. This allows running    |
    |                            |                                  | resources to stay where they are, even if they       |
    |                            |                                  | would be placed elsewhere if they were being         |
    |                            |                                  | started from a stopped state.                        |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | requires                   | ``quorum`` for resources         | .. _requires:                                        |
    |                            | with a ``class`` of ``stonith``, |                                                      |
    |                            | otherwise ``unfencing`` if       | .. index::                                           |
    |                            | unfencing is active in the       |    single: requires; resource option                 |
    |                            | cluster, otherwise ``fencing``   |    single: resource; option, requires                |
    |                            | if ``stonith-enabled`` is true,  |                                                      |
    |                            | otherwise ``quorum``             | Conditions under which the resource can be           |
    |                            |                                  | started. Allowed values:                             |
    |                            |                                  |                                                      |
    |                            |                                  | * ``nothing:`` can always be started                 |
    |                            |                                  | * ``quorum:`` The cluster can only start this        |
    |                            |                                  |   resource if a majority of the configured nodes     |
    |                            |                                  |   are active                                         |
    |                            |                                  | * ``fencing:`` The cluster can only start this       |
    |                            |                                  |   resource if a majority of the configured nodes     |
    |                            |                                  |   are active *and* any failed or unknown nodes       |
    |                            |                                  |   have been :ref:`fenced <fencing>`                  |
    |                            |                                  | * ``unfencing:`` The cluster can only start this     |
    |                            |                                  |   resource if a majority of the configured nodes     |
    |                            |                                  |   are active *and* any failed or unknown nodes have  |
    |                            |                                  |   been fenced *and* only on nodes that have been     |
    |                            |                                  |   :ref:`unfenced <unfencing>`                        |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | migration-threshold        | INFINITY                         | .. index::                                           |
    |                            |                                  |    single: migration-threshold; resource option      |
    |                            |                                  |    single: resource; option, migration-threshold     |
    |                            |                                  |                                                      |
    |                            |                                  | How many failures may occur for this resource on     |
    |                            |                                  | a node, before this node is marked ineligible to     |
    |                            |                                  | host this resource. A value of 0 indicates that this |
    |                            |                                  | feature is disabled (the node will never be marked   |
    |                            |                                  | ineligible); by constrast, the cluster treats        |
    |                            |                                  | INFINITY (the default) as a very large but finite    |
    |                            |                                  | number. This option has an effect only if the        |
    |                            |                                  | failed operation specifies ``on-fail`` as            |
    |                            |                                  | ``restart`` (the default), and additionally for      |
    |                            |                                  | failed ``start`` operations, if the cluster          |
    |                            |                                  | property ``start-failure-is-fatal`` is ``false``.    |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | failure-timeout            | 0                                | .. index::                                           |
    |                            |                                  |    single: failure-timeout; resource option          |
    |                            |                                  |    single: resource; option, failure-timeout         |
    |                            |                                  |                                                      |
    |                            |                                  | How many seconds to wait before acting as if the     |
    |                            |                                  | failure had not occurred, and potentially allowing   |
    |                            |                                  | the resource back to the node on which it failed.    |
    |                            |                                  | A value of 0 indicates that this feature is          |
    |                            |                                  | disabled.                                            |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | multiple-active            | stop_start                       | .. index::                                           |
    |                            |                                  |    single: multiple-active; resource option          |
    |                            |                                  |    single: resource; option, multiple-active         |
    |                            |                                  |                                                      |
    |                            |                                  | What should the cluster do if it ever finds the      |
    |                            |                                  | resource active on more than one node? Allowed       |
    |                            |                                  | values:                                              |
    |                            |                                  |                                                      |
    |                            |                                  | * ``block``: mark the resource as unmanaged          |
    |                            |                                  | * ``stop_only``: stop all active instances and       |
    |                            |                                  |   leave them that way                                |
    |                            |                                  | * ``stop_start``: stop all active instances and      |
    |                            |                                  |   start the resource in one location only            |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | allow-migrate              | TRUE for ocf:pacemaker:remote    | Whether the cluster should try to "live migrate"     |
    |                            | resources, FALSE otherwise       | this resource when it needs to be moved (see         |
    |                            |                                  | :ref:`live-migration`)                               |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | container-attribute-target |                                  | Specific to bundle resources; see                    |
    |                            |                                  | :ref:`s-bundle-attributes`                           |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | remote-node                |                                  | The name of the Pacemaker Remote guest node this     |
    |                            |                                  | resource is associated with, if any. If              |
    |                            |                                  | specified, this both enables the resource as a       |
    |                            |                                  | guest node and defines the unique name used to       |
    |                            |                                  | identify the guest node. The guest must be           |
    |                            |                                  | configured to run the Pacemaker Remote daemon        |
    |                            |                                  | when it is started. **WARNING:** This value          |
    |                            |                                  | cannot overlap with any resource or node IDs.        |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | remote-port                | 3121                             | If ``remote-node`` is specified, the port on the     |
    |                            |                                  | guest used for its Pacemaker Remote connection.      |
    |                            |                                  | The Pacemaker Remote daemon on the guest must        |
    |                            |                                  | be configured to listen on this port.                |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | remote-addr                | value of ``remote-node``         | If ``remote-node`` is specified, the IP              |
    |                            |                                  | address or hostname used to connect to the           |
    |                            |                                  | guest via Pacemaker Remote. The Pacemaker Remote     |
    |                            |                                  | daemon on the guest must be configured to accept     |
    |                            |                                  | connections on this address.                         |
    +----------------------------+----------------------------------+------------------------------------------------------+
    | remote-connect-timeout     | 60s                              | If ``remote-node`` is specified, how long before     |
    |                            |                                  | a pending guest connection will time out.            |
    +----------------------------+----------------------------------+------------------------------------------------------+
 
 As an example of setting resource options, if you performed the following
 commands on an LSB Email resource:
 
 .. code-block:: none
 
    # crm_resource --meta --resource Email --set-parameter priority --parameter-value 100
    # crm_resource -m -r Email -p multiple-active -v block
 
 the resulting resource definition might be:
 
 .. topic:: An LSB resource with cluster options
 
    .. code-block:: xml
 
       <primitive id="Email" class="lsb" type="exim">
         <meta_attributes id="Email-meta_attributes">
           <nvpair id="Email-meta_attributes-priority" name="priority" value="100"/>
           <nvpair id="Email-meta_attributes-multiple-active" name="multiple-active" value="block"/>
         </meta_attributes>
       </primitive>
 
 In addition to the cluster-defined meta-attributes described above, you may
 also configure arbitrary meta-attributes of your own choosing. Most commonly,
 this would be done for use in :ref:`rules <rules>`. For example, an IT department
 might define a custom meta-attribute to indicate which company department each
 resource is intended for. To reduce the chance of name collisions with
 cluster-defined meta-attributes added in the future, it is recommended to use
 a unique, organization-specific prefix for such attributes.
 
 .. _s-resource-defaults:
 
 Setting Global Defaults for Resource Meta-Attributes
 ____________________________________________________
 
 To set a default value for a resource option, add it to the
 ``rsc_defaults`` section with ``crm_attribute``. For example,
 
 .. code-block:: none
 
    # crm_attribute --type rsc_defaults --name is-managed --update false
 
 would prevent the cluster from starting or stopping any of the
 resources in the configuration (unless of course the individual
 resources were specifically enabled by having their ``is-managed`` set to
 ``true``).
 
 Resource Instance Attributes
 ____________________________
 
 The resource agents of some resource classes (lsb, systemd and upstart *not* among them)
 can be given parameters which determine how they behave and which instance
 of a service they control.
 
 If your resource agent supports parameters, you can add them with the
 ``crm_resource`` command. For example,
 
 .. code-block:: none
 
    # crm_resource --resource Public-IP --set-parameter ip --parameter-value 192.0.2.2
 
 would create an entry in the resource like this:
 
 .. topic:: An example OCF resource with instance attributes
 
    .. code-block:: xml
 
       <primitive id="Public-IP" class="ocf" type="IPaddr" provider="heartbeat">
          <instance_attributes id="params-public-ip">
             <nvpair id="public-ip-addr" name="ip" value="192.0.2.2"/>
          </instance_attributes>
       </primitive>
 
 For an OCF resource, the result would be an environment variable
 called ``OCF_RESKEY_ip`` with a value of ``192.0.2.2``.
 
 The list of instance attributes supported by an OCF resource agent can be
 found by calling the resource agent with the ``meta-data`` command.
 The output contains an XML description of all the supported
 attributes, their purpose and default values.
 
 .. topic:: Displaying the metadata for the Dummy resource agent template
 
    .. code-block:: none
 
       # export OCF_ROOT=/usr/lib/ocf
       # $OCF_ROOT/resource.d/pacemaker/Dummy meta-data
 
    .. code-block:: xml
 
       <?xml version="1.0"?>
       <!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
       <resource-agent name="Dummy" version="2.0">
       <version>1.1</version>
 
       <longdesc lang="en">
       This is a dummy OCF resource agent. It does absolutely nothing except keep track
       of whether it is running or not, and can be configured so that actions fail or
       take a long time. Its purpose is primarily for testing, and to serve as a
       template for resource agent writers.
       </longdesc>
       <shortdesc lang="en">Example stateless resource agent</shortdesc>
 
       <parameters>
       <parameter name="state" unique-group="state">
       <longdesc lang="en">
       Location to store the resource state in.
       </longdesc>
       <shortdesc lang="en">State file</shortdesc>
       <content type="string" default="/var/run/Dummy-RESOURCE_ID.state" />
       </parameter>
 
       <parameter name="passwd" reloadable="1">
       <longdesc lang="en">
       Fake password field
       </longdesc>
       <shortdesc lang="en">Password</shortdesc>
       <content type="string" default="" />
       </parameter>
 
       <parameter name="fake" reloadable="1">
       <longdesc lang="en">
       Fake attribute that can be changed to cause a reload
       </longdesc>
       <shortdesc lang="en">Fake attribute that can be changed to cause a reload</shortdesc>
       <content type="string" default="dummy" />
       </parameter>
 
       <parameter name="op_sleep" reloadable="1">
       <longdesc lang="en">
       Number of seconds to sleep during operations.  This can be used to test how
       the cluster reacts to operation timeouts.
       </longdesc>
       <shortdesc lang="en">Operation sleep duration in seconds.</shortdesc>
       <content type="string" default="0" />
       </parameter>
 
       <parameter name="fail_start_on" reloadable="1">
       <longdesc lang="en">
       Start, migrate_from, and reload-agent actions will return failure if running on
       the host specified here, but the resource will run successfully anyway (future
       monitor calls will find it running). This can be used to test on-fail=ignore.
       </longdesc>
       <shortdesc lang="en">Report bogus start failure on specified host</shortdesc>
       <content type="string" default="" />
       </parameter>
       <parameter name="envfile" reloadable="1">
       <longdesc lang="en">
       If this is set, the environment will be dumped to this file for every call.
       </longdesc>
       <shortdesc lang="en">Environment dump file</shortdesc>
       <content type="string" default="" />
       </parameter>
 
       </parameters>
 
       <actions>
       <action name="start"        timeout="20s" />
       <action name="stop"         timeout="20s" />
       <action name="monitor"      timeout="20s" interval="10s" depth="0"/>
       <action name="reload"       timeout="20s" />
       <action name="reload-agent" timeout="20s" />
       <action name="migrate_to"   timeout="20s" />
       <action name="migrate_from" timeout="20s" />
       <action name="validate-all" timeout="20s" />
       <action name="meta-data"    timeout="5s" />
       </actions>
       </resource-agent>
 
 .. index::
    single: resource; action
    single: resource; operation
 
 .. _operation:
 
 Resource Operations
 ###################
 
 *Operations* are actions the cluster can perform on a resource by calling the
 resource agent. Resource agents must support certain common operations such as
 start, stop, and monitor, and may implement any others.
 
 Operations may be explicitly configured for two purposes: to override defaults
 for options (such as timeout) that the cluster will use whenever it initiates
 the operation, and to run an operation on a recurring basis (for example, to
 monitor the resource for failure).
 
 .. topic:: An OCF resource with a non-default start timeout
 
    .. code-block:: xml
 
       <primitive id="Public-IP" class="ocf" type="IPaddr" provider="heartbeat">
         <operations>
            <op id="Public-IP-start" name="start" timeout="60s"/>
         </operations>
         <instance_attributes id="params-public-ip">
            <nvpair id="public-ip-addr" name="ip" value="192.0.2.2"/>
         </instance_attributes>
       </primitive>
 
 Pacemaker identifies operations by a combination of name and interval, so this
 combination must be unique for each resource. That is, you should not configure
 two operations for the same resource with the same name and interval.
 
 .. _operation_properties:
 
 Operation Properties
 ____________________
 
 Operation properties may be specified directly in the ``op`` element as
 XML attributes, or in a separate ``meta_attributes`` block as ``nvpair`` elements.
 XML attributes take precedence over ``nvpair`` elements if both are specified.
 
 .. table:: **Properties of an Operation**
 
    +----------------+-----------------------------------+-----------------------------------------------------+
    | Field          | Default                           | Description                                         |
    +================+===================================+=====================================================+
    | id             |                                   | .. index::                                          |
    |                |                                   |    single: id; action property                      |
    |                |                                   |    single: action; property, id                     |
    |                |                                   |                                                     |
    |                |                                   | A unique name for the operation.                    |
    +----------------+-----------------------------------+-----------------------------------------------------+
    | name           |                                   | .. index::                                          |
    |                |                                   |    single: name; action property                    |
    |                |                                   |    single: action; property, name                   |
    |                |                                   |                                                     |
    |                |                                   | The action to perform. This can be any action       |
    |                |                                   | supported by the agent; common values include       |
    |                |                                   | ``monitor``, ``start``, and ``stop``.               |
    +----------------+-----------------------------------+-----------------------------------------------------+
    | interval       | 0                                 | .. index::                                          |
    |                |                                   |    single: interval; action property                |
    |                |                                   |    single: action; property, interval               |
    |                |                                   |                                                     |
    |                |                                   | How frequently (in seconds) to perform the          |
    |                |                                   | operation. A value of 0 means "when needed".        |
    |                |                                   | A positive value defines a *recurring action*,      |
    |                |                                   | which is typically used with                        |
    |                |                                   | :ref:`monitor <s-resource-monitoring>`.             |
    +----------------+-----------------------------------+-----------------------------------------------------+
    | timeout        |                                   | .. index::                                          |
    |                |                                   |    single: timeout; action property                 |
    |                |                                   |    single: action; property, timeout                |
    |                |                                   |                                                     |
    |                |                                   | How long to wait before declaring the action        |
    |                |                                   | has failed                                          |
    +----------------+-----------------------------------+-----------------------------------------------------+
    | on-fail        | Varies by action:                 | .. index::                                          |
    |                |                                   |    single: on-fail; action property                 |
    |                | * ``stop``: ``fence`` if          |    single: action; property, on-fail                |
    |                |   ``stonith-enabled`` is true     |                                                     |
    |                |   or ``block`` otherwise          | The action to take if this action ever fails.       |
    |                | * ``demote``: ``on-fail`` of the  | Allowed values:                                     |
    |                |   ``monitor`` action with         |                                                     |
    |                |   ``role`` set to ``Promoted``,   | * ``ignore:`` Pretend the resource did not fail.    |
    |                |   if present, enabled, and        | * ``block:`` Don't perform any further operations   |
    |                |   configured to a value other     |   on the resource.                                  |
    |                |   than ``demote``, or ``restart`` | * ``stop:`` Stop the resource and do not start      |
    |                |   otherwise                       |   it elsewhere.                                     |
    |                | * all other actions: ``restart``  | * ``demote:`` Demote the resource, without a        |
    |                |                                   |   full restart. This is valid only for ``promote``  |
    |                |                                   |   actions, and for ``monitor`` actions with both    |
    |                |                                   |   a nonzero ``interval`` and ``role`` set to        |
    |                |                                   |   ``Promoted``; for any other action, a             |
    |                |                                   |   configuration error will be logged, and the       |
    |                |                                   |   default behavior will be used. *(since 2.0.5)*    |
    |                |                                   | * ``restart:`` Stop the resource and start it       |
    |                |                                   |   again (possibly on a different node).             |
    |                |                                   | * ``fence:`` STONITH the node on which the          |
    |                |                                   |   resource failed.                                  |
    |                |                                   | * ``standby:`` Move *all* resources away from the   |
    |                |                                   |   node on which the resource failed.                |
    +----------------+-----------------------------------+-----------------------------------------------------+
    | enabled        | TRUE                              | .. index::                                          |
    |                |                                   |    single: enabled; action property                 |
    |                |                                   |    single: action; property, enabled                |
    |                |                                   |                                                     |
    |                |                                   | If ``false``, ignore this operation definition.     |
    |                |                                   | This is typically used to pause a particular        |
    |                |                                   | recurring ``monitor`` operation; for instance, it   |
    |                |                                   | can complement the respective resource being        |
    |                |                                   | unmanaged (``is-managed=false``), as this alone     |
    |                |                                   | will :ref:`not block any configured monitoring      |
    |                |                                   | <s-monitoring-unmanaged>`.  Disabling the operation |
    |                |                                   | does not suppress all actions of the given type.    |
    |                |                                   | Allowed values: ``true``, ``false``.                |
    +----------------+-----------------------------------+-----------------------------------------------------+
    | record-pending | TRUE                              | .. index::                                          |
    |                |                                   |    single: record-pending; action property          |
    |                |                                   |    single: action; property, record-pending         |
    |                |                                   |                                                     |
    |                |                                   | If ``true``, the intention to perform the operation |
    |                |                                   | is recorded so that GUIs and CLI tools can indicate |
    |                |                                   | that an operation is in progress.  This is best set |
    |                |                                   | as an *operation default*                           |
    |                |                                   | (see :ref:`s-operation-defaults`).  Allowed values: |
    |                |                                   | ``true``, ``false``.                                |
    +----------------+-----------------------------------+-----------------------------------------------------+
    | role           |                                   | .. index::                                          |
    |                |                                   |    single: role; action property                    |
    |                |                                   |    single: action; property, role                   |
    |                |                                   |                                                     |
    |                |                                   | Run the operation only on node(s) that the cluster  |
    |                |                                   | thinks should be in the specified role. This only   |
    |                |                                   | makes sense for recurring ``monitor`` operations.   |
    |                |                                   | Allowed (case-sensitive) values: ``Stopped``,       |
    |                |                                   | ``Started``, and in the case of :ref:`promotable    |
    |                |                                   | clone resources <s-resource-promotable>`,           |
    |                |                                   | ``Unpromoted`` and ``Promoted``.                    |
    +----------------+-----------------------------------+-----------------------------------------------------+
 
 .. note::
 
    When ``on-fail`` is set to ``demote``, recovery from failure by a successful
    demote causes the cluster to recalculate whether and where a new instance
    should be promoted. The node with the failure is eligible, so if promotion
    scores have not changed, it will be promoted again.
 
    There is no direct equivalent of ``migration-threshold`` for the promoted
    role, but the same effect can be achieved with a location constraint using a
    :ref:`rule <rules>` with a node attribute expression for the resource's fail
    count.
 
    For example, to immediately ban the promoted role from a node with any
    failed promote or promoted instance monitor:
 
    .. code-block:: xml
 
       <rsc_location id="loc1" rsc="my_primitive">
           <rule id="rule1" score="-INFINITY" role="Promoted" boolean-op="or">
             <expression id="expr1" attribute="fail-count-my_primitive#promote_0"
               operation="gte" value="1"/>
             <expression id="expr2" attribute="fail-count-my_primitive#monitor_10000"
               operation="gte" value="1"/>
           </rule>
       </rsc_location>
 
    This example assumes that there is a promotable clone of the ``my_primitive``
    resource (note that the primitive name, not the clone name, is used in the
    rule), and that there is a recurring 10-second-interval monitor configured for
    the promoted role (fail count attributes specify the interval in
    milliseconds).
 
 .. _s-resource-monitoring:
 
 Monitoring Resources for Failure
 ________________________________
 
 When Pacemaker first starts a resource, it runs one-time ``monitor`` operations
 (referred to as *probes*) to ensure the resource is running where it's
 supposed to be, and not running where it's not supposed to be. (This behavior
 can be affected by the ``resource-discovery`` location constraint property.)
 
 Other than those initial probes, Pacemaker will *not* (by default) check that
 the resource continues to stay healthy [#]_.  You must configure ``monitor``
 operations explicitly to perform these checks.
 
 .. topic:: An OCF resource with a recurring health check
 
    .. code-block:: xml
 
       <primitive id="Public-IP" class="ocf" type="IPaddr" provider="heartbeat">
         <operations>
            <op id="Public-IP-start" name="start" timeout="60s"/>
            <op id="Public-IP-monitor" name="monitor" interval="60s"/>
         </operations>
         <instance_attributes id="params-public-ip">
            <nvpair id="public-ip-addr" name="ip" value="192.0.2.2"/>
         </instance_attributes>
       </primitive>
 
 By default, a ``monitor`` operation will ensure that the resource is running
 where it is supposed to. The ``target-role`` property can be used for further
 checking.
 
 For example, if a resource has one ``monitor`` operation with
 ``interval=10 role=Started`` and a second ``monitor`` operation with
 ``interval=11 role=Stopped``, the cluster will run the first monitor on any nodes
 it thinks *should* be running the resource, and the second monitor on any nodes
 that it thinks *should not* be running the resource (for the truly paranoid,
 who want to know when an administrator manually starts a service by mistake).
 
 .. note::
 
    Currently, monitors with ``role=Stopped`` are not implemented for
    :ref:`clone <s-resource-clone>` resources.
 
 .. _s-monitoring-unmanaged:
 
 Monitoring Resources When Administration is Disabled
 ____________________________________________________
 
 Recurring ``monitor`` operations behave differently under various administrative
 settings:
 
 * When a resource is unmanaged (by setting ``is-managed=false``): No monitors
   will be stopped.
 
   If the unmanaged resource is stopped on a node where the cluster thinks it
   should be running, the cluster will detect and report that it is not, but it
   will not consider the monitor failed, and will not try to start the resource
   until it is managed again.
 
   Starting the unmanaged resource on a different node is strongly discouraged
   and will at least cause the cluster to consider the resource failed, and
   may require the resource's ``target-role`` to be set to ``Stopped`` then
   ``Started`` to be recovered.
 
 * When a node is put into standby: All resources will be moved away from the
   node, and all ``monitor`` operations will be stopped on the node, except those
   specifying ``role`` as ``Stopped`` (which will be newly initiated if
   appropriate).
 
 * When the cluster is put into maintenance mode: All resources will be marked
   as unmanaged. All monitor operations will be stopped, except those
   specifying ``role`` as ``Stopped`` (which will be newly initiated if
   appropriate). As with single unmanaged resources, starting
   a resource on a node other than where the cluster expects it to be will
   cause problems.
 
 .. _s-operation-defaults:
 
 Setting Global Defaults for Operations
 ______________________________________
 
 You can change the global default values for operation properties
 in a given cluster. These are defined in an ``op_defaults`` section 
 of the CIB's ``configuration`` section, and can be set with
 ``crm_attribute``.  For example,
 
 .. code-block:: none
 
    # crm_attribute --type op_defaults --name timeout --update 20s
 
 would default each operation's ``timeout`` to 20 seconds.  If an
 operation's definition also includes a value for ``timeout``, then that
 value would be used for that operation instead.
 
 When Implicit Operations Take a Long Time
 _________________________________________
 
 The cluster will always perform a number of implicit operations: ``start``,
 ``stop`` and a non-recurring ``monitor`` operation used at startup to check
 whether the resource is already active.  If one of these is taking too long,
 then you can create an entry for them and specify a longer timeout.
 
 .. topic:: An OCF resource with custom timeouts for its implicit actions
 
    .. code-block:: xml
 
       <primitive id="Public-IP" class="ocf" type="IPaddr" provider="heartbeat">
         <operations>
            <op id="public-ip-startup" name="monitor" interval="0" timeout="90s"/>
            <op id="public-ip-start" name="start" interval="0" timeout="180s"/>
            <op id="public-ip-stop" name="stop" interval="0" timeout="15min"/>
         </operations>
         <instance_attributes id="params-public-ip">
            <nvpair id="public-ip-addr" name="ip" value="192.0.2.2"/>
         </instance_attributes>
       </primitive>
 
 Multiple Monitor Operations
 ___________________________
 
 Provided no two operations (for a single resource) have the same name
 and interval, you can have as many ``monitor`` operations as you like.
 In this way, you can do a superficial health check every minute and
 progressively more intense ones at higher intervals.
 
 To tell the resource agent what kind of check to perform, you need to
 provide each monitor with a different value for a common parameter.
 The OCF standard creates a special parameter called ``OCF_CHECK_LEVEL``
 for this purpose and dictates that it is "made available to the
 resource agent without the normal ``OCF_RESKEY`` prefix".
 
 Whatever name you choose, you can specify it by adding an
 ``instance_attributes`` block to the ``op`` tag. It is up to each
 resource agent to look for the parameter and decide how to use it.
 
 .. topic:: An OCF resource with two recurring health checks, performing
            different levels of checks specified via ``OCF_CHECK_LEVEL``.
 
    .. code-block:: xml
 
       <primitive id="Public-IP" class="ocf" type="IPaddr" provider="heartbeat">
          <operations>
             <op id="public-ip-health-60" name="monitor" interval="60">
                <instance_attributes id="params-public-ip-depth-60">
                   <nvpair id="public-ip-depth-60" name="OCF_CHECK_LEVEL" value="10"/>
                </instance_attributes>
             </op>
             <op id="public-ip-health-300" name="monitor" interval="300">
                <instance_attributes id="params-public-ip-depth-300">
                   <nvpair id="public-ip-depth-300" name="OCF_CHECK_LEVEL" value="20"/>
                </instance_attributes>
            </op>
          </operations>
          <instance_attributes id="params-public-ip">
              <nvpair id="public-ip-level" name="ip" value="192.0.2.2"/>
          </instance_attributes>
       </primitive>
 
 Disabling a Monitor Operation
 _____________________________
 
 The easiest way to stop a recurring monitor is to just delete it.
 However, there can be times when you only want to disable it
 temporarily.  In such cases, simply add ``enabled=false`` to the
 operation's definition.
 
 .. topic:: Example of an OCF resource with a disabled health check
 
    .. code-block:: xml
 
       <primitive id="Public-IP" class="ocf" type="IPaddr" provider="heartbeat">
          <operations>
             <op id="public-ip-check" name="monitor" interval="60s" enabled="false"/>
          </operations>
          <instance_attributes id="params-public-ip">
             <nvpair id="public-ip-addr" name="ip" value="192.0.2.2"/>
          </instance_attributes>
       </primitive>
 
 This can be achieved from the command line by executing:
 
 .. code-block:: none
 
    # cibadmin --modify --xml-text '<op id="public-ip-check" enabled="false"/>'
 
 Once you've done whatever you needed to do, you can then re-enable it with
 
 .. code-block:: none
 
    # cibadmin --modify --xml-text '<op id="public-ip-check" enabled="true"/>'
 
 .. [#] See https://github.com/ClusterLabs/OCF-spec/tree/master/ra. The
        Pacemaker implementation has been somewhat extended from the OCF specs.
 
 .. [#] The resource-agents source code includes the **ocf-tester** script,
        which can be useful in this regard.
 
 .. [#] See http://refspecs.linux-foundation.org/LSB_3.0.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html
        for the LSB Spec as it relates to init scripts.
 
 .. [#] For example, http://0pointer.de/blog/projects/systemd-for-admins-3.html
 
 .. [#] The project has two independent forks, hosted at
        https://www.nagios-plugins.org/ and https://www.monitoring-plugins.org/. Output
        from both projects' plugins is similar, so plugins from either project can be
        used with pacemaker.
 
 .. [#] Currently, anyway. Automatic monitoring operations may be added in a future
        version of Pacemaker.
diff --git a/include/crm/services.h b/include/crm/services.h
index ae75af4faa..25bb66d463 100644
--- a/include/crm/services.h
+++ b/include/crm/services.h
@@ -1,401 +1,372 @@
 /*
  * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef __PCMK_SERVICES__
 #  define __PCMK_SERVICES__
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
  * \file
  * \brief Services API
  * \ingroup core
  */
 
 #  include <glib.h>
 #  include <stdio.h>
 #  include <stdint.h>
 #  include <string.h>
 #  include <stdbool.h>
 #  include <sys/types.h>
 
 #  include <crm_config.h>       // OCF_ROOT_DIR
 #  include "common/results.h"
 
-#  ifndef LSB_ROOT_DIR
-#    define LSB_ROOT_DIR "/etc/init.d"
-#  endif
-
 /* TODO: Autodetect these two ?*/
 #  ifndef SYSTEMCTL
 #    define SYSTEMCTL "/bin/systemctl"
 #  endif
 
 /* Known resource classes */
 #define PCMK_RESOURCE_CLASS_OCF     "ocf"
 #define PCMK_RESOURCE_CLASS_SERVICE "service"
 #define PCMK_RESOURCE_CLASS_LSB     "lsb"
 #define PCMK_RESOURCE_CLASS_SYSTEMD "systemd"
 #define PCMK_RESOURCE_CLASS_UPSTART "upstart"
 #define PCMK_RESOURCE_CLASS_NAGIOS  "nagios"
 #define PCMK_RESOURCE_CLASS_STONITH "stonith"
 
 /* This is the string passed in the OCF_EXIT_REASON_PREFIX environment variable.
  * The stderr output that occurs after this prefix is encountered is considered
  * the exit reason for a completed operation.
  */
 #define PCMK_OCF_REASON_PREFIX "ocf-exit-reason:"
 
 // Agent version to use if agent doesn't specify one
 #define PCMK_DEFAULT_AGENT_VERSION "0.1"
 
 enum lsb_exitcode {
     PCMK_LSB_OK                  = 0,
     PCMK_LSB_UNKNOWN_ERROR       = 1,
     PCMK_LSB_INVALID_PARAM       = 2,
     PCMK_LSB_UNIMPLEMENT_FEATURE = 3,
     PCMK_LSB_INSUFFICIENT_PRIV   = 4,
     PCMK_LSB_NOT_INSTALLED       = 5,
     PCMK_LSB_NOT_CONFIGURED      = 6,
     PCMK_LSB_NOT_RUNNING         = 7,
 };
 
 /* The return codes for the status operation are not the same for other
  * operatios - go figure
  */
 enum lsb_status_exitcode {
     PCMK_LSB_STATUS_OK             = 0,
     PCMK_LSB_STATUS_VAR_PID        = 1,
     PCMK_LSB_STATUS_VAR_LOCK       = 2,
     PCMK_LSB_STATUS_NOT_RUNNING    = 3,
     PCMK_LSB_STATUS_UNKNOWN        = 4,
 
     /* custom codes should be in the 150-199 range reserved for application use */
     PCMK_LSB_STATUS_NOT_INSTALLED      = 150,
     PCMK_LSB_STATUS_INSUFFICIENT_PRIV  = 151,
 };
 
 enum nagios_exitcode {
     NAGIOS_STATE_OK        = 0,
     NAGIOS_STATE_WARNING   = 1,
     NAGIOS_STATE_CRITICAL  = 2,
     NAGIOS_STATE_UNKNOWN   = 3,
-    NAGIOS_STATE_DEPENDENT = 4,
 
+    /* This is a custom Pacemaker value (not a nagios convention), used as an
+     * intermediate value between the services library and the executor, so the
+     * executor can map it to the corresponding OCF code.
+     */
     NAGIOS_INSUFFICIENT_PRIV = 100,
-    NAGIOS_NOT_INSTALLED     = 101,
+
+#if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
+    NAGIOS_STATE_DEPENDENT   = 4,   //! \deprecated Unused
+    NAGIOS_NOT_INSTALLED     = 101, //! \deprecated Unused
+#endif
 };
 
 enum svc_action_flags {
     /* On timeout, only kill pid, do not kill entire pid group */
     SVC_ACTION_LEAVE_GROUP = 0x01,
     SVC_ACTION_NON_BLOCKED = 0x02,
 };
 
 typedef struct svc_action_private_s svc_action_private_t;
 
 /*!
  * \brief Object for executing external actions
  *
  * \note This object should never be instantiated directly, but instead created
  *       using one of the constructor functions (resources_action_create() for
  *       resource agents, services_alert_create() for alert agents, or
  *       services_action_create_generic() for generic executables). Similarly,
  *       do not use sizeof() on this struct.
  *
  * \internal Internally, services__create_resource_action() is preferable to
  *           resources_action_create().
  */
 typedef struct svc_action_s {
     /*! Operation key (<resource>_<action>_<interval>) for resource actions,
      *  XML ID for alert actions, or NULL for generic actions
      */
     char *id;
 
     //! XML ID of resource being executed for resource actions, otherwise NULL
     char *rsc;
 
     //! Name of action being executed for resource actions, otherwise NULL
     char *action;
 
     //! Action interval for recurring resource actions, otherwise 0
     guint interval_ms;
 
     //! Resource standard for resource actions, otherwise NULL
     char *standard;
 
     //! Resource provider for resource actions that require it, otherwise NULL
     char *provider;
 
     //! Resource agent name for resource actions, otherwise NULL
     char *agent;
 
     int timeout;    //!< Action timeout (in milliseconds)
 
     /*! A hash table of name/value pairs to use as parameters for resource and
      *  alert actions, otherwise NULL. These will be used to set environment
      *  variables for non-fencing resource agents and alert agents, and to send
      *  stdin to fence agents.
      */
     GHashTable *params;
 
     int rc;         //!< Exit status of action (set by library upon completion)
 
     //!@{
     //! This field should be treated as internal to Pacemaker
     int pid;        // Process ID of child
     int cancel;     // Whether this is a cancellation of a recurring action
     //!@}
 
     int status;     //!< Execution status (enum pcmk_exec_status set by library)
 
     /*! Action counter (set by library for resource actions, or by caller
      * otherwise)
      */
     int sequence;
 
     //!@{
     //! This field should be treated as internal to Pacemaker
     int expected_rc;    // Unused
     int synchronous;    // Whether execution should be synchronous (blocking)
     //!@}
 
     enum svc_action_flags flags;    //!< Flag group of enum svc_action_flags
     char *stderr_data;              //!< Action stderr (set by library)
     char *stdout_data;              //!< Action stdout (set by library)
     void *cb_data;                  //!< For caller's use (not used by library)
 
     //! This field should be treated as internal to Pacemaker
     svc_action_private_t *opaque;
 } svc_action_t;
 
 /**
  * \brief Get a list of files or directories in a given path
  *
  * \param[in] root       full path to a directory to read
  * \param[in] files      return list of files if TRUE or directories if FALSE
  * \param[in] executable if TRUE and files is TRUE, only return executable files
  *
  * \return a list of what was found.  The list items are char *.
  * \note It is the caller's responsibility to free the result with g_list_free_full(list, free).
  */
     GList *get_directory_list(const char *root, gboolean files, gboolean executable);
 
 /**
  * \brief Get a list of providers
  *
  * \param[in] standard  list providers of this standard (e.g. ocf, lsb, etc.)
  *
  * \return a list of providers as char * list items (or NULL if standard does not support providers)
  * \note The caller is responsible for freeing the result using g_list_free_full(list, free).
  */
     GList *resources_list_providers(const char *standard);
 
 /**
  * \brief Get a list of resource agents
  *
  * \param[in] standard  list agents using this standard (e.g. ocf, lsb, etc.) (or NULL for all)
  * \param[in] provider  list agents from this provider (or NULL for all)
  *
  * \return a list of resource agents.  The list items are char *.
  * \note The caller is responsible for freeing the result using g_list_free_full(list, free).
  */
     GList *resources_list_agents(const char *standard, const char *provider);
 
 /**
  * Get list of available standards
  *
  * \return a list of resource standards. The list items are char *. This list _must_
  *         be destroyed using g_list_free_full(list, free).
  */
     GList *resources_list_standards(void);
 
 /**
  * Does the given standard, provider, and agent describe a resource that can exist?
  *
  * \param[in] standard  Which class of agent does the resource belong to?
  * \param[in] provider  What provides the agent (NULL for most standards)?
  * \param[in] agent     What is the name of the agent?
  *
  * \return A boolean
  */
     gboolean resources_agent_exists(const char *standard, const char *provider, const char *agent);
 
 /**
  * \brief Create a new resource action
  *
  * \param[in] name        Name of resource
  * \param[in] standard    Resource agent standard (ocf, lsb, etc.)
  * \param[in] provider    Resource agent provider
  * \param[in] agent       Resource agent name
  * \param[in] action      action (start, stop, monitor, etc.)
  * \param[in] interval_ms How often to repeat this action (if 0, execute once)
  * \param[in] timeout     Consider action failed if it does not complete in this many milliseconds
  * \param[in] params      Action parameters
  *
  * \return newly allocated action instance
  *
  * \post After the call, 'params' is owned, and later free'd by the svc_action_t result
  * \note The caller is responsible for freeing the return value using
  *       services_action_free().
  */
 svc_action_t *resources_action_create(const char *name, const char *standard,
                                       const char *provider, const char *agent,
                                       const char *action, guint interval_ms,
                                       int timeout /* ms */, GHashTable *params,
                                       enum svc_action_flags flags);
 
 /**
  * Kick a recurring action so it is scheduled immediately for re-execution
  */
 gboolean services_action_kick(const char *name, const char *action,
                               guint interval_ms);
 
     const char *resources_find_service_class(const char *agent);
 
 /**
  * Utilize services API to execute an arbitrary command.
  *
  * This API has useful infrastructure in place to be able to run a command
  * in the background and get notified via a callback when the command finishes.
  *
  * \param[in] exec command to execute
  * \param[in] args arguments to the command, NULL terminated
  *
  * \return a svc_action_t object, used to pass to the execute function
  * (services_action_sync() or services_action_async()) and is
  * provided to the callback.
  */
     svc_action_t *services_action_create_generic(const char *exec, const char *args[]);
 
     void services_action_cleanup(svc_action_t * op);
     void services_action_free(svc_action_t * op);
     int services_action_user(svc_action_t *op, const char *user);
 
     gboolean services_action_sync(svc_action_t * op);
 
 /**
  * \brief Run an action asynchronously
  *
  * \param[in] op                    Action to run
  * \param[in] action_callback       Function to call when the action completes
  * \param[in] action_fork_callback  Function to call after action process forks
  *
  * \return TRUE if execution was successfully initiated, FALSE otherwise (in
  *              which case the callback will not be called)
  */
     gboolean services_action_async_fork_notify(svc_action_t * op,
         void (*action_callback) (svc_action_t *),
         void (*action_fork_callback) (svc_action_t *));
 
     gboolean services_action_async(svc_action_t * op,
                                    void (*action_callback) (svc_action_t *));
 
 gboolean services_action_cancel(const char *name, const char *action,
                                 guint interval_ms);
 
 /* functions for alert agents */
 svc_action_t *services_alert_create(const char *id, const char *exec,
                                    int timeout, GHashTable *params,
                                    int sequence, void *cb_data);
 gboolean services_alert_async(svc_action_t *action,
                               void (*cb)(svc_action_t *op));
 
+enum ocf_exitcode services_result2ocf(const char *standard, const char *action,
+                                      int exit_status);
+
     static inline const char *services_ocf_exitcode_str(enum ocf_exitcode code) {
         switch (code) {
             case PCMK_OCF_OK:
                 return "ok";
             case PCMK_OCF_UNKNOWN_ERROR:
                 return "error";
             case PCMK_OCF_INVALID_PARAM:
                 return "invalid parameter";
             case PCMK_OCF_UNIMPLEMENT_FEATURE:
                 return "unimplemented feature";
             case PCMK_OCF_INSUFFICIENT_PRIV:
                 return "insufficient privileges";
             case PCMK_OCF_NOT_INSTALLED:
                 return "not installed";
             case PCMK_OCF_NOT_CONFIGURED:
                 return "not configured";
             case PCMK_OCF_NOT_RUNNING:
                 return "not running";
             case PCMK_OCF_RUNNING_PROMOTED:
                 return "promoted";
             case PCMK_OCF_FAILED_PROMOTED:
                 return "promoted (failed)";
             case PCMK_OCF_DEGRADED:
                 return "OCF_DEGRADED";
             case PCMK_OCF_DEGRADED_PROMOTED:
                 return "promoted (degraded)";
 
 #if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
             case PCMK_OCF_NOT_SUPPORTED:
                 return "not supported (DEPRECATED STATUS)";
             case PCMK_OCF_CANCELLED:
                 return "cancelled (DEPRECATED STATUS)";
             case PCMK_OCF_OTHER_ERROR:
                 return "other error (DEPRECATED STATUS)";
             case PCMK_OCF_SIGNAL:
                 return "interrupted by signal (DEPRECATED STATUS)";
             case PCMK_OCF_PENDING:
                 return "pending (DEPRECATED STATUS)";
             case PCMK_OCF_TIMEOUT:
                 return "timeout (DEPRECATED STATUS)";
 #endif
             default:
                 return "unknown";
         }
     }
 
-    /**
-     * \brief Get OCF equivalent of LSB exit code
-     *
-     * \param[in] action        LSB action that produced exit code
-     * \param[in] lsb_exitcode  Exit code of LSB action
-     *
-     * \return PCMK_OCF_* constant that corresponds to LSB exit code
-     */
-    static inline enum ocf_exitcode
-    services_get_ocf_exitcode(const char *action, int lsb_exitcode)
-    {
-        /* For non-status actions, LSB and OCF share error code meaning <= 7 */
-        if (action && strcmp(action, "status") && strcmp(action, "monitor")) {
-            if ((lsb_exitcode < 0) || (lsb_exitcode > PCMK_LSB_NOT_RUNNING)) {
-                return PCMK_OCF_UNKNOWN_ERROR;
-            }
-            return (enum ocf_exitcode)lsb_exitcode;
-        }
-
-        /* status has different return codes */
-        switch (lsb_exitcode) {
-            case PCMK_LSB_STATUS_OK:
-                return PCMK_OCF_OK;
-            case PCMK_LSB_STATUS_NOT_INSTALLED:
-                return PCMK_OCF_NOT_INSTALLED;
-            case PCMK_LSB_STATUS_INSUFFICIENT_PRIV:
-                return PCMK_OCF_INSUFFICIENT_PRIV;
-            case PCMK_LSB_STATUS_VAR_PID:
-            case PCMK_LSB_STATUS_VAR_LOCK:
-            case PCMK_LSB_STATUS_NOT_RUNNING:
-                return PCMK_OCF_NOT_RUNNING;
-        }
-        return PCMK_OCF_UNKNOWN_ERROR;
-    }
-
 #if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
 #include <crm/services_compat.h>
 #endif
 
 #  ifdef __cplusplus
 }
 #  endif
 
 #endif                          /* __PCMK_SERVICES__ */
diff --git a/include/crm/services_compat.h b/include/crm/services_compat.h
index 5a9ce032bb..32b1cacebd 100644
--- a/include/crm/services_compat.h
+++ b/include/crm/services_compat.h
@@ -1,62 +1,95 @@
 /*
  * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PCMK__SERVICES_COMPAT__H
 #  define PCMK__SERVICES_COMPAT__H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
  * \file
  * \brief Deprecated services API
  * \ingroup core
  * \deprecated Do not include this header directly. The service APIs in this
  *             header, and the header itself, will be removed in a future
  *             release.
  */
 
 #include <crm/common/results.h>
 
+#  ifndef LSB_ROOT_DIR
+     //! \deprecated Do not use
+#    define LSB_ROOT_DIR "/etc/init.d"
+#  endif
+
 //! \deprecated Use enum pcmk_exec_status instead
 enum op_status {
     PCMK_LRM_OP_UNKNOWN = PCMK_EXEC_UNKNOWN,
     PCMK_LRM_OP_PENDING = PCMK_EXEC_PENDING,
     PCMK_LRM_OP_DONE = PCMK_EXEC_DONE,
     PCMK_LRM_OP_CANCELLED = PCMK_EXEC_CANCELLED,
     PCMK_LRM_OP_TIMEOUT = PCMK_EXEC_TIMEOUT,
     PCMK_LRM_OP_NOTSUPPORTED = PCMK_EXEC_NOT_SUPPORTED,
     PCMK_LRM_OP_ERROR = PCMK_EXEC_ERROR,
     PCMK_LRM_OP_ERROR_HARD = PCMK_EXEC_ERROR_HARD,
     PCMK_LRM_OP_ERROR_FATAL = PCMK_EXEC_ERROR_FATAL,
     PCMK_LRM_OP_NOT_INSTALLED = PCMK_EXEC_NOT_INSTALLED,
     PCMK_LRM_OP_NOT_CONNECTED = PCMK_EXEC_NOT_CONNECTED,
     PCMK_LRM_OP_INVALID = PCMK_EXEC_INVALID,
 };
 
 //! \deprecated Use resources_action_create() instead
 svc_action_t *services_action_create(const char *name, const char *action,
                                      guint interval_ms, int timeout);
 
 //! \deprecated Use resources_list_agents() instead
 GList *services_list(void);
 
 //! \deprecated Use pcmk_exec_status_str() instead
 static inline const char *
 services_lrm_status_str(enum op_status status)
 {
     return pcmk_exec_status_str((enum pcmk_exec_status) status);
 }
 
+//! \deprecated Use services_result2ocf() instead
+static inline enum ocf_exitcode
+services_get_ocf_exitcode(const char *action, int lsb_exitcode)
+{
+    /* For non-status actions, LSB and OCF share error code meaning <= 7 */
+    if (action && strcmp(action, "status") && strcmp(action, "monitor")) {
+        if ((lsb_exitcode < 0) || (lsb_exitcode > PCMK_LSB_NOT_RUNNING)) {
+            return PCMK_OCF_UNKNOWN_ERROR;
+        }
+        return (enum ocf_exitcode)lsb_exitcode;
+    }
+
+    /* status has different return codes */
+    switch (lsb_exitcode) {
+        case PCMK_LSB_STATUS_OK:
+            return PCMK_OCF_OK;
+        case PCMK_LSB_STATUS_NOT_INSTALLED:
+            return PCMK_OCF_NOT_INSTALLED;
+        case PCMK_LSB_STATUS_INSUFFICIENT_PRIV:
+            return PCMK_OCF_INSUFFICIENT_PRIV;
+        case PCMK_LSB_STATUS_VAR_PID:
+        case PCMK_LSB_STATUS_VAR_LOCK:
+        case PCMK_LSB_STATUS_NOT_RUNNING:
+            return PCMK_OCF_NOT_RUNNING;
+    }
+    return PCMK_OCF_UNKNOWN_ERROR;
+}
+
 #ifdef __cplusplus
 }
 #endif
 
 #endif
diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
index 9a2b16bc67..fc3e2cb576 100644
--- a/lib/common/mainloop.c
+++ b/lib/common/mainloop.c
@@ -1,1502 +1,1502 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <stdlib.h>
 #include <string.h>
 #include <signal.h>
 #include <errno.h>
 
 #include <sys/wait.h>
 
 #include <crm/crm.h>
 #include <crm/common/xml.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/ipc_internal.h>
 
 #include <qb/qbarray.h>
 
 struct mainloop_child_s {
     pid_t pid;
     char *desc;
     unsigned timerid;
     gboolean timeout;
     void *privatedata;
 
     enum mainloop_child_flags flags;
 
     /* Called when a process dies */
     void (*callback) (mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode);
 };
 
 struct trigger_s {
     GSource source;
     gboolean running;
     gboolean trigger;
     void *user_data;
     guint id;
 
 };
 
 struct mainloop_timer_s {
         guint id;
         guint period_ms;
         bool repeat;
         char *name;
         GSourceFunc cb;
         void *userdata;
 };
 
 static gboolean
 crm_trigger_prepare(GSource * source, gint * timeout)
 {
     crm_trigger_t *trig = (crm_trigger_t *) source;
 
     /* cluster-glue's FD and IPC related sources make use of
      * g_source_add_poll() but do not set a timeout in their prepare
      * functions
      *
      * This means mainloop's poll() will block until an event for one
      * of these sources occurs - any /other/ type of source, such as
      * this one or g_idle_*, that doesn't use g_source_add_poll() is
      * S-O-L and won't be processed until there is something fd-based
      * happens.
      *
      * Luckily the timeout we can set here affects all sources and
      * puts an upper limit on how long poll() can take.
      *
      * So unconditionally set a small-ish timeout, not too small that
      * we're in constant motion, which will act as an upper bound on
      * how long the signal handling might be delayed for.
      */
     *timeout = 500;             /* Timeout in ms */
 
     return trig->trigger;
 }
 
 static gboolean
 crm_trigger_check(GSource * source)
 {
     crm_trigger_t *trig = (crm_trigger_t *) source;
 
     return trig->trigger;
 }
 
 /*!
  * \internal
  * \brief GSource dispatch function for crm_trigger_t
  *
  * \param[in] source    crm_trigger_t being dispatched
  * \param[in] callback  Callback passed at source creation
  * \param[in] userdata  User data passed at source creation
  *
  * \return G_SOURCE_REMOVE to remove source, G_SOURCE_CONTINUE to keep it
  */
 static gboolean
 crm_trigger_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
 {
     gboolean rc = G_SOURCE_CONTINUE;
     crm_trigger_t *trig = (crm_trigger_t *) source;
 
     if (trig->running) {
         /* Wait until the existing job is complete before starting the next one */
         return G_SOURCE_CONTINUE;
     }
     trig->trigger = FALSE;
 
     if (callback) {
         int callback_rc = callback(trig->user_data);
 
         if (callback_rc < 0) {
             crm_trace("Trigger handler %p not yet complete", trig);
             trig->running = TRUE;
         } else if (callback_rc == 0) {
             rc = G_SOURCE_REMOVE;
         }
     }
     return rc;
 }
 
 static void
 crm_trigger_finalize(GSource * source)
 {
     crm_trace("Trigger %p destroyed", source);
 }
 
 static GSourceFuncs crm_trigger_funcs = {
     crm_trigger_prepare,
     crm_trigger_check,
     crm_trigger_dispatch,
     crm_trigger_finalize,
 };
 
 static crm_trigger_t *
 mainloop_setup_trigger(GSource * source, int priority, int (*dispatch) (gpointer user_data),
                        gpointer userdata)
 {
     crm_trigger_t *trigger = NULL;
 
     trigger = (crm_trigger_t *) source;
 
     trigger->id = 0;
     trigger->trigger = FALSE;
     trigger->user_data = userdata;
 
     if (dispatch) {
         g_source_set_callback(source, dispatch, trigger, NULL);
     }
 
     g_source_set_priority(source, priority);
     g_source_set_can_recurse(source, FALSE);
 
     trigger->id = g_source_attach(source, NULL);
     return trigger;
 }
 
 void
 mainloop_trigger_complete(crm_trigger_t * trig)
 {
     crm_trace("Trigger handler %p complete", trig);
     trig->running = FALSE;
 }
 
 /*!
  * \brief Create a trigger to be used as a mainloop source
  *
  * \param[in] priority  Relative priority of source (lower number is higher priority)
  * \param[in] dispatch  Trigger dispatch function (should return 0 to remove the
  *                      trigger from the mainloop, -1 if the trigger should be
  *                      kept but the job is still running and not complete, and
  *                      1 if the trigger should be kept and the job is complete)
  *
  * \return Newly allocated mainloop source for trigger
  */
 crm_trigger_t *
 mainloop_add_trigger(int priority, int (*dispatch) (gpointer user_data), gpointer userdata)
 {
     GSource *source = NULL;
 
     CRM_ASSERT(sizeof(crm_trigger_t) > sizeof(GSource));
     source = g_source_new(&crm_trigger_funcs, sizeof(crm_trigger_t));
     CRM_ASSERT(source != NULL);
 
     return mainloop_setup_trigger(source, priority, dispatch, userdata);
 }
 
 void
 mainloop_set_trigger(crm_trigger_t * source)
 {
     if(source) {
         source->trigger = TRUE;
     }
 }
 
 gboolean
 mainloop_destroy_trigger(crm_trigger_t * source)
 {
     GSource *gs = NULL;
 
     if(source == NULL) {
         return TRUE;
     }
 
     gs = (GSource *)source;
 
     g_source_destroy(gs); /* Remove from mainloop, ref_count-- */
     g_source_unref(gs); /* The caller no longer carries a reference to source
                          *
                          * At this point the source should be free'd,
                          * unless we're currently processing said
                          * source, in which case mainloop holds an
                          * additional reference and it will be free'd
                          * once our processing completes
                          */
     return TRUE;
 }
 
 // Define a custom glib source for signal handling
 
 // Data structure for custom glib source
 typedef struct signal_s {
     crm_trigger_t trigger;      // trigger that invoked source (must be first)
     void (*handler) (int sig);  // signal handler
     int signal;                 // signal that was received
 } crm_signal_t;
 
 // Table to associate signal handlers with signal numbers
 static crm_signal_t *crm_signals[NSIG];
 
 /*!
  * \internal
  * \brief Dispatch an event from custom glib source for signals
  *
  * Given an signal event, clear the event trigger and call any registered
  * signal handler.
  *
  * \param[in] source    glib source that triggered this dispatch
  * \param[in] callback  (ignored)
  * \param[in] userdata  (ignored)
  */
 static gboolean
 crm_signal_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
 {
     crm_signal_t *sig = (crm_signal_t *) source;
 
     if(sig->signal != SIGCHLD) {
         crm_notice("Caught '%s' signal "CRM_XS" %d (%s handler)",
                    strsignal(sig->signal), sig->signal,
                    (sig->handler? "invoking" : "no"));
     }
 
     sig->trigger.trigger = FALSE;
     if (sig->handler) {
         sig->handler(sig->signal);
     }
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Handle a signal by setting a trigger for signal source
  *
  * \param[in] sig  Signal number that was received
  *
  * \note This is the true signal handler for the mainloop signal source, and
  *       must be async-safe.
  */
 static void
 mainloop_signal_handler(int sig)
 {
     if (sig > 0 && sig < NSIG && crm_signals[sig] != NULL) {
         mainloop_set_trigger((crm_trigger_t *) crm_signals[sig]);
     }
 }
 
 // Functions implementing our custom glib source for signal handling
 static GSourceFuncs crm_signal_funcs = {
     crm_trigger_prepare,
     crm_trigger_check,
     crm_signal_dispatch,
     crm_trigger_finalize,
 };
 
 /*!
  * \internal
  * \brief Set a true signal handler
  *
  * signal()-like interface to sigaction()
  *
  * \param[in] sig       Signal number to register handler for
  * \param[in] dispatch  Signal handler
  *
  * \return The previous value of the signal handler, or SIG_ERR on error
  * \note The dispatch function must be async-safe.
  */
 sighandler_t
 crm_signal_handler(int sig, sighandler_t dispatch)
 {
     sigset_t mask;
     struct sigaction sa;
     struct sigaction old;
 
     if (sigemptyset(&mask) < 0) {
         crm_err("Could not set handler for signal %d: %s",
                 sig, pcmk_strerror(errno));
         return SIG_ERR;
     }
 
     memset(&sa, 0, sizeof(struct sigaction));
     sa.sa_handler = dispatch;
     sa.sa_flags = SA_RESTART;
     sa.sa_mask = mask;
 
     if (sigaction(sig, &sa, &old) < 0) {
         crm_err("Could not set handler for signal %d: %s",
                 sig, pcmk_strerror(errno));
         return SIG_ERR;
     }
     return old.sa_handler;
 }
 
 static void
 mainloop_destroy_signal_entry(int sig)
 {
     crm_signal_t *tmp = crm_signals[sig];
 
     crm_signals[sig] = NULL;
 
     crm_trace("Destroying signal %d", sig);
     mainloop_destroy_trigger((crm_trigger_t *) tmp);
 }
 
 /*!
  * \internal
  * \brief Add a signal handler to a mainloop
  *
  * \param[in] sig       Signal number to handle
  * \param[in] dispatch  Signal handler function
  *
  * \note The true signal handler merely sets a mainloop trigger to call this
  *       dispatch function via the mainloop. Therefore, the dispatch function
  *       does not need to be async-safe.
  */
 gboolean
 mainloop_add_signal(int sig, void (*dispatch) (int sig))
 {
     GSource *source = NULL;
     int priority = G_PRIORITY_HIGH - 1;
 
     if (sig == SIGTERM) {
         /* TERM is higher priority than other signals,
          *   signals are higher priority than other ipc.
          * Yes, minus: smaller is "higher"
          */
         priority--;
     }
 
     if (sig >= NSIG || sig < 0) {
         crm_err("Signal %d is out of range", sig);
         return FALSE;
 
     } else if (crm_signals[sig] != NULL && crm_signals[sig]->handler == dispatch) {
         crm_trace("Signal handler for %d is already installed", sig);
         return TRUE;
 
     } else if (crm_signals[sig] != NULL) {
         crm_err("Different signal handler for %d is already installed", sig);
         return FALSE;
     }
 
     CRM_ASSERT(sizeof(crm_signal_t) > sizeof(GSource));
     source = g_source_new(&crm_signal_funcs, sizeof(crm_signal_t));
 
     crm_signals[sig] = (crm_signal_t *) mainloop_setup_trigger(source, priority, NULL, NULL);
     CRM_ASSERT(crm_signals[sig] != NULL);
 
     crm_signals[sig]->handler = dispatch;
     crm_signals[sig]->signal = sig;
 
     if (crm_signal_handler(sig, mainloop_signal_handler) == SIG_ERR) {
         mainloop_destroy_signal_entry(sig);
         return FALSE;
     }
 #if 0
     /* If we want signals to interrupt mainloop's poll(), instead of waiting for
      * the timeout, then we should call siginterrupt() below
      *
      * For now, just enforce a low timeout
      */
     if (siginterrupt(sig, 1) < 0) {
         crm_perror(LOG_INFO, "Could not enable system call interruptions for signal %d", sig);
     }
 #endif
 
     return TRUE;
 }
 
 gboolean
 mainloop_destroy_signal(int sig)
 {
     if (sig >= NSIG || sig < 0) {
         crm_err("Signal %d is out of range", sig);
         return FALSE;
 
     } else if (crm_signal_handler(sig, NULL) == SIG_ERR) {
         crm_perror(LOG_ERR, "Could not uninstall signal handler for signal %d", sig);
         return FALSE;
 
     } else if (crm_signals[sig] == NULL) {
         return TRUE;
     }
     mainloop_destroy_signal_entry(sig);
     return TRUE;
 }
 
 static qb_array_t *gio_map = NULL;
 
 void
 mainloop_cleanup(void) 
 {
     if (gio_map) {
         qb_array_free(gio_map);
     }
 
     for (int sig = 0; sig < NSIG; ++sig) {
         mainloop_destroy_signal_entry(sig);
     }
 }
 
 /*
  * libqb...
  */
 struct gio_to_qb_poll {
     int32_t is_used;
     guint source;
     int32_t events;
     void *data;
     qb_ipcs_dispatch_fn_t fn;
     enum qb_loop_priority p;
 };
 
 static gboolean
 gio_read_socket(GIOChannel * gio, GIOCondition condition, gpointer data)
 {
     struct gio_to_qb_poll *adaptor = (struct gio_to_qb_poll *)data;
     gint fd = g_io_channel_unix_get_fd(gio);
 
     crm_trace("%p.%d %d", data, fd, condition);
 
     /* if this assert get's hit, then there is a race condition between
      * when we destroy a fd and when mainloop actually gives it up */
     CRM_ASSERT(adaptor->is_used > 0);
 
     return (adaptor->fn(fd, condition, adaptor->data) == 0);
 }
 
 static void
 gio_poll_destroy(gpointer data)
 {
     struct gio_to_qb_poll *adaptor = (struct gio_to_qb_poll *)data;
 
     adaptor->is_used--;
     CRM_ASSERT(adaptor->is_used >= 0);
 
     if (adaptor->is_used == 0) {
         crm_trace("Marking adaptor %p unused", adaptor);
         adaptor->source = 0;
     }
 }
 
 /*!
  * \internal
  * \brief Convert libqb's poll priority into GLib's one
  *
  * \param[in] prio  libqb's poll priority (#QB_LOOP_MED assumed as fallback)
  *
  * \return  best matching GLib's priority
  */
 static gint
 conv_prio_libqb2glib(enum qb_loop_priority prio)
 {
     gint ret = G_PRIORITY_DEFAULT;
     switch (prio) {
         case QB_LOOP_LOW:
             ret = G_PRIORITY_LOW;
             break;
         case QB_LOOP_HIGH:
             ret = G_PRIORITY_HIGH;
             break;
         default:
             crm_trace("Invalid libqb's loop priority %d, assuming QB_LOOP_MED",
                       prio);
             /* fall-through */
         case QB_LOOP_MED:
             break;
     }
     return ret;
 }
 
 /*!
  * \internal
  * \brief Convert libqb's poll priority to rate limiting spec
  *
  * \param[in] prio  libqb's poll priority (#QB_LOOP_MED assumed as fallback)
  *
  * \return  best matching rate limiting spec
  */
 static enum qb_ipcs_rate_limit
 conv_libqb_prio2ratelimit(enum qb_loop_priority prio)
 {
     /* this is an inversion of what libqb's qb_ipcs_request_rate_limit does */
     enum qb_ipcs_rate_limit ret = QB_IPCS_RATE_NORMAL;
     switch (prio) {
         case QB_LOOP_LOW:
             ret = QB_IPCS_RATE_SLOW;
             break;
         case QB_LOOP_HIGH:
             ret = QB_IPCS_RATE_FAST;
             break;
         default:
             crm_trace("Invalid libqb's loop priority %d, assuming QB_LOOP_MED",
                       prio);
             /* fall-through */
         case QB_LOOP_MED:
             break;
     }
     return ret;
 }
 
 static int32_t
 gio_poll_dispatch_update(enum qb_loop_priority p, int32_t fd, int32_t evts,
                          void *data, qb_ipcs_dispatch_fn_t fn, int32_t add)
 {
     struct gio_to_qb_poll *adaptor;
     GIOChannel *channel;
     int32_t res = 0;
 
     res = qb_array_index(gio_map, fd, (void **)&adaptor);
     if (res < 0) {
         crm_err("Array lookup failed for fd=%d: %d", fd, res);
         return res;
     }
 
     crm_trace("Adding fd=%d to mainloop as adaptor %p", fd, adaptor);
 
     if (add && adaptor->source) {
         crm_err("Adaptor for descriptor %d is still in-use", fd);
         return -EEXIST;
     }
     if (!add && !adaptor->is_used) {
         crm_err("Adaptor for descriptor %d is not in-use", fd);
         return -ENOENT;
     }
 
     /* channel is created with ref_count = 1 */
     channel = g_io_channel_unix_new(fd);
     if (!channel) {
         crm_err("No memory left to add fd=%d", fd);
         return -ENOMEM;
     }
 
     if (adaptor->source) {
         g_source_remove(adaptor->source);
         adaptor->source = 0;
     }
 
     /* Because unlike the poll() API, glib doesn't tell us about HUPs by default */
     evts |= (G_IO_HUP | G_IO_NVAL | G_IO_ERR);
 
     adaptor->fn = fn;
     adaptor->events = evts;
     adaptor->data = data;
     adaptor->p = p;
     adaptor->is_used++;
     adaptor->source =
         g_io_add_watch_full(channel, conv_prio_libqb2glib(p), evts,
                             gio_read_socket, adaptor, gio_poll_destroy);
 
     /* Now that mainloop now holds a reference to channel,
      * thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
      *
      * This means that channel will be free'd by:
      * g_main_context_dispatch()
      *  -> g_source_destroy_internal()
      *      -> g_source_callback_unref()
      * shortly after gio_poll_destroy() completes
      */
     g_io_channel_unref(channel);
 
     crm_trace("Added to mainloop with gsource id=%d", adaptor->source);
     if (adaptor->source > 0) {
         return 0;
     }
 
     return -EINVAL;
 }
 
 static int32_t
 gio_poll_dispatch_add(enum qb_loop_priority p, int32_t fd, int32_t evts,
                       void *data, qb_ipcs_dispatch_fn_t fn)
 {
     return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_TRUE);
 }
 
 static int32_t
 gio_poll_dispatch_mod(enum qb_loop_priority p, int32_t fd, int32_t evts,
                       void *data, qb_ipcs_dispatch_fn_t fn)
 {
     return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_FALSE);
 }
 
 static int32_t
 gio_poll_dispatch_del(int32_t fd)
 {
     struct gio_to_qb_poll *adaptor;
 
     crm_trace("Looking for fd=%d", fd);
     if (qb_array_index(gio_map, fd, (void **)&adaptor) == 0) {
         if (adaptor->source) {
             g_source_remove(adaptor->source);
             adaptor->source = 0;
         }
     }
     return 0;
 }
 
 struct qb_ipcs_poll_handlers gio_poll_funcs = {
     .job_add = NULL,
     .dispatch_add = gio_poll_dispatch_add,
     .dispatch_mod = gio_poll_dispatch_mod,
     .dispatch_del = gio_poll_dispatch_del,
 };
 
 static enum qb_ipc_type
 pick_ipc_type(enum qb_ipc_type requested)
 {
     const char *env = getenv("PCMK_ipc_type");
 
     if (env && strcmp("shared-mem", env) == 0) {
         return QB_IPC_SHM;
     } else if (env && strcmp("socket", env) == 0) {
         return QB_IPC_SOCKET;
     } else if (env && strcmp("posix", env) == 0) {
         return QB_IPC_POSIX_MQ;
     } else if (env && strcmp("sysv", env) == 0) {
         return QB_IPC_SYSV_MQ;
     } else if (requested == QB_IPC_NATIVE) {
         /* We prefer shared memory because the server never blocks on
          * send.  If part of a message fits into the socket, libqb
          * needs to block until the remainder can be sent also.
          * Otherwise the client will wait forever for the remaining
          * bytes.
          */
         return QB_IPC_SHM;
     }
     return requested;
 }
 
 qb_ipcs_service_t *
 mainloop_add_ipc_server(const char *name, enum qb_ipc_type type,
                         struct qb_ipcs_service_handlers *callbacks)
 {
     return mainloop_add_ipc_server_with_prio(name, type, callbacks, QB_LOOP_MED);
 }
 
 qb_ipcs_service_t *
 mainloop_add_ipc_server_with_prio(const char *name, enum qb_ipc_type type,
                                   struct qb_ipcs_service_handlers *callbacks,
                                   enum qb_loop_priority prio)
 {
     int rc = 0;
     qb_ipcs_service_t *server = NULL;
 
     if (gio_map == NULL) {
         gio_map = qb_array_create_2(64, sizeof(struct gio_to_qb_poll), 1);
     }
 
     server = qb_ipcs_create(name, 0, pick_ipc_type(type), callbacks);
 
     if (server == NULL) {
         crm_err("Could not create %s IPC server: %s (%d)", name, pcmk_strerror(rc), rc);
         return NULL;
     }
 
     if (prio != QB_LOOP_MED) {
         qb_ipcs_request_rate_limit(server, conv_libqb_prio2ratelimit(prio));
     }
 
     /* All clients should use at least ipc_buffer_max as their buffer size */
     qb_ipcs_enforce_buffer_size(server, crm_ipc_default_buffer_size());
     qb_ipcs_poll_handlers_set(server, &gio_poll_funcs);
 
     rc = qb_ipcs_run(server);
     if (rc < 0) {
         crm_err("Could not start %s IPC server: %s (%d)", name, pcmk_strerror(rc), rc);
         return NULL;
     }
 
     return server;
 }
 
 void
 mainloop_del_ipc_server(qb_ipcs_service_t * server)
 {
     if (server) {
         qb_ipcs_destroy(server);
     }
 }
 
 struct mainloop_io_s {
     char *name;
     void *userdata;
 
     int fd;
     guint source;
     crm_ipc_t *ipc;
     GIOChannel *channel;
 
     int (*dispatch_fn_ipc) (const char *buffer, ssize_t length, gpointer userdata);
     int (*dispatch_fn_io) (gpointer userdata);
     void (*destroy_fn) (gpointer userdata);
 
 };
 
 /*!
  * \internal
  * \brief I/O watch callback function (GIOFunc)
  *
  * \param[in] gio        I/O channel being watched
  * \param[in] condition  I/O condition satisfied
  * \param[in] data       User data passed when source was created
  *
  * \return G_SOURCE_REMOVE to remove source, G_SOURCE_CONTINUE to keep it
  */
 static gboolean
 mainloop_gio_callback(GIOChannel * gio, GIOCondition condition, gpointer data)
 {
     gboolean rc = G_SOURCE_CONTINUE;
     mainloop_io_t *client = data;
 
     CRM_ASSERT(client->fd == g_io_channel_unix_get_fd(gio));
 
     if (condition & G_IO_IN) {
         if (client->ipc) {
             long read_rc = 0L;
             int max = 10;
 
             do {
                 read_rc = crm_ipc_read(client->ipc);
                 if (read_rc <= 0) {
                     crm_trace("Could not read IPC message from %s: %s (%ld)",
                               client->name, pcmk_strerror(read_rc), read_rc);
 
                 } else if (client->dispatch_fn_ipc) {
                     const char *buffer = crm_ipc_buffer(client->ipc);
 
                     crm_trace("New %ld-byte IPC message from %s "
                               "after I/O condition %d",
                               read_rc, client->name, (int) condition);
                     if (client->dispatch_fn_ipc(buffer, read_rc, client->userdata) < 0) {
                         crm_trace("Connection to %s no longer required", client->name);
                         rc = G_SOURCE_REMOVE;
                     }
                 }
 
             } while ((rc == G_SOURCE_CONTINUE) && (read_rc > 0) && --max > 0);
 
         } else {
             crm_trace("New I/O event for %s after I/O condition %d",
                       client->name, (int) condition);
             if (client->dispatch_fn_io) {
                 if (client->dispatch_fn_io(client->userdata) < 0) {
                     crm_trace("Connection to %s no longer required", client->name);
                     rc = G_SOURCE_REMOVE;
                 }
             }
         }
     }
 
     if (client->ipc && crm_ipc_connected(client->ipc) == FALSE) {
         crm_err("Connection to %s closed " CRM_XS "client=%p condition=%d",
                 client->name, client, condition);
         rc = G_SOURCE_REMOVE;
 
     } else if (condition & (G_IO_HUP | G_IO_NVAL | G_IO_ERR)) {
         crm_trace("The connection %s[%p] has been closed (I/O condition=%d)",
                   client->name, client, condition);
         rc = G_SOURCE_REMOVE;
 
     } else if ((condition & G_IO_IN) == 0) {
         /*
            #define      GLIB_SYSDEF_POLLIN     =1
            #define      GLIB_SYSDEF_POLLPRI    =2
            #define      GLIB_SYSDEF_POLLOUT    =4
            #define      GLIB_SYSDEF_POLLERR    =8
            #define      GLIB_SYSDEF_POLLHUP    =16
            #define      GLIB_SYSDEF_POLLNVAL   =32
 
            typedef enum
            {
            G_IO_IN      GLIB_SYSDEF_POLLIN,
            G_IO_OUT     GLIB_SYSDEF_POLLOUT,
            G_IO_PRI     GLIB_SYSDEF_POLLPRI,
            G_IO_ERR     GLIB_SYSDEF_POLLERR,
            G_IO_HUP     GLIB_SYSDEF_POLLHUP,
            G_IO_NVAL    GLIB_SYSDEF_POLLNVAL
            } GIOCondition;
 
            A bitwise combination representing a condition to watch for on an event source.
 
            G_IO_IN      There is data to read.
            G_IO_OUT     Data can be written (without blocking).
            G_IO_PRI     There is urgent data to read.
            G_IO_ERR     Error condition.
            G_IO_HUP     Hung up (the connection has been broken, usually for pipes and sockets).
            G_IO_NVAL    Invalid request. The file descriptor is not open.
          */
         crm_err("Strange condition: %d", condition);
     }
 
     /* G_SOURCE_REMOVE results in mainloop_gio_destroy() being called
      * just before the source is removed from mainloop
      */
     return rc;
 }
 
 static void
 mainloop_gio_destroy(gpointer c)
 {
     mainloop_io_t *client = c;
     char *c_name = strdup(client->name);
 
     /* client->source is valid but about to be destroyed (ref_count == 0) in gmain.c
      * client->channel will still have ref_count > 0... should be == 1
      */
     crm_trace("Destroying client %s[%p]", c_name, c);
 
     if (client->ipc) {
         crm_ipc_close(client->ipc);
     }
 
     if (client->destroy_fn) {
         void (*destroy_fn) (gpointer userdata) = client->destroy_fn;
 
         client->destroy_fn = NULL;
         destroy_fn(client->userdata);
     }
 
     if (client->ipc) {
         crm_ipc_t *ipc = client->ipc;
 
         client->ipc = NULL;
         crm_ipc_destroy(ipc);
     }
 
     crm_trace("Destroyed client %s[%p]", c_name, c);
 
     free(client->name); client->name = NULL;
     free(client);
 
     free(c_name);
 }
 
 /*!
  * \brief Connect to IPC and add it as a main loop source
  *
  * \param[in]  ipc        IPC connection to add
  * \param[in]  priority   Event source priority to use for connection
  * \param[in]  userdata   Data to register with callbacks
  * \param[in]  callbacks  Dispatch and destroy callbacks for connection
  * \param[out] source     Newly allocated event source
  *
  * \return Standard Pacemaker return code
  *
  * \note On failure, the caller is still responsible for ipc. On success, the
  *       caller should call mainloop_del_ipc_client() when source is no longer
  *       needed, which will lead to the disconnection of the IPC later in the
  *       main loop if it is connected. However the IPC disconnects,
  *       mainloop_gio_destroy() will free ipc and source after calling the
  *       destroy callback.
  */
 int
 pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata,
                        struct ipc_client_callbacks *callbacks,
                        mainloop_io_t **source)
 {
     CRM_CHECK((ipc != NULL) && (callbacks != NULL), return EINVAL);
 
     if (!crm_ipc_connect(ipc)) {
         int rc = errno;
         crm_debug("Connection to %s failed: %d", crm_ipc_name(ipc), errno);
         return rc;
     }
     *source = mainloop_add_fd(crm_ipc_name(ipc), priority, crm_ipc_get_fd(ipc),
                               userdata, NULL);
     if (*source == NULL) {
         int rc = errno;
 
         crm_ipc_close(ipc);
         return rc;
     }
     (*source)->ipc = ipc;
     (*source)->destroy_fn = callbacks->destroy;
     (*source)->dispatch_fn_ipc = callbacks->dispatch;
     return pcmk_rc_ok;
 }
 
 /*!
  * \brief Get period for mainloop timer
  *
  * \param[in]  timer      Timer
  *
  * \return Period in ms
  */
 guint
 pcmk__mainloop_timer_get_period(mainloop_timer_t *timer)
 {
     if (timer) {
         return timer->period_ms;
     }
     return 0;
 }
 
 mainloop_io_t *
 mainloop_add_ipc_client(const char *name, int priority, size_t max_size,
                         void *userdata, struct ipc_client_callbacks *callbacks)
 {
     crm_ipc_t *ipc = crm_ipc_new(name, max_size);
     mainloop_io_t *source = NULL;
     int rc = pcmk__add_mainloop_ipc(ipc, priority, userdata, callbacks,
                                     &source);
 
     if (rc != pcmk_rc_ok) {
         if (crm_log_level == LOG_STDOUT) {
             fprintf(stderr, "Connection to %s failed: %s",
                     name, pcmk_rc_str(rc));
         }
         crm_ipc_destroy(ipc);
         if (rc > 0) {
             errno = rc;
         } else {
             errno = ENOTCONN;
         }
         return NULL;
     }
     return source;
 }
 
 void
 mainloop_del_ipc_client(mainloop_io_t * client)
 {
     mainloop_del_fd(client);
 }
 
 crm_ipc_t *
 mainloop_get_ipc_client(mainloop_io_t * client)
 {
     if (client) {
         return client->ipc;
     }
     return NULL;
 }
 
 mainloop_io_t *
 mainloop_add_fd(const char *name, int priority, int fd, void *userdata,
                 struct mainloop_fd_callbacks * callbacks)
 {
     mainloop_io_t *client = NULL;
 
     if (fd >= 0) {
         client = calloc(1, sizeof(mainloop_io_t));
         if (client == NULL) {
             return NULL;
         }
         client->name = strdup(name);
         client->userdata = userdata;
 
         if (callbacks) {
             client->destroy_fn = callbacks->destroy;
             client->dispatch_fn_io = callbacks->dispatch;
         }
 
         client->fd = fd;
         client->channel = g_io_channel_unix_new(fd);
         client->source =
             g_io_add_watch_full(client->channel, priority,
                                 (G_IO_IN | G_IO_HUP | G_IO_NVAL | G_IO_ERR), mainloop_gio_callback,
                                 client, mainloop_gio_destroy);
 
         /* Now that mainloop now holds a reference to channel,
          * thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
          *
          * This means that channel will be free'd by:
          * g_main_context_dispatch() or g_source_remove()
          *  -> g_source_destroy_internal()
          *      -> g_source_callback_unref()
          * shortly after mainloop_gio_destroy() completes
          */
         g_io_channel_unref(client->channel);
         crm_trace("Added connection %d for %s[%p].%d", client->source, client->name, client, fd);
     } else {
         errno = EINVAL;
     }
 
     return client;
 }
 
 void
 mainloop_del_fd(mainloop_io_t * client)
 {
     if (client != NULL) {
         crm_trace("Removing client %s[%p]", client->name, client);
         if (client->source) {
             /* Results in mainloop_gio_destroy() being called just
              * before the source is removed from mainloop
              */
             g_source_remove(client->source);
         }
     }
 }
 
 static GList *child_list = NULL;
 
 pid_t
 mainloop_child_pid(mainloop_child_t * child)
 {
     return child->pid;
 }
 
 const char *
 mainloop_child_name(mainloop_child_t * child)
 {
     return child->desc;
 }
 
 int
 mainloop_child_timeout(mainloop_child_t * child)
 {
     return child->timeout;
 }
 
 void *
 mainloop_child_userdata(mainloop_child_t * child)
 {
     return child->privatedata;
 }
 
 void
 mainloop_clear_child_userdata(mainloop_child_t * child)
 {
     child->privatedata = NULL;
 }
 
 /* good function name */
 static void
 child_free(mainloop_child_t *child)
 {
     if (child->timerid != 0) {
         crm_trace("Removing timer %d", child->timerid);
         g_source_remove(child->timerid);
         child->timerid = 0;
     }
     free(child->desc);
     free(child);
 }
 
 /* terrible function name */
 static int
 child_kill_helper(mainloop_child_t *child)
 {
     int rc;
     if (child->flags & mainloop_leave_pid_group) {
         crm_debug("Kill pid %d only. leave group intact.", child->pid);
         rc = kill(child->pid, SIGKILL);
     } else {
         crm_debug("Kill pid %d's group", child->pid);
         rc = kill(-child->pid, SIGKILL);
     }
 
     if (rc < 0) {
         if (errno != ESRCH) {
             crm_perror(LOG_ERR, "kill(%d, KILL) failed", child->pid);
         }
         return -errno;
     }
     return 0;
 }
 
 static gboolean
 child_timeout_callback(gpointer p)
 {
     mainloop_child_t *child = p;
     int rc = 0;
 
     child->timerid = 0;
     if (child->timeout) {
-        crm_crit("%s process (PID %d) will not die!", child->desc, (int)child->pid);
+        crm_warn("%s process (PID %d) will not die!", child->desc, (int)child->pid);
         return FALSE;
     }
 
     rc = child_kill_helper(child);
     if (rc == -ESRCH) {
         /* Nothing left to do. pid doesn't exist */
         return FALSE;
     }
 
     child->timeout = TRUE;
-    crm_warn("%s process (PID %d) timed out", child->desc, (int)child->pid);
+    crm_debug("%s process (PID %d) timed out", child->desc, (int)child->pid);
 
     child->timerid = g_timeout_add(5000, child_timeout_callback, child);
     return FALSE;
 }
 
 static bool
 child_waitpid(mainloop_child_t *child, int flags)
 {
     int rc = 0;
     int core = 0;
     int signo = 0;
     int status = 0;
     int exitcode = 0;
     bool callback_needed = true;
 
     rc = waitpid(child->pid, &status, flags);
     if (rc == 0) { // WNOHANG in flags, and child status is not available
         crm_trace("Child process %d (%s) still active",
                   child->pid, child->desc);
         callback_needed = false;
 
     } else if (rc != child->pid) {
         /* According to POSIX, possible conditions:
          * - child->pid was non-positive (process group or any child),
          *   and rc is specific child
          * - errno ECHILD (pid does not exist or is not child)
          * - errno EINVAL (invalid flags)
          * - errno EINTR (caller interrupted by signal)
          *
          * @TODO Handle these cases more specifically.
          */
         signo = SIGCHLD;
         exitcode = 1;
         crm_notice("Wait for child process %d (%s) interrupted: %s",
                    child->pid, child->desc, pcmk_strerror(errno));
 
     } else if (WIFEXITED(status)) {
         exitcode = WEXITSTATUS(status);
         crm_trace("Child process %d (%s) exited with status %d",
                   child->pid, child->desc, exitcode);
 
     } else if (WIFSIGNALED(status)) {
         signo = WTERMSIG(status);
         crm_trace("Child process %d (%s) exited with signal %d (%s)",
                   child->pid, child->desc, signo, strsignal(signo));
 
 #ifdef WCOREDUMP // AIX, SunOS, maybe others
     } else if (WCOREDUMP(status)) {
         core = 1;
         crm_err("Child process %d (%s) dumped core",
                 child->pid, child->desc);
 #endif
 
     } else { // flags must contain WUNTRACED and/or WCONTINUED to reach this
         crm_trace("Child process %d (%s) stopped or continued",
                   child->pid, child->desc);
         callback_needed = false;
     }
 
     if (callback_needed && child->callback) {
         child->callback(child, child->pid, core, signo, exitcode);
     }
     return callback_needed;
 }
 
 static void
 child_death_dispatch(int signal)
 {
     for (GList *iter = child_list; iter; ) {
         GList *saved = iter;
         mainloop_child_t *child = iter->data;
 
         iter = iter->next;
         if (child_waitpid(child, WNOHANG)) {
             crm_trace("Removing completed process %d from child list",
                       child->pid);
             child_list = g_list_remove_link(child_list, saved);
             g_list_free(saved);
             child_free(child);
         }
     }
 }
 
 static gboolean
 child_signal_init(gpointer p)
 {
     crm_trace("Installed SIGCHLD handler");
     /* Do NOT use g_child_watch_add() and friends, they rely on pthreads */
     mainloop_add_signal(SIGCHLD, child_death_dispatch);
 
     /* In case they terminated before the signal handler was installed */
     child_death_dispatch(SIGCHLD);
     return FALSE;
 }
 
 gboolean
 mainloop_child_kill(pid_t pid)
 {
     GList *iter;
     mainloop_child_t *child = NULL;
     mainloop_child_t *match = NULL;
     /* It is impossible to block SIGKILL, this allows us to
      * call waitpid without WNOHANG flag.*/
     int waitflags = 0, rc = 0;
 
     for (iter = child_list; iter != NULL && match == NULL; iter = iter->next) {
         child = iter->data;
         if (pid == child->pid) {
             match = child;
         }
     }
 
     if (match == NULL) {
         return FALSE;
     }
 
     rc = child_kill_helper(match);
     if(rc == -ESRCH) {
         /* It's gone, but hasn't shown up in waitpid() yet. Wait until we get
          * SIGCHLD and let handler clean it up as normal (so we get the correct
          * return code/status). The blocking alternative would be to call
          * child_waitpid(match, 0).
          */
         crm_trace("Waiting for signal that child process %d completed",
                   match->pid);
         return TRUE;
 
     } else if(rc != 0) {
         /* If KILL for some other reason set the WNOHANG flag since we
          * can't be certain what happened.
          */
         waitflags = WNOHANG;
     }
 
     if (!child_waitpid(match, waitflags)) {
         /* not much we can do if this occurs */
         return FALSE;
     }
 
     child_list = g_list_remove(child_list, match);
     child_free(match);
     return TRUE;
 }
 
 /* Create/Log a new tracked process
  * To track a process group, use -pid
  *
  * @TODO Using a non-positive pid (i.e. any child, or process group) would
  *       likely not be useful since we will free the child after the first
  *       completed process.
  */
 void
 mainloop_child_add_with_flags(pid_t pid, int timeout, const char *desc, void *privatedata, enum mainloop_child_flags flags, 
                    void (*callback) (mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode))
 {
     static bool need_init = TRUE;
     mainloop_child_t *child = g_new(mainloop_child_t, 1);
 
     child->pid = pid;
     child->timerid = 0;
     child->timeout = FALSE;
     child->privatedata = privatedata;
     child->callback = callback;
     child->flags = flags;
 
     if(desc) {
         child->desc = strdup(desc);
     }
 
     if (timeout) {
         child->timerid = g_timeout_add(timeout, child_timeout_callback, child);
     }
 
     child_list = g_list_append(child_list, child);
 
     if(need_init) {
         need_init = FALSE;
         /* SIGCHLD processing has to be invoked from mainloop.
          * We do not want it to be possible to both add a child pid
          * to mainloop, and have the pid's exit callback invoked within
          * the same callstack. */
         g_timeout_add(1, child_signal_init, NULL);
     }
 }
 
 void
 mainloop_child_add(pid_t pid, int timeout, const char *desc, void *privatedata,
                    void (*callback) (mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode))
 {
     mainloop_child_add_with_flags(pid, timeout, desc, privatedata, 0, callback);
 }
 
 static gboolean
 mainloop_timer_cb(gpointer user_data)
 {
     int id = 0;
     bool repeat = FALSE;
     struct mainloop_timer_s *t = user_data;
 
     CRM_ASSERT(t != NULL);
 
     id = t->id;
     t->id = 0; /* Ensure it's unset during callbacks so that
                 * mainloop_timer_running() works as expected
                 */
 
     if(t->cb) {
         crm_trace("Invoking callbacks for timer %s", t->name);
         repeat = t->repeat;
         if(t->cb(t->userdata) == FALSE) {
             crm_trace("Timer %s complete", t->name);
             repeat = FALSE;
         }
     }
 
     if(repeat) {
         /* Restore if repeating */
         t->id = id;
     }
 
     return repeat;
 }
 
 bool
 mainloop_timer_running(mainloop_timer_t *t)
 {
     if(t && t->id != 0) {
         return TRUE;
     }
     return FALSE;
 }
 
 void
 mainloop_timer_start(mainloop_timer_t *t)
 {
     mainloop_timer_stop(t);
     if(t && t->period_ms > 0) {
         crm_trace("Starting timer %s", t->name);
         t->id = g_timeout_add(t->period_ms, mainloop_timer_cb, t);
     }
 }
 
 void
 mainloop_timer_stop(mainloop_timer_t *t)
 {
     if(t && t->id != 0) {
         crm_trace("Stopping timer %s", t->name);
         g_source_remove(t->id);
         t->id = 0;
     }
 }
 
 guint
 mainloop_timer_set_period(mainloop_timer_t *t, guint period_ms)
 {
     guint last = 0;
 
     if(t) {
         last = t->period_ms;
         t->period_ms = period_ms;
     }
 
     if(t && t->id != 0 && last != t->period_ms) {
         mainloop_timer_start(t);
     }
     return last;
 }
 
 mainloop_timer_t *
 mainloop_timer_add(const char *name, guint period_ms, bool repeat, GSourceFunc cb, void *userdata)
 {
     mainloop_timer_t *t = calloc(1, sizeof(mainloop_timer_t));
 
     if(t) {
         if(name) {
             t->name = crm_strdup_printf("%s-%u-%d", name, period_ms, repeat);
         } else {
             t->name = crm_strdup_printf("%p-%u-%d", t, period_ms, repeat);
         }
         t->id = 0;
         t->period_ms = period_ms;
         t->repeat = repeat;
         t->cb = cb;
         t->userdata = userdata;
         crm_trace("Created timer %s with %p %p", t->name, userdata, t->userdata);
     }
     return t;
 }
 
 void
 mainloop_timer_del(mainloop_timer_t *t)
 {
     if(t) {
         crm_trace("Destroying timer %s", t->name);
         mainloop_timer_stop(t);
         free(t->name);
         free(t);
     }
 }
 
 /*
  * Helpers to make sure certain events aren't lost at shutdown
  */
 
 static gboolean
 drain_timeout_cb(gpointer user_data)
 {
     bool *timeout_popped = (bool*) user_data;
 
     *timeout_popped = TRUE;
     return FALSE;
 }
 
 /*!
  * \brief Drain some remaining main loop events then quit it
  *
  * \param[in] mloop  Main loop to drain and quit
  * \param[in] n      Drain up to this many pending events
  */
 void
 pcmk_quit_main_loop(GMainLoop *mloop, unsigned int n)
 {
     if ((mloop != NULL) && g_main_loop_is_running(mloop)) {
         GMainContext *ctx = g_main_loop_get_context(mloop);
 
         /* Drain up to n events in case some memory clean-up is pending
          * (helpful to reduce noise in valgrind output).
          */
         for (int i = 0; (i < n) && g_main_context_pending(ctx); ++i) {
             g_main_context_dispatch(ctx);
         }
         g_main_loop_quit(mloop);
     }
 }
 
 /*!
  * \brief Process main loop events while a certain condition is met
  *
  * \param[in] mloop     Main loop to process
  * \param[in] timer_ms  Don't process longer than this amount of time
  * \param[in] check     Function that returns TRUE if events should be processed
  *
  * \note This function is intended to be called at shutdown if certain important
  *       events should not be missed. The caller would likely quit the main loop
  *       or exit after calling this function. The check() function will be
  *       passed the remaining timeout in milliseconds.
  */
 void
 pcmk_drain_main_loop(GMainLoop *mloop, guint timer_ms, bool (*check)(guint))
 {
     bool timeout_popped = FALSE;
     guint timer = 0;
     GMainContext *ctx = NULL;
 
     CRM_CHECK(mloop && check, return);
 
     ctx = g_main_loop_get_context(mloop);
     if (ctx) {
         time_t start_time = time(NULL);
 
         timer = g_timeout_add(timer_ms, drain_timeout_cb, &timeout_popped);
         while (!timeout_popped
                && check(timer_ms - (time(NULL) - start_time) * 1000)) {
             g_main_context_iteration(ctx, TRUE);
         }
     }
     if (!timeout_popped && (timer > 0)) {
         g_source_remove(timer);
     }
 }
 
 // Deprecated functions kept only for backward API compatibility
 // LCOV_EXCL_START
 
 #include <crm/common/mainloop_compat.h>
 
 gboolean
 crm_signal(int sig, void (*dispatch) (int sig))
 {
     return crm_signal_handler(sig, dispatch) != SIG_ERR;
 }
 
 // LCOV_EXCL_STOP
 // End deprecated API
diff --git a/lib/services/services.c b/lib/services/services.c
index 5c17be17a8..1a496c92dc 100644
--- a/lib/services/services.c
+++ b/lib/services/services.c
@@ -1,1266 +1,1311 @@
 /*
  * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <stdio.h>
 #include <errno.h>
 #include <unistd.h>
 #include <dirent.h>
 #include <fcntl.h>
 
 #include <crm/crm.h>
 #include <crm/common/mainloop.h>
 #include <crm/services.h>
 #include <crm/services_internal.h>
 #include <crm/stonith-ng.h>
 #include <crm/msg_xml.h>
 #include "services_private.h"
 #include "services_ocf.h"
 #include "services_lsb.h"
 
 #if SUPPORT_UPSTART
 #  include <upstart.h>
 #endif
 
 #if SUPPORT_SYSTEMD
 #  include <systemd.h>
 #endif
 
 #if SUPPORT_NAGIOS
 #  include <services_nagios.h>
 #endif
 
 /* TODO: Develop a rollover strategy */
 
 static int operations = 0;
 static GHashTable *recurring_actions = NULL;
 
 /* ops waiting to run async because of conflicting active
  * pending ops */
 static GList *blocked_ops = NULL;
 
 /* ops currently active (in-flight) */
 static GList *inflight_ops = NULL;
 
 static void handle_blocked_ops(void);
 
 /*!
  * \brief Find first service class that can provide a specified agent
  *
  * \param[in] agent  Name of agent to search for
  *
  * \return Service class if found, NULL otherwise
  *
  * \note The priority is LSB, then systemd, then upstart. It would be preferable
  *       to put systemd first, but LSB merely requires a file existence check,
  *       while systemd requires contacting D-Bus.
  */
 const char *
 resources_find_service_class(const char *agent)
 {
     if (services__lsb_agent_exists(agent)) {
         return PCMK_RESOURCE_CLASS_LSB;
     }
 
 #if SUPPORT_SYSTEMD
     if (systemd_unit_exists(agent)) {
         return PCMK_RESOURCE_CLASS_SYSTEMD;
     }
 #endif
 
 #if SUPPORT_UPSTART
     if (upstart_job_exists(agent)) {
         return PCMK_RESOURCE_CLASS_UPSTART;
     }
 #endif
     return NULL;
 }
 
 static inline void
 init_recurring_actions(void)
 {
     if (recurring_actions == NULL) {
         recurring_actions = pcmk__strkey_table(NULL, NULL);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether op is in-flight systemd or upstart op
  *
  * \param[in] op  Operation to check
  *
  * \return TRUE if op is in-flight systemd or upstart op
  */
 static inline gboolean
 inflight_systemd_or_upstart(svc_action_t *op)
 {
     return pcmk__strcase_any_of(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD,
                            PCMK_RESOURCE_CLASS_UPSTART, NULL) &&
            g_list_find(inflight_ops, op) != NULL;
 }
 
 /*!
  * \internal
  * \brief Expand "service" alias to an actual resource class
  *
  * \param[in] rsc       Resource name (for logging only)
  * \param[in] standard  Resource class as configured
  * \param[in] agent     Agent name to look for
  *
  * \return Newly allocated string with actual resource class
  *
  * \note The caller is responsible for calling free() on the result.
  */
 static char *
 expand_resource_class(const char *rsc, const char *standard, const char *agent)
 {
     char *expanded_class = NULL;
 
     if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0) {
         const char *found_class = resources_find_service_class(agent);
 
         if (found_class) {
             crm_debug("Found %s agent %s for %s", found_class, agent, rsc);
             expanded_class = strdup(found_class);
         } else {
             crm_info("Assuming resource class lsb for agent %s for %s",
                      agent, rsc);
             expanded_class = strdup(PCMK_RESOURCE_CLASS_LSB);
         }
     } else {
         expanded_class = strdup(standard);
     }
     CRM_ASSERT(expanded_class);
     return expanded_class;
 }
 
 /*!
  * \internal
  * \brief Create a simple svc_action_t instance
  *
  * \return Newly allocated instance (or NULL if not enough memory)
  */
 static svc_action_t *
 new_action(void)
 {
     svc_action_t *op = calloc(1, sizeof(svc_action_t));
 
     if (op == NULL) {
         return NULL;
     }
 
     op->opaque = calloc(1, sizeof(svc_action_private_t));
     if (op->opaque == NULL) {
         free(op);
         return NULL;
     }
 
     // Initialize result
     services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_UNKNOWN, NULL);
     return op;
 }
 
 static bool
 required_argument_missing(uint32_t ra_caps, const char *name,
                           const char *standard, const char *provider,
                           const char *agent, const char *action)
 {
     if (pcmk__str_empty(name)) {
         crm_info("Cannot create operation without resource name (bug?)");
         return true;
     }
 
     if (pcmk__str_empty(standard)) {
         crm_info("Cannot create operation for %s without resource class (bug?)",
                  name);
         return true;
     }
 
     if (pcmk_is_set(ra_caps, pcmk_ra_cap_provider)
         && pcmk__str_empty(provider)) {
         crm_info("Cannot create operation for %s resource %s "
                  "without provider (bug?)", standard, name);
         return true;
     }
 
     if (pcmk__str_empty(agent)) {
         crm_info("Cannot create operation for %s without agent name (bug?)",
                  name);
         return true;
     }
 
     if (pcmk__str_empty(action)) {
         crm_info("Cannot create operation for %s without action name (bug?)",
                  name);
         return true;
     }
     return false;
 }
 
 // \return Standard Pacemaker return code (pcmk_rc_ok or ENOMEM)
 static int
 copy_action_arguments(svc_action_t *op, uint32_t ra_caps, const char *name,
                       const char *standard, const char *provider,
                       const char *agent, const char *action)
 {
     op->rsc = strdup(name);
     if (op->rsc == NULL) {
         return ENOMEM;
     }
 
     op->agent = strdup(agent);
     if (op->agent == NULL) {
         return ENOMEM;
     }
 
     op->standard = expand_resource_class(name, standard, agent);
     if (op->standard == NULL) {
         return ENOMEM;
     }
 
     if (pcmk_is_set(ra_caps, pcmk_ra_cap_status)
         && pcmk__str_eq(action, "monitor", pcmk__str_casei)) {
         action = "status";
     }
     op->action = strdup(action);
     if (op->action == NULL) {
         return ENOMEM;
     }
 
     if (pcmk_is_set(ra_caps, pcmk_ra_cap_provider)) {
         op->provider = strdup(provider);
         if (op->provider == NULL) {
             return ENOMEM;
         }
     }
     return pcmk_rc_ok;
 }
 
 svc_action_t *
 services__create_resource_action(const char *name, const char *standard,
                         const char *provider, const char *agent,
                         const char *action, guint interval_ms, int timeout,
                         GHashTable *params, enum svc_action_flags flags)
 {
     svc_action_t *op = NULL;
     uint32_t ra_caps = pcmk_get_ra_caps(standard);
     int rc = pcmk_rc_ok;
 
     op = new_action();
     if (op == NULL) {
         crm_crit("Cannot prepare action: %s", strerror(ENOMEM));
         if (params != NULL) {
             g_hash_table_destroy(params);
         }
         return NULL;
     }
 
     op->interval_ms = interval_ms;
     op->timeout = timeout;
     op->flags = flags;
     op->sequence = ++operations;
 
     // Take ownership of params
     if (pcmk_is_set(ra_caps, pcmk_ra_cap_params)) {
         op->params = params;
     } else if (params != NULL) {
         g_hash_table_destroy(params);
         params = NULL;
     }
 
     if (required_argument_missing(ra_caps, name, standard, provider, agent,
                                   action)) {
         services__set_result(op, services__generic_error(op),
                              PCMK_EXEC_ERROR_FATAL,
                              "Required agent or action information missing");
         return op;
     }
 
     op->id = pcmk__op_key(name, action, interval_ms);
 
     if (copy_action_arguments(op, ra_caps, name, standard, provider, agent,
                               action) != pcmk_rc_ok) {
         crm_crit("Cannot prepare %s action for %s: %s",
                  action, name, strerror(ENOMEM));
         services__handle_exec_error(op, ENOMEM);
         return op;
     }
 
     if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
         rc = services__ocf_prepare(op);
 
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
         rc = services__lsb_prepare(op);
 
 #if SUPPORT_SYSTEMD
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
         rc = services__systemd_prepare(op);
 #endif
 #if SUPPORT_UPSTART
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
         rc = services__upstart_prepare(op);
 #endif
 #if SUPPORT_NAGIOS
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
         rc = services__nagios_prepare(op);
 #endif
     } else {
         crm_err("Unknown resource standard: %s", op->standard);
         rc = ENOENT;
     }
 
     if (rc != pcmk_rc_ok) {
         crm_err("Cannot prepare %s operation for %s: %s",
                 action, name, strerror(rc));
         services__handle_exec_error(op, rc);
     }
     return op;
 }
 
 svc_action_t *
 resources_action_create(const char *name, const char *standard,
                         const char *provider, const char *agent,
                         const char *action, guint interval_ms, int timeout,
                         GHashTable *params, enum svc_action_flags flags)
 {
     svc_action_t *op = services__create_resource_action(name, standard,
                             provider, agent, action, interval_ms, timeout,
                             params, flags);
     if (op == NULL || op->rc != 0) {
         services_action_free(op);
         return NULL;
     } else {
         // Preserve public API backward compatibility
         op->rc = PCMK_OCF_OK;
         op->status = PCMK_EXEC_DONE;
 
         return op;
     }
 }
 
 svc_action_t *
 services_action_create_generic(const char *exec, const char *args[])
 {
     svc_action_t *op = new_action();
 
     CRM_ASSERT(op != NULL);
 
     op->opaque->exec = strdup(exec);
     op->opaque->args[0] = strdup(exec);
     if ((op->opaque->exec == NULL) || (op->opaque->args[0] == NULL)) {
         crm_crit("Cannot prepare action for '%s': %s", exec, strerror(ENOMEM));
         services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                              strerror(ENOMEM));
         return op;
     }
 
     if (args == NULL) {
         return op;
     }
 
     for (int cur_arg = 1; args[cur_arg - 1] != NULL; cur_arg++) {
 
         if (cur_arg == PCMK__NELEM(op->opaque->args)) {
             crm_info("Cannot prepare action for '%s': Too many arguments",
                      exec);
             services__set_result(op, PCMK_OCF_UNKNOWN_ERROR,
                                  PCMK_EXEC_ERROR_HARD, "Too many arguments");
             break;
         }
 
         op->opaque->args[cur_arg] = strdup(args[cur_arg - 1]);
         if (op->opaque->args[cur_arg] == NULL) {
             crm_crit("Cannot prepare action for '%s': %s",
                      exec, strerror(ENOMEM));
             services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                  strerror(ENOMEM));
             break;
         }
     }
 
     return op;
 }
 
 /*!
  * \brief Create an alert agent action
  *
  * \param[in] id        Alert ID
  * \param[in] exec      Path to alert agent executable
  * \param[in] timeout   Action timeout
  * \param[in] params    Parameters to use with action
  * \param[in] sequence  Action sequence number
  * \param[in] cb_data   Data to pass to callback function
  *
  * \return New action on success, NULL on error
  * \note It is the caller's responsibility to free cb_data.
  *       The caller should not free params explicitly.
  */
 svc_action_t *
 services_alert_create(const char *id, const char *exec, int timeout,
                       GHashTable *params, int sequence, void *cb_data)
 {
     svc_action_t *action = services_action_create_generic(exec, NULL);
 
     action->timeout = timeout;
     action->id = strdup(id);
     action->params = params;
     action->sequence = sequence;
     action->cb_data = cb_data;
     return action;
 }
 
 /*!
  * \brief Set the user and group that an action will execute as
  *
  * \param[in,out] action  Action to modify
  * \param[in]     user    Name of user to execute action as
  * \param[in]     group   Name of group to execute action as
  *
  * \return pcmk_ok on success, -errno otherwise
  *
  * \note This will have no effect unless the process executing the action runs
  *       as root, and the action is not a systemd or upstart action.
  *       We could implement this for systemd by adding User= and Group= to
  *       [Service] in the override file, but that seems more likely to cause
  *       problems than be useful.
  */
 int
 services_action_user(svc_action_t *op, const char *user)
 {
     CRM_CHECK((op != NULL) && (user != NULL), return -EINVAL);
     return crm_user_lookup(user, &(op->opaque->uid), &(op->opaque->gid));
 }
 
 /*!
  * \brief Execute an alert agent action
  *
  * \param[in] action  Action to execute
  * \param[in] cb      Function to call when action completes
  *
  * \return TRUE if the library will free action, FALSE otherwise
  *
  * \note If this function returns FALSE, it is the caller's responsibility to
  *       free the action with services_action_free().
  */
 gboolean
 services_alert_async(svc_action_t *action, void (*cb)(svc_action_t *op))
 {
     action->synchronous = false;
     action->opaque->callback = cb;
     return services__execute_file(action) == pcmk_rc_ok;
 }
 
 #if SUPPORT_DBUS
 /*!
  * \internal
  * \brief Update operation's pending DBus call, unreferencing old one if needed
  *
  * \param[in,out] op       Operation to modify
  * \param[in]     pending  Pending call to set
  */
 void
 services_set_op_pending(svc_action_t *op, DBusPendingCall *pending)
 {
     if (op->opaque->pending && (op->opaque->pending != pending)) {
         if (pending) {
             crm_info("Lost pending %s DBus call (%p)", op->id, op->opaque->pending);
         } else {
             crm_trace("Done with pending %s DBus call (%p)", op->id, op->opaque->pending);
         }
         dbus_pending_call_unref(op->opaque->pending);
     }
     op->opaque->pending = pending;
     if (pending) {
         crm_trace("Updated pending %s DBus call (%p)", op->id, pending);
     } else {
         crm_trace("Cleared pending %s DBus call", op->id);
     }
 }
 #endif
 
 void
 services_action_cleanup(svc_action_t * op)
 {
     if ((op == NULL) || (op->opaque == NULL)) {
         return;
     }
 
 #if SUPPORT_DBUS
     if(op->opaque->timerid != 0) {
         crm_trace("Removing timer for call %s to %s", op->action, op->rsc);
         g_source_remove(op->opaque->timerid);
         op->opaque->timerid = 0;
     }
 
     if(op->opaque->pending) {
         if (dbus_pending_call_get_completed(op->opaque->pending)) {
             // This should never be the case
             crm_warn("Result of %s op %s was unhandled",
                      op->standard, op->id);
         } else {
             crm_debug("Will ignore any result of canceled %s op %s",
                       op->standard, op->id);
         }
         dbus_pending_call_cancel(op->opaque->pending);
         services_set_op_pending(op, NULL);
     }
 #endif
 
     if (op->opaque->stderr_gsource) {
         mainloop_del_fd(op->opaque->stderr_gsource);
         op->opaque->stderr_gsource = NULL;
     }
 
     if (op->opaque->stdout_gsource) {
         mainloop_del_fd(op->opaque->stdout_gsource);
         op->opaque->stdout_gsource = NULL;
     }
 }
 
+/*!
+ * \internal
+ * \brief Map an actual resource action result to a standard OCF result
+ *
+ * \param[in] standard     Agent standard (must not be "service")
+ * \param[in] action       Action that result is for
+ * \param[in] exit_status  Actual agent exit status
+ *
+ * \return Standard OCF result
+ */
+enum ocf_exitcode
+services_result2ocf(const char *standard, const char *action, int exit_status)
+{
+    if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
+        return services__ocf2ocf(exit_status);
+
+#if SUPPORT_SYSTEMD
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD,
+                            pcmk__str_casei)) {
+        return services__systemd2ocf(exit_status);
+#endif
+
+#if SUPPORT_UPSTART
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART,
+                            pcmk__str_casei)) {
+        return services__upstart2ocf(exit_status);
+#endif
+
+#if SUPPORT_NAGIOS
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS,
+                            pcmk__str_casei)) {
+        return services__nagios2ocf(exit_status);
+#endif
+
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_LSB,
+                            pcmk__str_casei)) {
+        return services__lsb2ocf(action, exit_status);
+
+    } else {
+        crm_warn("Treating result from unknown standard '%s' as OCF",
+                 ((standard == NULL)? "unspecified" : standard));
+        return services__ocf2ocf(exit_status);
+    }
+}
+
 void
 services_action_free(svc_action_t * op)
 {
     unsigned int i;
 
     if (op == NULL) {
         return;
     }
 
     /* The operation should be removed from all tracking lists by this point.
      * If it's not, we have a bug somewhere, so bail. That may lead to a
      * memory leak, but it's better than a use-after-free segmentation fault.
      */
     CRM_CHECK(g_list_find(inflight_ops, op) == NULL, return);
     CRM_CHECK(g_list_find(blocked_ops, op) == NULL, return);
     CRM_CHECK((recurring_actions == NULL)
               || (g_hash_table_lookup(recurring_actions, op->id) == NULL),
               return);
 
     services_action_cleanup(op);
 
     if (op->opaque->repeat_timer) {
         g_source_remove(op->opaque->repeat_timer);
         op->opaque->repeat_timer = 0;
     }
 
     free(op->id);
     free(op->opaque->exec);
 
     for (i = 0; i < PCMK__NELEM(op->opaque->args); i++) {
         free(op->opaque->args[i]);
     }
 
     free(op->opaque->exit_reason);
     free(op->opaque);
     free(op->rsc);
     free(op->action);
 
     free(op->standard);
     free(op->agent);
     free(op->provider);
 
     free(op->stdout_data);
     free(op->stderr_data);
 
     if (op->params) {
         g_hash_table_destroy(op->params);
         op->params = NULL;
     }
 
     free(op);
 }
 
 gboolean
 cancel_recurring_action(svc_action_t * op)
 {
     crm_info("Cancelling %s operation %s", op->standard, op->id);
 
     if (recurring_actions) {
         g_hash_table_remove(recurring_actions, op->id);
     }
 
     if (op->opaque->repeat_timer) {
         g_source_remove(op->opaque->repeat_timer);
         op->opaque->repeat_timer = 0;
     }
 
     return TRUE;
 }
 
 /*!
  * \brief Cancel a recurring action
  *
  * \param[in] name         Name of resource that operation is for
  * \param[in] action       Name of operation to cancel
  * \param[in] interval_ms  Interval of operation to cancel
  *
  * \return TRUE if action was successfully cancelled, FALSE otherwise
  */
 gboolean
 services_action_cancel(const char *name, const char *action, guint interval_ms)
 {
     gboolean cancelled = FALSE;
     char *id = pcmk__op_key(name, action, interval_ms);
     svc_action_t *op = NULL;
 
     /* We can only cancel a recurring action */
     init_recurring_actions();
     op = g_hash_table_lookup(recurring_actions, id);
     if (op == NULL) {
         goto done;
     }
 
     // Tell services__finalize_async_op() not to reschedule the operation
     op->cancel = TRUE;
 
     /* Stop tracking it as a recurring operation, and stop its repeat timer */
     cancel_recurring_action(op);
 
     /* If the op has a PID, it's an in-flight child process, so kill it.
      *
      * Whether the kill succeeds or fails, the main loop will send the op to
      * async_action_complete() (and thus services__finalize_async_op()) when the
      * process goes away.
      */
     if (op->pid != 0) {
         crm_info("Terminating in-flight op %s[%d] early because it was cancelled",
                  id, op->pid);
         cancelled = mainloop_child_kill(op->pid);
         if (cancelled == FALSE) {
             crm_err("Termination of %s[%d] failed", id, op->pid);
         }
         goto done;
     }
 
 #if SUPPORT_DBUS
     // In-flight systemd and upstart ops don't have a pid
     if (inflight_systemd_or_upstart(op)) {
         inflight_ops = g_list_remove(inflight_ops, op);
 
         /* This will cause any result that comes in later to be discarded, so we
          * don't call the callback and free the operation twice.
          */
         services_action_cleanup(op);
     }
 #endif
 
     /* The rest of this is essentially equivalent to
      * services__finalize_async_op(), minus the handle_blocked_ops() call.
      */
 
     // Report operation as cancelled
     services__set_cancelled(op);
     if (op->opaque->callback) {
         op->opaque->callback(op);
     }
 
     blocked_ops = g_list_remove(blocked_ops, op);
     services_action_free(op);
     cancelled = TRUE;
     // @TODO Initiate handle_blocked_ops() asynchronously
 
 done:
     free(id);
     return cancelled;
 }
 
 gboolean
 services_action_kick(const char *name, const char *action, guint interval_ms)
 {
     svc_action_t * op = NULL;
     char *id = pcmk__op_key(name, action, interval_ms);
 
     init_recurring_actions();
     op = g_hash_table_lookup(recurring_actions, id);
     free(id);
 
     if (op == NULL) {
         return FALSE;
     }
 
 
     if (op->pid || inflight_systemd_or_upstart(op)) {
         return TRUE;
     } else {
         if (op->opaque->repeat_timer) {
             g_source_remove(op->opaque->repeat_timer);
             op->opaque->repeat_timer = 0;
         }
         recurring_action_timer(op);
         return TRUE;
     }
 
 }
 
 /*!
  * \internal
  * \brief Add a new recurring operation, checking for duplicates
  *
  * \param[in] op               Operation to add
  *
  * \return TRUE if duplicate found (and reschedule), FALSE otherwise
  */
 static gboolean
 handle_duplicate_recurring(svc_action_t * op)
 {
     svc_action_t * dup = NULL;
 
     /* check for duplicates */
     dup = g_hash_table_lookup(recurring_actions, op->id);
 
     if (dup && (dup != op)) {
         /* update user data */
         if (op->opaque->callback) {
             dup->opaque->callback = op->opaque->callback;
             dup->cb_data = op->cb_data;
             op->cb_data = NULL;
         }
         /* immediately execute the next interval */
         if (dup->pid != 0) {
             if (op->opaque->repeat_timer) {
                 g_source_remove(op->opaque->repeat_timer);
                 op->opaque->repeat_timer = 0;
             }
             recurring_action_timer(dup);
         }
         /* free the duplicate */
         services_action_free(op);
         return TRUE;
     }
 
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Execute an action appropriately according to its standard
  *
  * \param[in] op  Action to execute
  *
  * \return Standard Pacemaker return code
  * \retval EBUSY          Recurring operation could not be initiated
  * \retval pcmk_rc_error  Synchronous action failed
  * \retval pcmk_rc_ok     Synchronous action succeeded, or asynchronous action
  *                        should not be freed (because it already was or is
  *                        pending)
  *
  * \note If the return value for an asynchronous action is not pcmk_rc_ok, the
  *       caller is responsible for freeing the action.
  */
 static int
 execute_action(svc_action_t *op)
 {
 #if SUPPORT_UPSTART
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_UPSTART,
                      pcmk__str_casei)) {
         return services__execute_upstart(op);
     }
 #endif
 
 #if SUPPORT_SYSTEMD
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD,
                      pcmk__str_casei)) {
         return services__execute_systemd(op);
     }
 #endif
 
     return services__execute_file(op);
 }
 
 void
 services_add_inflight_op(svc_action_t * op)
 {
     if (op == NULL) {
         return;
     }
 
     CRM_ASSERT(op->synchronous == FALSE);
 
     /* keep track of ops that are in-flight to avoid collisions in the same namespace */
     if (op->rsc) {
         inflight_ops = g_list_append(inflight_ops, op);
     }
 }
 
 /*!
  * \internal
  * \brief Stop tracking an operation that completed
  *
  * \param[in] op  Operation to stop tracking
  */
 void
 services_untrack_op(svc_action_t *op)
 {
     /* Op is no longer in-flight or blocked */
     inflight_ops = g_list_remove(inflight_ops, op);
     blocked_ops = g_list_remove(blocked_ops, op);
 
     /* Op is no longer blocking other ops, so check if any need to run */
     handle_blocked_ops();
 }
 
 gboolean
 services_action_async_fork_notify(svc_action_t * op,
                                   void (*action_callback) (svc_action_t *),
                                   void (*action_fork_callback) (svc_action_t *))
 {
     op->synchronous = false;
     if (action_callback) {
         op->opaque->callback = action_callback;
     }
     if (action_fork_callback) {
         op->opaque->fork_callback = action_fork_callback;
     }
 
     if (op->interval_ms > 0) {
         init_recurring_actions();
         if (handle_duplicate_recurring(op) == TRUE) {
             /* entry rescheduled, dup freed */
             /* exit early */
             return TRUE;
         }
         g_hash_table_replace(recurring_actions, op->id, op);
     }
 
     if (!pcmk_is_set(op->flags, SVC_ACTION_NON_BLOCKED)
         && op->rsc && is_op_blocked(op->rsc)) {
         blocked_ops = g_list_append(blocked_ops, op);
         return TRUE;
     }
 
     return execute_action(op) == pcmk_rc_ok;
 }
 
 gboolean
 services_action_async(svc_action_t * op,
                       void (*action_callback) (svc_action_t *))
 {
     return services_action_async_fork_notify(op, action_callback, NULL);
 }
 
 static gboolean processing_blocked_ops = FALSE;
 
 gboolean
 is_op_blocked(const char *rsc)
 {
     GList *gIter = NULL;
     svc_action_t *op = NULL;
 
     for (gIter = inflight_ops; gIter != NULL; gIter = gIter->next) {
         op = gIter->data;
         if (pcmk__str_eq(op->rsc, rsc, pcmk__str_casei)) {
             return TRUE;
         }
     }
 
     return FALSE;
 }
 
 static void
 handle_blocked_ops(void)
 {
     GList *executed_ops = NULL;
     GList *gIter = NULL;
     svc_action_t *op = NULL;
 
     if (processing_blocked_ops) {
         /* avoid nested calling of this function */
         return;
     }
 
     processing_blocked_ops = TRUE;
 
     /* n^2 operation here, but blocked ops are incredibly rare. this list
      * will be empty 99% of the time. */
     for (gIter = blocked_ops; gIter != NULL; gIter = gIter->next) {
         op = gIter->data;
         if (is_op_blocked(op->rsc)) {
             continue;
         }
         executed_ops = g_list_append(executed_ops, op);
         if (execute_action(op) != pcmk_rc_ok) {
             /* this can cause this function to be called recursively
              * which is why we have processing_blocked_ops static variable */
             services__finalize_async_op(op);
         }
     }
 
     for (gIter = executed_ops; gIter != NULL; gIter = gIter->next) {
         op = gIter->data;
         blocked_ops = g_list_remove(blocked_ops, op);
     }
     g_list_free(executed_ops);
 
     processing_blocked_ops = FALSE;
 }
 
 /*!
  * \internal
  * \brief Execute a meta-data action appropriately to standard
  *
  * \param[in] op  Meta-data action to execute
  *
  * \return Standard Pacemaker return code
  */
 static int
 execute_metadata_action(svc_action_t *op)
 {
     const char *class = op->standard;
 
     if (op->agent == NULL) {
         crm_err("meta-data requested without specifying agent");
         services__set_result(op, services__generic_error(op),
                              PCMK_EXEC_ERROR_FATAL, "Agent not specified");
         return EINVAL;
     }
 
     if (class == NULL) {
         crm_err("meta-data requested for agent %s without specifying class",
                 op->agent);
         services__set_result(op, services__generic_error(op),
                              PCMK_EXEC_ERROR_FATAL,
                              "Agent standard not specified");
         return EINVAL;
     }
 
     if (!strcmp(class, PCMK_RESOURCE_CLASS_SERVICE)) {
         class = resources_find_service_class(op->agent);
     }
     if (class == NULL) {
         crm_err("meta-data requested for %s, but could not determine class",
                 op->agent);
         services__set_result(op, services__generic_error(op),
                              PCMK_EXEC_ERROR_HARD,
                              "Agent standard could not be determined");
         return EINVAL;
     }
 
     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) {
         return pcmk_legacy2rc(services__get_lsb_metadata(op->agent,
                                                          &op->stdout_data));
     }
 
 #if SUPPORT_NAGIOS
     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
         return pcmk_legacy2rc(services__get_nagios_metadata(op->agent,
                                                             &op->stdout_data));
     }
 #endif
 
     return execute_action(op);
 }
 
 gboolean
 services_action_sync(svc_action_t * op)
 {
     gboolean rc = TRUE;
 
     if (op == NULL) {
         crm_trace("No operation to execute");
         return FALSE;
     }
 
     op->synchronous = true;
 
     if (pcmk__str_eq(op->action, "meta-data", pcmk__str_casei)) {
         /* Synchronous meta-data operations are handled specially. Since most
          * resource classes don't provide any meta-data, it has to be
          * synthesized from available information about the agent.
          *
          * services_action_async() doesn't treat meta-data actions specially, so
          * it will result in an error for classes that don't support the action.
          */
         rc = (execute_metadata_action(op) == pcmk_rc_ok);
     } else {
         rc = (execute_action(op) == pcmk_rc_ok);
     }
     crm_trace(" > " PCMK__OP_FMT ": %s = %d",
               op->rsc, op->action, op->interval_ms, op->opaque->exec, op->rc);
     if (op->stdout_data) {
         crm_trace(" >  stdout: %s", op->stdout_data);
     }
     if (op->stderr_data) {
         crm_trace(" >  stderr: %s", op->stderr_data);
     }
     return rc;
 }
 
 GList *
 get_directory_list(const char *root, gboolean files, gboolean executable)
 {
     return services_os_get_directory_list(root, files, executable);
 }
 
 GList *
 resources_list_standards(void)
 {
     GList *standards = NULL;
 
     standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_OCF));
     standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_LSB));
     standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_SERVICE));
 
 #if SUPPORT_SYSTEMD
     {
         GList *agents = systemd_unit_listall();
 
         if (agents != NULL) {
             standards = g_list_append(standards,
                                       strdup(PCMK_RESOURCE_CLASS_SYSTEMD));
             g_list_free_full(agents, free);
         }
     }
 #endif
 
 #if SUPPORT_UPSTART
     {
         GList *agents = upstart_job_listall();
 
         if (agents != NULL) {
             standards = g_list_append(standards,
                                       strdup(PCMK_RESOURCE_CLASS_UPSTART));
             g_list_free_full(agents, free);
         }
     }
 #endif
 
 #if SUPPORT_NAGIOS
     {
         GList *agents = services__list_nagios_agents();
 
         if (agents != NULL) {
             standards = g_list_append(standards,
                                       strdup(PCMK_RESOURCE_CLASS_NAGIOS));
             g_list_free_full(agents, free);
         }
     }
 #endif
 
     return standards;
 }
 
 GList *
 resources_list_providers(const char *standard)
 {
     if (pcmk_is_set(pcmk_get_ra_caps(standard), pcmk_ra_cap_provider)) {
         return resources_os_list_ocf_providers();
     }
 
     return NULL;
 }
 
 GList *
 resources_list_agents(const char *standard, const char *provider)
 {
     if ((standard == NULL)
         || (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0)) {
 
         GList *tmp1;
         GList *tmp2;
         GList *result = services__list_lsb_agents();
 
         if (standard == NULL) {
             tmp1 = result;
             tmp2 = resources_os_list_ocf_agents(NULL);
             if (tmp2) {
                 result = g_list_concat(tmp1, tmp2);
             }
         }
 #if SUPPORT_SYSTEMD
         tmp1 = result;
         tmp2 = systemd_unit_listall();
         if (tmp2) {
             result = g_list_concat(tmp1, tmp2);
         }
 #endif
 
 #if SUPPORT_UPSTART
         tmp1 = result;
         tmp2 = upstart_job_listall();
         if (tmp2) {
             result = g_list_concat(tmp1, tmp2);
         }
 #endif
 
         return result;
 
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
         return resources_os_list_ocf_agents(provider);
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
         return services__list_lsb_agents();
 #if SUPPORT_SYSTEMD
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
         return systemd_unit_listall();
 #endif
 #if SUPPORT_UPSTART
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
         return upstart_job_listall();
 #endif
 #if SUPPORT_NAGIOS
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
         return services__list_nagios_agents();
 #endif
     }
 
     return NULL;
 }
 
 gboolean
 resources_agent_exists(const char *standard, const char *provider, const char *agent)
 {
     GList *standards = NULL;
     GList *providers = NULL;
     GList *iter = NULL;
     gboolean rc = FALSE;
     gboolean has_providers = FALSE;
 
     standards = resources_list_standards();
     for (iter = standards; iter != NULL; iter = iter->next) {
         if (pcmk__str_eq(iter->data, standard, pcmk__str_none)) {
             rc = TRUE;
             break;
         }
     }
 
     if (rc == FALSE) {
         goto done;
     }
 
     rc = FALSE;
 
     has_providers = pcmk_is_set(pcmk_get_ra_caps(standard), pcmk_ra_cap_provider);
     if (has_providers == TRUE && provider != NULL) {
         providers = resources_list_providers(standard);
         for (iter = providers; iter != NULL; iter = iter->next) {
             if (pcmk__str_eq(iter->data, provider, pcmk__str_none)) {
                 rc = TRUE;
                 break;
             }
         }
     } else if (has_providers == FALSE && provider == NULL) {
         rc = TRUE;
     }
 
     if (rc == FALSE) {
         goto done;
     }
 
     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
         if (services__lsb_agent_exists(agent)) {
             rc = TRUE;
 #if SUPPORT_SYSTEMD
         } else if (systemd_unit_exists(agent)) {
             rc = TRUE;
 #endif
 
 #if SUPPORT_UPSTART
         } else if (upstart_job_exists(agent)) {
             rc = TRUE;
 #endif
         } else {
             rc = FALSE;
         }
 
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
         rc = services__ocf_agent_exists(provider, agent);
 
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) {
         rc = services__lsb_agent_exists(agent);
 
 #if SUPPORT_SYSTEMD
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
         rc = systemd_unit_exists(agent);
 #endif
 
 #if SUPPORT_UPSTART
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_casei)) {
         rc = upstart_job_exists(agent);
 #endif
 
 #if SUPPORT_NAGIOS
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
         rc = services__nagios_agent_exists(agent);
 #endif
 
     } else {
         rc = FALSE;
     }
 
 done:
     g_list_free(standards);
     g_list_free(providers);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Set the result of an action
  *
  * \param[out] action        Where to set action result
  * \param[in]  agent_status  Exit status to set
  * \param[in]  exec_status   Execution status to set
  * \param[in]  reason        Human-friendly description of event to set
  */
 void
 services__set_result(svc_action_t *action, int agent_status,
                      enum pcmk_exec_status exec_status, const char *reason)
 {
     if (action == NULL) {
         return;
     }
 
     action->rc = agent_status;
     action->status = exec_status;
 
     if (!pcmk__str_eq(action->opaque->exit_reason, reason,
                       pcmk__str_none)) {
         free(action->opaque->exit_reason);
         action->opaque->exit_reason = (reason == NULL)? NULL : strdup(reason);
     }
 }
 
 /*!
  * \internal
  * \brief Set the result of an action to cancelled
  *
  * \param[out] action        Where to set action result
  *
  * \note This sets execution status but leaves the exit status unchanged
  */
 void
 services__set_cancelled(svc_action_t *action)
 {
     if (action != NULL) {
         action->status = PCMK_EXEC_CANCELLED;
         free(action->opaque->exit_reason);
         action->opaque->exit_reason = NULL;
     }
 }
 
 /*!
  * \internal
  * \brief Get the exit reason of an action
  *
  * \param[in] action  Action to check
  *
  * \return Action's exit reason (or NULL if none)
  */
 const char *
 services__exit_reason(svc_action_t *action)
 {
     return action->opaque->exit_reason;
 }
diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c
index 1162287e95..82c7d13d83 100644
--- a/lib/services/services_linux.c
+++ b/lib/services/services_linux.c
@@ -1,1341 +1,1377 @@
 /*
  * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/wait.h>
 #include <errno.h>
 #include <unistd.h>
 #include <dirent.h>
 #include <grp.h>
 #include <string.h>
 #include <sys/time.h>
 #include <sys/resource.h>
 
 #include "crm/crm.h"
 #include "crm/common/mainloop.h"
 #include "crm/services.h"
 #include "crm/services_internal.h"
 
 #include "services_private.h"
 
 static void close_pipe(int fildes[]);
 
 /* We have two alternative ways of handling SIGCHLD when synchronously waiting
  * for spawned processes to complete. Both rely on polling a file descriptor to
  * discover SIGCHLD events.
  *
  * If sys/signalfd.h is available (e.g. on Linux), we call signalfd() to
  * generate the file descriptor. Otherwise, we use the "self-pipe trick"
  * (opening a pipe and writing a byte to it when SIGCHLD is received).
  */
 #ifdef HAVE_SYS_SIGNALFD_H
 
 // signalfd() implementation
 
 #include <sys/signalfd.h>
 
 // Everything needed to manage SIGCHLD handling
 struct sigchld_data_s {
     sigset_t mask;      // Signals to block now (including SIGCHLD)
     sigset_t old_mask;  // Previous set of blocked signals
 };
 
 // Initialize SIGCHLD data and prepare for use
 static bool
 sigchld_setup(struct sigchld_data_s *data)
 {
     sigemptyset(&(data->mask));
     sigaddset(&(data->mask), SIGCHLD);
 
     sigemptyset(&(data->old_mask));
 
     // Block SIGCHLD (saving previous set of blocked signals to restore later)
     if (sigprocmask(SIG_BLOCK, &(data->mask), &(data->old_mask)) < 0) {
         crm_err("Wait for child process completion failed: %s "
                 CRM_XS " source=sigprocmask", pcmk_strerror(errno));
         return false;
     }
     return true;
 }
 
 // Get a file descriptor suitable for polling for SIGCHLD events
 static int
 sigchld_open(struct sigchld_data_s *data)
 {
     int fd;
 
     CRM_CHECK(data != NULL, return -1);
 
     fd = signalfd(-1, &(data->mask), SFD_NONBLOCK);
     if (fd < 0) {
         crm_err("Wait for child process completion failed: %s "
                 CRM_XS " source=signalfd", pcmk_strerror(errno));
     }
     return fd;
 }
 
 // Close a file descriptor returned by sigchld_open()
 static void
 sigchld_close(int fd)
 {
     if (fd > 0) {
         close(fd);
     }
 }
 
 // Return true if SIGCHLD was received from polled fd
 static bool
 sigchld_received(int fd)
 {
     struct signalfd_siginfo fdsi;
     ssize_t s;
 
     if (fd < 0) {
         return false;
     }
     s = read(fd, &fdsi, sizeof(struct signalfd_siginfo));
     if (s != sizeof(struct signalfd_siginfo)) {
         crm_err("Wait for child process completion failed: %s "
                 CRM_XS " source=read", pcmk_strerror(errno));
 
     } else if (fdsi.ssi_signo == SIGCHLD) {
         return true;
     }
     return false;
 }
 
 // Do anything needed after done waiting for SIGCHLD
 static void
 sigchld_cleanup(struct sigchld_data_s *data)
 {
     // Restore the original set of blocked signals
     if ((sigismember(&(data->old_mask), SIGCHLD) == 0)
         && (sigprocmask(SIG_UNBLOCK, &(data->mask), NULL) < 0)) {
         crm_warn("Could not clean up after child process completion: %s",
                  pcmk_strerror(errno));
     }
 }
 
 #else // HAVE_SYS_SIGNALFD_H not defined
 
 // Self-pipe implementation (see above for function descriptions)
 
 struct sigchld_data_s {
     int pipe_fd[2];             // Pipe file descriptors
     struct sigaction sa;        // Signal handling info (with SIGCHLD)
     struct sigaction old_sa;    // Previous signal handling info
 };
 
 // We need a global to use in the signal handler
 volatile struct sigchld_data_s *last_sigchld_data = NULL;
 
 static void
 sigchld_handler()
 {
     // We received a SIGCHLD, so trigger pipe polling
     if ((last_sigchld_data != NULL)
         && (last_sigchld_data->pipe_fd[1] >= 0)
         && (write(last_sigchld_data->pipe_fd[1], "", 1) == -1)) {
         crm_err("Wait for child process completion failed: %s "
                 CRM_XS " source=write", pcmk_strerror(errno));
     }
 }
 
 static bool
 sigchld_setup(struct sigchld_data_s *data)
 {
     int rc;
 
     data->pipe_fd[0] = data->pipe_fd[1] = -1;
 
     if (pipe(data->pipe_fd) == -1) {
         crm_err("Wait for child process completion failed: %s "
                 CRM_XS " source=pipe", pcmk_strerror(errno));
         return false;
     }
 
     rc = pcmk__set_nonblocking(data->pipe_fd[0]);
     if (rc != pcmk_rc_ok) {
         crm_warn("Could not set pipe input non-blocking: %s " CRM_XS " rc=%d",
                  pcmk_rc_str(rc), rc);
     }
     rc = pcmk__set_nonblocking(data->pipe_fd[1]);
     if (rc != pcmk_rc_ok) {
         crm_warn("Could not set pipe output non-blocking: %s " CRM_XS " rc=%d",
                  pcmk_rc_str(rc), rc);
     }
 
     // Set SIGCHLD handler
     data->sa.sa_handler = sigchld_handler;
     data->sa.sa_flags = 0;
     sigemptyset(&(data->sa.sa_mask));
     if (sigaction(SIGCHLD, &(data->sa), &(data->old_sa)) < 0) {
         crm_err("Wait for child process completion failed: %s "
                 CRM_XS " source=sigaction", pcmk_strerror(errno));
     }
 
     // Remember data for use in signal handler
     last_sigchld_data = data;
     return true;
 }
 
 static int
 sigchld_open(struct sigchld_data_s *data)
 {
     CRM_CHECK(data != NULL, return -1);
     return data->pipe_fd[0];
 }
 
 static void
 sigchld_close(int fd)
 {
     // Pipe will be closed in sigchld_cleanup()
     return;
 }
 
 static bool
 sigchld_received(int fd)
 {
     char ch;
 
     if (fd < 0) {
         return false;
     }
 
     // Clear out the self-pipe
     while (read(fd, &ch, 1) == 1) /*omit*/;
     return true;
 }
 
 static void
 sigchld_cleanup(struct sigchld_data_s *data)
 {
     // Restore the previous SIGCHLD handler
     if (sigaction(SIGCHLD, &(data->old_sa), NULL) < 0) {
         crm_warn("Could not clean up after child process completion: %s",
                  pcmk_strerror(errno));
     }
 
     close_pipe(data->pipe_fd);
 }
 
 #endif
 
 /*!
  * \internal
  * \brief Close the two file descriptors of a pipe
  *
  * \param[in] fildes  Array of file descriptors opened by pipe()
  */
 static void
 close_pipe(int fildes[])
 {
     if (fildes[0] >= 0) {
         close(fildes[0]);
         fildes[0] = -1;
     }
     if (fildes[1] >= 0) {
         close(fildes[1]);
         fildes[1] = -1;
     }
 }
 
 static gboolean
 svc_read_output(int fd, svc_action_t * op, bool is_stderr)
 {
     char *data = NULL;
     int rc = 0, len = 0;
     char buf[500];
     static const size_t buf_read_len = sizeof(buf) - 1;
 
 
     if (fd < 0) {
         crm_trace("No fd for %s", op->id);
         return FALSE;
     }
 
     if (is_stderr && op->stderr_data) {
         len = strlen(op->stderr_data);
         data = op->stderr_data;
         crm_trace("Reading %s stderr into offset %d", op->id, len);
 
     } else if (is_stderr == FALSE && op->stdout_data) {
         len = strlen(op->stdout_data);
         data = op->stdout_data;
         crm_trace("Reading %s stdout into offset %d", op->id, len);
 
     } else {
         crm_trace("Reading %s %s into offset %d", op->id, is_stderr?"stderr":"stdout", len);
     }
 
     do {
         rc = read(fd, buf, buf_read_len);
         if (rc > 0) {
             buf[rc] = 0;
             crm_trace("Got %d chars: %.80s", rc, buf);
             data = pcmk__realloc(data, len + rc + 1);
             len += sprintf(data + len, "%s", buf);
 
         } else if (errno != EINTR) {
             /* error or EOF
              * Cleanup happens in pipe_done()
              */
             rc = FALSE;
             break;
         }
 
     } while (rc == buf_read_len || rc < 0);
 
     if (is_stderr) {
         op->stderr_data = data;
     } else {
         op->stdout_data = data;
     }
 
     return rc;
 }
 
 static int
 dispatch_stdout(gpointer userdata)
 {
     svc_action_t *op = (svc_action_t *) userdata;
 
     return svc_read_output(op->opaque->stdout_fd, op, FALSE);
 }
 
 static int
 dispatch_stderr(gpointer userdata)
 {
     svc_action_t *op = (svc_action_t *) userdata;
 
     return svc_read_output(op->opaque->stderr_fd, op, TRUE);
 }
 
 static void
 pipe_out_done(gpointer user_data)
 {
     svc_action_t *op = (svc_action_t *) user_data;
 
     crm_trace("%p", op);
 
     op->opaque->stdout_gsource = NULL;
     if (op->opaque->stdout_fd > STDOUT_FILENO) {
         close(op->opaque->stdout_fd);
     }
     op->opaque->stdout_fd = -1;
 }
 
 static void
 pipe_err_done(gpointer user_data)
 {
     svc_action_t *op = (svc_action_t *) user_data;
 
     op->opaque->stderr_gsource = NULL;
     if (op->opaque->stderr_fd > STDERR_FILENO) {
         close(op->opaque->stderr_fd);
     }
     op->opaque->stderr_fd = -1;
 }
 
 static struct mainloop_fd_callbacks stdout_callbacks = {
     .dispatch = dispatch_stdout,
     .destroy = pipe_out_done,
 };
 
 static struct mainloop_fd_callbacks stderr_callbacks = {
     .dispatch = dispatch_stderr,
     .destroy = pipe_err_done,
 };
 
 static void
 set_ocf_env(const char *key, const char *value, gpointer user_data)
 {
     if (setenv(key, value, 1) != 0) {
         crm_perror(LOG_ERR, "setenv failed for key:%s and value:%s", key, value);
     }
 }
 
 static void
 set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data)
 {
     char buffer[500];
 
     snprintf(buffer, sizeof(buffer), strcmp(key, "OCF_CHECK_LEVEL") != 0 ? "OCF_RESKEY_%s" : "%s", (char *)key);
     set_ocf_env(buffer, value, user_data);
 }
 
 static void
 set_alert_env(gpointer key, gpointer value, gpointer user_data)
 {
     int rc;
 
     if (value != NULL) {
         rc = setenv(key, value, 1);
     } else {
         rc = unsetenv(key);
     }
 
     if (rc < 0) {
         crm_perror(LOG_ERR, "setenv %s=%s",
                   (char*)key, (value? (char*)value : ""));
     } else {
         crm_trace("setenv %s=%s", (char*)key, (value? (char*)value : ""));
     }
 }
 
 /*!
  * \internal
  * \brief Add environment variables suitable for an action
  *
  * \param[in] op  Action to use
  */
 static void
 add_action_env_vars(const svc_action_t *op)
 {
     void (*env_setter)(gpointer, gpointer, gpointer) = NULL;
     if (op->agent == NULL) {
         env_setter = set_alert_env;  /* we deal with alert handler */
 
     } else if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
         env_setter = set_ocf_env_with_prefix;
     }
 
     if (env_setter != NULL && op->params != NULL) {
         g_hash_table_foreach(op->params, env_setter, NULL);
     }
 
     if (env_setter == NULL || env_setter == set_alert_env) {
         return;
     }
 
     set_ocf_env("OCF_RA_VERSION_MAJOR", PCMK_OCF_MAJOR_VERSION, NULL);
     set_ocf_env("OCF_RA_VERSION_MINOR", PCMK_OCF_MINOR_VERSION, NULL);
     set_ocf_env("OCF_ROOT", OCF_ROOT_DIR, NULL);
     set_ocf_env("OCF_EXIT_REASON_PREFIX", PCMK_OCF_REASON_PREFIX, NULL);
 
     if (op->rsc) {
         set_ocf_env("OCF_RESOURCE_INSTANCE", op->rsc, NULL);
     }
 
     if (op->agent != NULL) {
         set_ocf_env("OCF_RESOURCE_TYPE", op->agent, NULL);
     }
 
     /* Notes: this is not added to specification yet. Sept 10,2004 */
     if (op->provider != NULL) {
         set_ocf_env("OCF_RESOURCE_PROVIDER", op->provider, NULL);
     }
 }
 
 static void
 pipe_in_single_parameter(gpointer key, gpointer value, gpointer user_data)
 {
     svc_action_t *op = user_data;
     char *buffer = crm_strdup_printf("%s=%s\n", (char *)key, (char *) value);
     int ret, total = 0, len = strlen(buffer);
 
     do {
         errno = 0;
         ret = write(op->opaque->stdin_fd, buffer + total, len - total);
         if (ret > 0) {
             total += ret;
         }
 
     } while ((errno == EINTR) && (total < len));
     free(buffer);
 }
 
 /*!
  * \internal
  * \brief Pipe parameters in via stdin for action
  *
  * \param[in] op  Action to use
  */
 static void
 pipe_in_action_stdin_parameters(const svc_action_t *op)
 {
     crm_debug("sending args");
     if (op->params) {
         g_hash_table_foreach(op->params, pipe_in_single_parameter, (gpointer) op);
     }
 }
 
 gboolean
 recurring_action_timer(gpointer data)
 {
     svc_action_t *op = data;
 
     crm_debug("Scheduling another invocation of %s", op->id);
 
     /* Clean out the old result */
     free(op->stdout_data);
     op->stdout_data = NULL;
     free(op->stderr_data);
     op->stderr_data = NULL;
     op->opaque->repeat_timer = 0;
 
     services_action_async(op, NULL);
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Finalize handling of an asynchronous operation
  *
  * Given a completed asynchronous operation, cancel or reschedule it as
  * appropriate if recurring, call its callback if registered, stop tracking it,
  * and clean it up.
  *
  * \param[in,out] op  Operation to finalize
  *
  * \return Standard Pacemaker return code
  * \retval EINVAL      Caller supplied NULL or invalid \p op
  * \retval EBUSY       Uncanceled recurring action has only been cleaned up
  * \retval pcmk_rc_ok  Action has been freed
  *
  * \note If the return value is not pcmk_rc_ok, the caller is responsible for
  *       freeing the action.
  */
 int
 services__finalize_async_op(svc_action_t *op)
 {
     CRM_CHECK((op != NULL) && !(op->synchronous), return EINVAL);
 
     if (op->interval_ms != 0) {
         // Recurring operations must be either cancelled or rescheduled
         if (op->cancel) {
             services__set_cancelled(op);
             cancel_recurring_action(op);
         } else {
             op->opaque->repeat_timer = g_timeout_add(op->interval_ms,
                                                      recurring_action_timer,
                                                      (void *) op);
         }
     }
 
     if (op->opaque->callback != NULL) {
         op->opaque->callback(op);
     }
 
     // Stop tracking the operation (as in-flight or blocked)
     op->pid = 0;
     services_untrack_op(op);
 
     if ((op->interval_ms != 0) && !(op->cancel)) {
         // Do not free recurring actions (they will get freed when cancelled)
         services_action_cleanup(op);
         return EBUSY;
     }
 
     services_action_free(op);
     return pcmk_rc_ok;
 }
 
 static void
 close_op_input(svc_action_t *op)
 {
     if (op->opaque->stdin_fd >= 0) {
         close(op->opaque->stdin_fd);
     }
 }
 
 static void
 finish_op_output(svc_action_t *op, bool is_stderr)
 {
     mainloop_io_t **source;
     int fd;
 
     if (is_stderr) {
         source = &(op->opaque->stderr_gsource);
         fd = op->opaque->stderr_fd;
     } else {
         source = &(op->opaque->stdout_gsource);
         fd = op->opaque->stdout_fd;
     }
 
     if (op->synchronous || *source) {
         crm_trace("Finish reading %s[%d] %s",
                   op->id, op->pid, (is_stderr? "stdout" : "stderr"));
         svc_read_output(fd, op, is_stderr);
         if (op->synchronous) {
             close(fd);
         } else {
             mainloop_del_fd(*source);
             *source = NULL;
         }
     }
 }
 
 // Log an operation's stdout and stderr
 static void
 log_op_output(svc_action_t *op)
 {
     char *prefix = crm_strdup_printf("%s[%d] error output", op->id, op->pid);
 
     crm_log_output(LOG_NOTICE, prefix, op->stderr_data);
     strcpy(prefix + strlen(prefix) - strlen("error output"), "output");
     crm_log_output(LOG_DEBUG, prefix, op->stdout_data);
     free(prefix);
 }
 
 // Truncate exit reasons at this many characters
 #define EXIT_REASON_MAX_LEN 128
 
 static void
 parse_exit_reason_from_stderr(svc_action_t *op)
 {
     const char *reason_start = NULL;
     const char *reason_end = NULL;
     const int prefix_len = strlen(PCMK_OCF_REASON_PREFIX);
 
     if ((op->stderr_data == NULL) ||
         // Only OCF agents have exit reasons in stderr
         !pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_none)) {
         return;
     }
 
     // Find the last occurrence of the magic string indicating an exit reason
     for (const char *cur = strstr(op->stderr_data, PCMK_OCF_REASON_PREFIX);
          cur != NULL; cur = strstr(cur, PCMK_OCF_REASON_PREFIX)) {
 
         cur += prefix_len; // Skip over magic string
         reason_start = cur;
     }
 
     if ((reason_start == NULL) || (reason_start[0] == '\n')
         || (reason_start[0] == '\0')) {
         return; // No or empty exit reason
     }
 
     // Exit reason goes to end of line (or end of output)
     reason_end = strchr(reason_start, '\n');
     if (reason_end == NULL) {
         reason_end = reason_start + strlen(reason_start);
     }
 
     // Limit size of exit reason to something reasonable
     if (reason_end > (reason_start + EXIT_REASON_MAX_LEN)) {
         reason_end = reason_start + EXIT_REASON_MAX_LEN;
     }
 
     free(op->opaque->exit_reason);
     op->opaque->exit_reason = strndup(reason_start, reason_end - reason_start);
 }
 
 /*!
  * \internal
  * \brief Process the completion of an asynchronous child process
  *
  * \param[in] p         Child process that completed
  * \param[in] pid       Process ID of child
  * \param[in] core      (unused)
  * \param[in] signo     Signal that interrupted child, if any
  * \param[in] exitcode  Exit status of child process
  */
 static void
 async_action_complete(mainloop_child_t *p, pid_t pid, int core, int signo,
                       int exitcode)
 {
     svc_action_t *op = mainloop_child_userdata(p);
 
     mainloop_clear_child_userdata(p);
     CRM_CHECK(op->pid == pid,
               services__set_result(op, services__generic_error(op),
                                    PCMK_EXEC_ERROR, "Bug in mainloop handling");
               return);
 
     /* Depending on the priority the mainloop gives the stdout and stderr
      * file descriptors, this function could be called before everything has
      * been read from them, so force a final read now.
      */
     finish_op_output(op, true);
     finish_op_output(op, false);
 
     close_op_input(op);
 
     if (signo == 0) {
         crm_debug("%s[%d] exited with status %d", op->id, op->pid, exitcode);
         services__set_result(op, exitcode, PCMK_EXEC_DONE, NULL);
         log_op_output(op);
         parse_exit_reason_from_stderr(op);
 
     } else if (mainloop_child_timeout(p)) {
         crm_warn("%s[%d] timed out after %dms", op->id, op->pid, op->timeout);
         services__set_result(op, services__generic_error(op), PCMK_EXEC_TIMEOUT,
                              "Process did not exit within specified timeout");
 
     } else if (op->cancel) {
         /* If an in-flight recurring operation was killed because it was
          * cancelled, don't treat that as a failure.
          */
-        crm_info("%s[%d] terminated with signal: %s " CRM_XS " (%d)",
-                 op->id, op->pid, strsignal(signo), signo);
+        crm_info("%s[%d] terminated with signal %d (%s)",
+                 op->id, op->pid, signo, strsignal(signo));
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_CANCELLED, NULL);
 
     } else {
-        crm_warn("%s[%d] terminated with signal: %s " CRM_XS " (%d)",
-                 op->id, op->pid, strsignal(signo), signo);
+        crm_warn("%s[%d] terminated with signal %d (%s)",
+                 op->id, op->pid, signo, strsignal(signo));
         services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                              "Process interrupted by signal");
     }
 
     services__finalize_async_op(op);
 }
 
 /*!
  * \internal
  * \brief Return agent standard's exit status for "generic error"
  *
  * When returning an internal error for an action, a value that is appropriate
  * to the action's agent standard must be used. This function returns a value
  * appropriate for errors in general.
  *
  * \param[in] op  Action that error is for
  *
  * \return Exit status appropriate to agent standard
  * \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
  */
 int
 services__generic_error(svc_action_t *op)
 {
     if ((op == NULL) || (op->standard == NULL)) {
         return PCMK_OCF_UNKNOWN_ERROR;
     }
 
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
         && pcmk__str_eq(op->action, "status", pcmk__str_casei)) {
 
         return PCMK_LSB_STATUS_UNKNOWN;
     }
 
 #if SUPPORT_NAGIOS
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
         return NAGIOS_STATE_UNKNOWN;
     }
 #endif
 
     return PCMK_OCF_UNKNOWN_ERROR;
 }
 
 /*!
  * \internal
  * \brief Return agent standard's exit status for "not installed"
  *
  * When returning an internal error for an action, a value that is appropriate
  * to the action's agent standard must be used. This function returns a value
  * appropriate for "not installed" errors.
  *
  * \param[in] op  Action that error is for
  *
  * \return Exit status appropriate to agent standard
  * \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
  */
 int
 services__not_installed_error(svc_action_t *op)
 {
     if ((op == NULL) || (op->standard == NULL)) {
         return PCMK_OCF_UNKNOWN_ERROR;
     }
 
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
         && pcmk__str_eq(op->action, "status", pcmk__str_casei)) {
 
         return PCMK_LSB_STATUS_NOT_INSTALLED;
     }
 
 #if SUPPORT_NAGIOS
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
-        return NAGIOS_NOT_INSTALLED;
+        return NAGIOS_STATE_UNKNOWN;
     }
 #endif
 
     return PCMK_OCF_NOT_INSTALLED;
 }
 
 /*!
  * \internal
  * \brief Return agent standard's exit status for "insufficient privileges"
  *
  * When returning an internal error for an action, a value that is appropriate
  * to the action's agent standard must be used. This function returns a value
  * appropriate for "insufficient privileges" errors.
  *
  * \param[in] op  Action that error is for
  *
  * \return Exit status appropriate to agent standard
  * \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
  */
 int
 services__authorization_error(svc_action_t *op)
 {
     if ((op == NULL) || (op->standard == NULL)) {
         return PCMK_OCF_UNKNOWN_ERROR;
     }
 
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
         && pcmk__str_eq(op->action, "status", pcmk__str_casei)) {
 
         return PCMK_LSB_STATUS_INSUFFICIENT_PRIV;
     }
 
 #if SUPPORT_NAGIOS
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
         return NAGIOS_INSUFFICIENT_PRIV;
     }
 #endif
 
     return PCMK_OCF_INSUFFICIENT_PRIV;
 }
 
+/*!
+ * \internal
+ * \brief Return agent standard's exit status for "not configured"
+ *
+ * When returning an internal error for an action, a value that is appropriate
+ * to the action's agent standard must be used. This function returns a value
+ * appropriate for "not configured" errors.
+ *
+ * \param[in] op  Action that error is for
+ *
+ * \return Exit status appropriate to agent standard
+ * \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
+ */
+int
+services__configuration_error(svc_action_t *op)
+{
+    if ((op == NULL) || (op->standard == NULL)) {
+        return PCMK_OCF_UNKNOWN_ERROR;
+    }
+
+    if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
+        && pcmk__str_eq(op->action, "status", pcmk__str_casei)) {
+
+        return PCMK_LSB_NOT_CONFIGURED;
+    }
+
+#if SUPPORT_NAGIOS
+    if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
+        return NAGIOS_STATE_UNKNOWN;
+    }
+#endif
+
+    return PCMK_OCF_NOT_CONFIGURED;
+}
+
+
 /*!
  * \internal
  * \brief Set operation rc and status per errno from stat(), fork() or execvp()
  *
  * \param[in,out] op     Operation to set rc and status for
  * \param[in]     error  Value of errno after system call
  *
  * \return void
  */
 void
 services__handle_exec_error(svc_action_t * op, int error)
 {
     switch (error) {   /* see execve(2), stat(2) and fork(2) */
         case ENOENT:   /* No such file or directory */
         case EISDIR:   /* Is a directory */
         case ENOTDIR:  /* Path component is not a directory */
         case EINVAL:   /* Invalid executable format */
         case ENOEXEC:  /* Invalid executable format */
             services__set_result(op, services__not_installed_error(op),
                                  PCMK_EXEC_NOT_INSTALLED, pcmk_rc_str(error));
             break;
         case EACCES:   /* permission denied (various errors) */
         case EPERM:    /* permission denied (various errors) */
             services__set_result(op, services__authorization_error(op),
                                  PCMK_EXEC_ERROR, pcmk_rc_str(error));
             break;
         default:
             services__set_result(op, services__generic_error(op),
                                  PCMK_EXEC_ERROR, pcmk_rc_str(error));
     }
 }
 
 static void
 action_launch_child(svc_action_t *op)
 {
     /* SIGPIPE is ignored (which is different from signal blocking) by the gnutls library.
      * Depending on the libqb version in use, libqb may set SIGPIPE to be ignored as well. 
      * We do not want this to be inherited by the child process. By resetting this the signal
      * to the default behavior, we avoid some potential odd problems that occur during OCF
      * scripts when SIGPIPE is ignored by the environment. */
     signal(SIGPIPE, SIG_DFL);
 
 #if defined(HAVE_SCHED_SETSCHEDULER)
     if (sched_getscheduler(0) != SCHED_OTHER) {
         struct sched_param sp;
 
         memset(&sp, 0, sizeof(sp));
         sp.sched_priority = 0;
 
         if (sched_setscheduler(0, SCHED_OTHER, &sp) == -1) {
             crm_perror(LOG_ERR, "Could not reset scheduling policy to SCHED_OTHER for %s", op->id);
         }
     }
 #endif
     if (setpriority(PRIO_PROCESS, 0, 0) == -1) {
         crm_perror(LOG_ERR, "Could not reset process priority to 0 for %s", op->id);
     }
 
     /* Man: The call setpgrp() is equivalent to setpgid(0,0)
      * _and_ compiles on BSD variants too
      * need to investigate if it works the same too.
      */
     setpgid(0, 0);
 
     pcmk__close_fds_in_child(false);
 
 #if SUPPORT_CIBSECRETS
     if (pcmk__substitute_secrets(op->rsc, op->params) != pcmk_rc_ok) {
         /* replacing secrets failed! */
         if (pcmk__str_eq(op->action, "stop", pcmk__str_casei)) {
             /* don't fail on stop! */
             crm_info("proceeding with the stop operation for %s", op->rsc);
 
         } else {
             crm_err("failed to get secrets for %s, "
                     "considering resource not configured", op->rsc);
-            _exit(PCMK_OCF_NOT_CONFIGURED);
+            _exit(services__configuration_error(op));
         }
     }
 #endif
 
     add_action_env_vars(op);
 
     /* Become the desired user */
     if (op->opaque->uid && (geteuid() == 0)) {
 
         // If requested, set effective group
         if (op->opaque->gid && (setgid(op->opaque->gid) < 0)) {
             crm_perror(LOG_ERR, "Could not set child group to %d", op->opaque->gid);
-            _exit(PCMK_OCF_NOT_CONFIGURED);
+            _exit(services__authorization_error(op));
         }
 
         // Erase supplementary group list
         // (We could do initgroups() if we kept a copy of the username)
         if (setgroups(0, NULL) < 0) {
             crm_perror(LOG_ERR, "Could not set child groups");
-            _exit(PCMK_OCF_NOT_CONFIGURED);
+            _exit(services__authorization_error(op));
         }
 
         // Set effective user
         if (setuid(op->opaque->uid) < 0) {
             crm_perror(LOG_ERR, "setting user to %d", op->opaque->uid);
-            _exit(PCMK_OCF_NOT_CONFIGURED);
+            _exit(services__authorization_error(op));
         }
     }
 
     /* execute the RA */
     execvp(op->opaque->exec, op->opaque->args);
 
     /* Most cases should have been already handled by stat() */
     services__handle_exec_error(op, errno);
 
     _exit(op->rc);
 }
 
 /*!
  * \internal
  * \brief Wait for synchronous action to complete, and set its result
  *
  * \param[in] op    Action to wait for
  * \param[in] data  Child signal data
  */
 static void
 wait_for_sync_result(svc_action_t *op, struct sigchld_data_s *data)
 {
     int status = 0;
     int timeout = op->timeout;
     time_t start = time(NULL);
     struct pollfd fds[3];
     int wait_rc = 0;
     const char *wait_reason = NULL;
 
     fds[0].fd = op->opaque->stdout_fd;
     fds[0].events = POLLIN;
     fds[0].revents = 0;
 
     fds[1].fd = op->opaque->stderr_fd;
     fds[1].events = POLLIN;
     fds[1].revents = 0;
 
     fds[2].fd = sigchld_open(data);
     fds[2].events = POLLIN;
     fds[2].revents = 0;
 
     crm_trace("Waiting for %s[%d]", op->id, op->pid);
     do {
         int poll_rc = poll(fds, 3, timeout);
 
         wait_reason = NULL;
 
         if (poll_rc > 0) {
             if (fds[0].revents & POLLIN) {
                 svc_read_output(op->opaque->stdout_fd, op, FALSE);
             }
 
             if (fds[1].revents & POLLIN) {
                 svc_read_output(op->opaque->stderr_fd, op, TRUE);
             }
 
             if ((fds[2].revents & POLLIN) && sigchld_received(fds[2].fd)) {
                 wait_rc = waitpid(op->pid, &status, WNOHANG);
 
                 if ((wait_rc > 0) || ((wait_rc < 0) && (errno == ECHILD))) {
                     // Child process exited or doesn't exist
                     break;
 
                 } else if (wait_rc < 0) {
                     wait_reason = pcmk_rc_str(errno);
                     crm_warn("Wait for completion of %s[%d] failed: %s "
                              CRM_XS " source=waitpid",
                              op->id, op->pid, wait_reason);
                     wait_rc = 0; // Act as if process is still running
                 }
             }
 
         } else if (poll_rc == 0) {
             // Poll timed out with no descriptors ready
             timeout = 0;
             break;
 
         } else if ((poll_rc < 0) && (errno != EINTR)) {
             wait_reason = pcmk_rc_str(errno);
             crm_err("Wait for completion of %s[%d] failed: %s "
                     CRM_XS " source=poll", op->id, op->pid, wait_reason);
             break;
         }
 
         timeout = op->timeout - (time(NULL) - start) * 1000;
 
     } while ((op->timeout < 0 || timeout > 0));
 
     crm_trace("Stopped waiting for %s[%d]", op->id, op->pid);
     finish_op_output(op, true);
     finish_op_output(op, false);
     close_op_input(op);
     sigchld_close(fds[2].fd);
 
     if (wait_rc <= 0) {
 
         if ((op->timeout > 0) && (timeout <= 0)) {
             services__set_result(op, services__generic_error(op),
                                  PCMK_EXEC_TIMEOUT,
                                  "Process did not exit within specified timeout");
             crm_warn("%s[%d] timed out after %dms",
                      op->id, op->pid, op->timeout);
 
         } else {
             services__set_result(op, services__generic_error(op),
                                  PCMK_EXEC_ERROR, wait_reason);
         }
 
         /* If only child hasn't been successfully waited for, yet.
            This is to limit killing wrong target a bit more. */
         if ((wait_rc == 0) && (waitpid(op->pid, &status, WNOHANG) == 0)) {
             if (kill(op->pid, SIGKILL)) {
                 crm_warn("Could not kill rogue child %s[%d]: %s",
                          op->id, op->pid, pcmk_strerror(errno));
             }
             /* Safe to skip WNOHANG here as we sent non-ignorable signal. */
             while ((waitpid(op->pid, &status, 0) == (pid_t) -1)
                    && (errno == EINTR)) {
                 /* keep waiting */;
             }
         }
 
     } else if (WIFEXITED(status)) {
         services__set_result(op, WEXITSTATUS(status), PCMK_EXEC_DONE, NULL);
         parse_exit_reason_from_stderr(op);
         crm_info("%s[%d] exited with status %d", op->id, op->pid, op->rc);
 
     } else if (WIFSIGNALED(status)) {
         int signo = WTERMSIG(status);
 
         services__set_result(op, services__generic_error(op), PCMK_EXEC_ERROR,
                              "Process interrupted by signal");
-        crm_err("%s[%d] terminated with signal: %s " CRM_XS " (%d)",
-                op->id, op->pid, strsignal(signo), signo);
+        crm_err("%s[%d] terminated with signal %d (%s)",
+                op->id, op->pid, signo, strsignal(signo));
 
 #ifdef WCOREDUMP
         if (WCOREDUMP(status)) {
             crm_warn("%s[%d] dumped core", op->id, op->pid);
         }
 #endif
 
     } else {
         // Shouldn't be possible to get here
         services__set_result(op, services__generic_error(op), PCMK_EXEC_ERROR,
                              "Unable to wait for child to complete");
     }
 }
 
 /*!
  * \internal
  * \brief Execute an action whose standard uses executable files
  *
  * \param[in] op  Action to execute
  *
  * \return Standard Pacemaker return value
  * \retval EBUSY          Recurring operation could not be initiated
  * \retval pcmk_rc_error  Synchronous action failed
  * \retval pcmk_rc_ok     Synchronous action succeeded, or asynchronous action
  *                        should not be freed (because it already was or is
  *                        pending)
  *
  * \note If the return value for an asynchronous action is not pcmk_rc_ok, the
  *       caller is responsible for freeing the action.
  */
 int
 services__execute_file(svc_action_t *op)
 {
     int stdout_fd[2];
     int stderr_fd[2];
     int stdin_fd[2] = {-1, -1};
     int rc;
     struct stat st;
     struct sigchld_data_s data;
 
     // Catch common failure conditions early
     if (stat(op->opaque->exec, &st) != 0) {
         rc = errno;
         crm_warn("Cannot execute '%s': %s " CRM_XS " stat rc=%d",
                  op->opaque->exec, pcmk_strerror(rc), rc);
         services__handle_exec_error(op, rc);
         goto done;
     }
 
     if (pipe(stdout_fd) < 0) {
         rc = errno;
         crm_err("Cannot execute '%s': %s " CRM_XS " pipe(stdout) rc=%d",
                 op->opaque->exec, pcmk_strerror(rc), rc);
         services__handle_exec_error(op, rc);
         goto done;
     }
 
     if (pipe(stderr_fd) < 0) {
         rc = errno;
 
         close_pipe(stdout_fd);
 
         crm_err("Cannot execute '%s': %s " CRM_XS " pipe(stderr) rc=%d",
                 op->opaque->exec, pcmk_strerror(rc), rc);
         services__handle_exec_error(op, rc);
         goto done;
     }
 
     if (pcmk_is_set(pcmk_get_ra_caps(op->standard), pcmk_ra_cap_stdin)) {
         if (pipe(stdin_fd) < 0) {
             rc = errno;
 
             close_pipe(stdout_fd);
             close_pipe(stderr_fd);
 
             crm_err("Cannot execute '%s': %s " CRM_XS " pipe(stdin) rc=%d",
                     op->opaque->exec, pcmk_strerror(rc), rc);
             services__handle_exec_error(op, rc);
             goto done;
         }
     }
 
     if (op->synchronous && !sigchld_setup(&data)) {
         close_pipe(stdin_fd);
         close_pipe(stdout_fd);
         close_pipe(stderr_fd);
         sigchld_cleanup(&data);
         services__set_result(op, services__generic_error(op), PCMK_EXEC_ERROR,
                              "Could not manage signals for child process");
         goto done;
     }
 
     op->pid = fork();
     switch (op->pid) {
         case -1:
             rc = errno;
             close_pipe(stdin_fd);
             close_pipe(stdout_fd);
             close_pipe(stderr_fd);
 
             crm_err("Cannot execute '%s': %s " CRM_XS " fork rc=%d",
                     op->opaque->exec, pcmk_strerror(rc), rc);
             services__handle_exec_error(op, rc);
             if (op->synchronous) {
                 sigchld_cleanup(&data);
             }
             goto done;
             break;
 
         case 0:                /* Child */
             close(stdout_fd[0]);
             close(stderr_fd[0]);
             if (stdin_fd[1] >= 0) {
                 close(stdin_fd[1]);
             }
             if (STDOUT_FILENO != stdout_fd[1]) {
                 if (dup2(stdout_fd[1], STDOUT_FILENO) != STDOUT_FILENO) {
                     crm_warn("Can't redirect output from '%s': %s "
                              CRM_XS " errno=%d",
                              op->opaque->exec, pcmk_strerror(errno), errno);
                 }
                 close(stdout_fd[1]);
             }
             if (STDERR_FILENO != stderr_fd[1]) {
                 if (dup2(stderr_fd[1], STDERR_FILENO) != STDERR_FILENO) {
                     crm_warn("Can't redirect error output from '%s': %s "
                              CRM_XS " errno=%d",
                              op->opaque->exec, pcmk_strerror(errno), errno);
                 }
                 close(stderr_fd[1]);
             }
             if ((stdin_fd[0] >= 0) &&
                 (STDIN_FILENO != stdin_fd[0])) {
                 if (dup2(stdin_fd[0], STDIN_FILENO) != STDIN_FILENO) {
                     crm_warn("Can't redirect input to '%s': %s "
                              CRM_XS " errno=%d",
                              op->opaque->exec, pcmk_strerror(errno), errno);
                 }
                 close(stdin_fd[0]);
             }
 
             if (op->synchronous) {
                 sigchld_cleanup(&data);
             }
 
             action_launch_child(op);
             CRM_ASSERT(0);  /* action_launch_child is effectively noreturn */
     }
 
     /* Only the parent reaches here */
     close(stdout_fd[1]);
     close(stderr_fd[1]);
     if (stdin_fd[0] >= 0) {
         close(stdin_fd[0]);
     }
 
     op->opaque->stdout_fd = stdout_fd[0];
     rc = pcmk__set_nonblocking(op->opaque->stdout_fd);
     if (rc != pcmk_rc_ok) {
         crm_warn("Could not set '%s' output non-blocking: %s "
                  CRM_XS " rc=%d",
                  op->opaque->exec, pcmk_rc_str(rc), rc);
     }
 
     op->opaque->stderr_fd = stderr_fd[0];
     rc = pcmk__set_nonblocking(op->opaque->stderr_fd);
     if (rc != pcmk_rc_ok) {
         crm_warn("Could not set '%s' error output non-blocking: %s "
                  CRM_XS " rc=%d",
                  op->opaque->exec, pcmk_rc_str(rc), rc);
     }
 
     op->opaque->stdin_fd = stdin_fd[1];
     if (op->opaque->stdin_fd >= 0) {
         // using buffer behind non-blocking-fd here - that could be improved
         // as long as no other standard uses stdin_fd assume stonith
         rc = pcmk__set_nonblocking(op->opaque->stdin_fd);
         if (rc != pcmk_rc_ok) {
             crm_warn("Could not set '%s' input non-blocking: %s "
                     CRM_XS " fd=%d,rc=%d", op->opaque->exec,
                     pcmk_rc_str(rc), op->opaque->stdin_fd, rc);
         }
         pipe_in_action_stdin_parameters(op);
         // as long as we are handling parameters directly in here just close
         close(op->opaque->stdin_fd);
         op->opaque->stdin_fd = -1;
     }
 
     // after fds are setup properly and before we plug anything into mainloop
     if (op->opaque->fork_callback) {
         op->opaque->fork_callback(op);
     }
 
     if (op->synchronous) {
         wait_for_sync_result(op, &data);
         sigchld_cleanup(&data);
         goto done;
     }
 
     crm_trace("Waiting async for '%s'[%d]", op->opaque->exec, op->pid);
     mainloop_child_add_with_flags(op->pid, op->timeout, op->id, op,
                                   pcmk_is_set(op->flags, SVC_ACTION_LEAVE_GROUP)? mainloop_leave_pid_group : 0,
                                   async_action_complete);
 
     op->opaque->stdout_gsource = mainloop_add_fd(op->id,
                                                  G_PRIORITY_LOW,
                                                  op->opaque->stdout_fd, op,
                                                  &stdout_callbacks);
     op->opaque->stderr_gsource = mainloop_add_fd(op->id,
                                                  G_PRIORITY_LOW,
                                                  op->opaque->stderr_fd, op,
                                                  &stderr_callbacks);
     services_add_inflight_op(op);
     return pcmk_rc_ok;
 
 done:
     if (op->synchronous) {
         return (op->rc == PCMK_OCF_OK)? pcmk_rc_ok : pcmk_rc_error;
     } else {
         return services__finalize_async_op(op);
     }
 }
 
 GList *
 services_os_get_single_directory_list(const char *root, gboolean files, gboolean executable)
 {
     GList *list = NULL;
     struct dirent **namelist;
     int entries = 0, lpc = 0;
     char buffer[PATH_MAX];
 
     entries = scandir(root, &namelist, NULL, alphasort);
     if (entries <= 0) {
         return list;
     }
 
     for (lpc = 0; lpc < entries; lpc++) {
         struct stat sb;
 
         if ('.' == namelist[lpc]->d_name[0]) {
             free(namelist[lpc]);
             continue;
         }
 
         snprintf(buffer, sizeof(buffer), "%s/%s", root, namelist[lpc]->d_name);
 
         if (stat(buffer, &sb)) {
             continue;
         }
 
         if (S_ISDIR(sb.st_mode)) {
             if (files) {
                 free(namelist[lpc]);
                 continue;
             }
 
         } else if (S_ISREG(sb.st_mode)) {
             if (files == FALSE) {
                 free(namelist[lpc]);
                 continue;
 
             } else if (executable
                        && (sb.st_mode & S_IXUSR) == 0
                        && (sb.st_mode & S_IXGRP) == 0 && (sb.st_mode & S_IXOTH) == 0) {
                 free(namelist[lpc]);
                 continue;
             }
         }
 
         list = g_list_append(list, strdup(namelist[lpc]->d_name));
 
         free(namelist[lpc]);
     }
 
     free(namelist);
     return list;
 }
 
 GList *
 services_os_get_directory_list(const char *root, gboolean files, gboolean executable)
 {
     GList *result = NULL;
     char *dirs = strdup(root);
     char *dir = NULL;
 
     if (pcmk__str_empty(dirs)) {
         free(dirs);
         return result;
     }
 
     for (dir = strtok(dirs, ":"); dir != NULL; dir = strtok(NULL, ":")) {
         GList *tmp = services_os_get_single_directory_list(dir, files, executable);
 
         if (tmp) {
             result = g_list_concat(result, tmp);
         }
     }
 
     free(dirs);
 
     return result;
 }
diff --git a/lib/services/services_lsb.c b/lib/services/services_lsb.c
index 4e233085dd..4db167578a 100644
--- a/lib/services/services_lsb.c
+++ b/lib/services/services_lsb.c
@@ -1,302 +1,343 @@
 /*
  * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <stdio.h>
 #include <errno.h>
 #include <sys/stat.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include "services_private.h"
 #include "services_lsb.h"
 
 #define lsb_metadata_template  \
     "<?xml version='1.0'?>\n"                                           \
     "<!DOCTYPE resource-agent SYSTEM 'ra-api-1.dtd'>\n"                 \
     "<resource-agent name='%s' version='" PCMK_DEFAULT_AGENT_VERSION "'>\n" \
     "  <version>1.0</version>\n"                                        \
     "  <longdesc lang='en'>\n"                                          \
     "%s"                                                                \
     "  </longdesc>\n"                                                   \
     "  <shortdesc lang='en'>%s</shortdesc>\n"                           \
     "  <parameters>\n"                                                  \
     "  </parameters>\n"                                                 \
     "  <actions>\n"                                                     \
     "    <action name='meta-data'    timeout='5' />\n"                  \
     "    <action name='start'        timeout='15' />\n"                 \
     "    <action name='stop'         timeout='15' />\n"                 \
     "    <action name='status'       timeout='15' />\n"                 \
     "    <action name='restart'      timeout='15' />\n"                 \
     "    <action name='force-reload' timeout='15' />\n"                 \
     "    <action name='monitor'      timeout='15' interval='15' />\n"   \
     "  </actions>\n"                                                    \
     "  <special tag='LSB'>\n"                                           \
     "    <Provides>%s</Provides>\n"                                     \
     "    <Required-Start>%s</Required-Start>\n"                         \
     "    <Required-Stop>%s</Required-Stop>\n"                           \
     "    <Should-Start>%s</Should-Start>\n"                             \
     "    <Should-Stop>%s</Should-Stop>\n"                               \
     "    <Default-Start>%s</Default-Start>\n"                           \
     "    <Default-Stop>%s</Default-Stop>\n"                             \
     "  </special>\n"                                                    \
     "</resource-agent>\n"
 
 /* See "Comment Conventions for Init Scripts" in the LSB core specification at:
  * http://refspecs.linuxfoundation.org/lsb.shtml
  */
 #define LSB_INITSCRIPT_INFOBEGIN_TAG "### BEGIN INIT INFO"
 #define LSB_INITSCRIPT_INFOEND_TAG "### END INIT INFO"
 #define PROVIDES    "# Provides:"
 #define REQ_START   "# Required-Start:"
 #define REQ_STOP    "# Required-Stop:"
 #define SHLD_START  "# Should-Start:"
 #define SHLD_STOP   "# Should-Stop:"
 #define DFLT_START  "# Default-Start:"
 #define DFLT_STOP   "# Default-Stop:"
 #define SHORT_DSCR  "# Short-Description:"
 #define DESCRIPTION "# Description:"
 
 #define lsb_meta_helper_free_value(m)           \
     do {                                        \
         if ((m) != NULL) {                      \
             xmlFree(m);                         \
             (m) = NULL;                         \
         }                                       \
     } while(0)
 
 /*!
  * \internal
  * \brief Grab an LSB header value
  *
  * \param[in]     line    Line read from LSB init script
  * \param[in,out] value   If not set, will be set to XML-safe copy of value
  * \param[in]     prefix  Set value if line starts with this pattern
  *
  * \return TRUE if value was set, FALSE otherwise
  */
 static inline gboolean
 lsb_meta_helper_get_value(const char *line, char **value, const char *prefix)
 {
     if (!*value && pcmk__starts_with(line, prefix)) {
         *value = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST line+strlen(prefix));
         return TRUE;
     }
     return FALSE;
 }
 
 #define DESC_MAX 2048
 
 int
 services__get_lsb_metadata(const char *type, char **output)
 {
     char ra_pathname[PATH_MAX] = { 0, };
     FILE *fp = NULL;
     char buffer[1024] = { 0, };
     char *provides = NULL;
     char *req_start = NULL;
     char *req_stop = NULL;
     char *shld_start = NULL;
     char *shld_stop = NULL;
     char *dflt_start = NULL;
     char *dflt_stop = NULL;
     char *s_dscrpt = NULL;
     char *xml_l_dscrpt = NULL;
     int offset = 0;
     bool in_header = FALSE;
     char description[DESC_MAX] = { 0, };
 
     if (type[0] == '/') {
         snprintf(ra_pathname, sizeof(ra_pathname), "%s", type);
     } else {
         snprintf(ra_pathname, sizeof(ra_pathname), "%s/%s",
-                 LSB_ROOT_DIR, type);
+                 PCMK__LSB_INIT_DIR, type);
     }
 
     crm_trace("Looking into %s", ra_pathname);
     fp = fopen(ra_pathname, "r");
     if (fp == NULL) {
         return -errno;
     }
 
     /* Enter into the LSB-compliant comment block */
     while (fgets(buffer, sizeof(buffer), fp)) {
 
         // Ignore lines up to and including the block delimiter
         if (pcmk__starts_with(buffer, LSB_INITSCRIPT_INFOBEGIN_TAG)) {
             in_header = TRUE;
             continue;
         }
         if (!in_header) {
             continue;
         }
 
         /* Assume each of the following eight arguments contain one line */
         if (lsb_meta_helper_get_value(buffer, &provides, PROVIDES)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &req_start, REQ_START)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &req_stop, REQ_STOP)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &shld_start, SHLD_START)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &shld_stop, SHLD_STOP)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &dflt_start, DFLT_START)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &dflt_stop, DFLT_STOP)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &s_dscrpt, SHORT_DSCR)) {
             continue;
         }
 
         /* Long description may cross multiple lines */
         if ((offset == 0) // haven't already found long description
             && pcmk__starts_with(buffer, DESCRIPTION)) {
             bool processed_line = TRUE;
 
             // Get remainder of description line itself
             offset += snprintf(description, DESC_MAX, "%s",
                                buffer + strlen(DESCRIPTION));
 
             // Read any continuation lines of the description
             buffer[0] = '\0';
             while (fgets(buffer, sizeof(buffer), fp)) {
                 if (pcmk__starts_with(buffer, "#  ")
                     || pcmk__starts_with(buffer, "#\t")) {
                     /* '#' followed by a tab or more than one space indicates a
                      * continuation of the long description.
                      */
                     offset += snprintf(description + offset, DESC_MAX - offset,
                                        "%s", buffer + 1);
                 } else {
                     /* This line is not part of the long description,
                      * so continue with normal processing.
                      */
                     processed_line = FALSE;
                     break;
                 }
             }
 
             // Make long description safe to use in XML
             xml_l_dscrpt = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST(description));
 
             if (processed_line) {
                 // We grabbed the line into the long description
                 continue;
             }
         }
 
         // Stop if we leave the header block
         if (pcmk__starts_with(buffer, LSB_INITSCRIPT_INFOEND_TAG)) {
             break;
         }
         if (buffer[0] != '#') {
             break;
         }
     }
     fclose(fp);
 
     *output = crm_strdup_printf(lsb_metadata_template, type,
                                 (xml_l_dscrpt? xml_l_dscrpt : type),
                                 (s_dscrpt? s_dscrpt : type),
                                 (provides? provides : ""),
                                 (req_start? req_start : ""),
                                 (req_stop? req_stop : ""),
                                 (shld_start? shld_start : ""),
                                 (shld_stop? shld_stop : ""),
                                 (dflt_start? dflt_start : ""),
                                 (dflt_stop? dflt_stop : ""));
 
     lsb_meta_helper_free_value(xml_l_dscrpt);
     lsb_meta_helper_free_value(s_dscrpt);
     lsb_meta_helper_free_value(provides);
     lsb_meta_helper_free_value(req_start);
     lsb_meta_helper_free_value(req_stop);
     lsb_meta_helper_free_value(shld_start);
     lsb_meta_helper_free_value(shld_stop);
     lsb_meta_helper_free_value(dflt_start);
     lsb_meta_helper_free_value(dflt_stop);
 
     crm_trace("Created fake metadata: %llu",
               (unsigned long long) strlen(*output));
     return pcmk_ok;
 }
 
 GList *
 services__list_lsb_agents(void)
 {
-    return services_os_get_directory_list(LSB_ROOT_DIR, TRUE, TRUE);
+    return services_os_get_directory_list(PCMK__LSB_INIT_DIR, TRUE, TRUE);
 }
 
 bool
 services__lsb_agent_exists(const char *agent)
 {
     bool rc = FALSE;
     struct stat st;
-    char *path = pcmk__full_path(agent, LSB_ROOT_DIR);
+    char *path = pcmk__full_path(agent, PCMK__LSB_INIT_DIR);
 
     rc = (stat(path, &st) == 0);
     free(path);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Prepare an LSB action
  *
  * \param[in] op  Action to prepare
  *
  * \return Standard Pacemaker return code
  */
 int
 services__lsb_prepare(svc_action_t *op)
 {
-    op->opaque->exec = pcmk__full_path(op->agent, LSB_ROOT_DIR);
+    op->opaque->exec = pcmk__full_path(op->agent, PCMK__LSB_INIT_DIR);
     op->opaque->args[0] = strdup(op->opaque->exec);
     op->opaque->args[1] = strdup(op->action);
     if ((op->opaque->args[0] == NULL) || (op->opaque->args[1] == NULL)) {
         return ENOMEM;
     }
     return pcmk_rc_ok;
 }
 
+/*!
+ * \internal
+ * \brief Map an LSB result to a standard OCF result
+ *
+ * \param[in] action       Action that result is for
+ * \param[in] exit_status  LSB agent exit status
+ *
+ * \return Standard OCF result
+ */
+enum ocf_exitcode
+services__lsb2ocf(const char *action, int exit_status)
+{
+    // For non-status actions, LSB and OCF share error codes <= 7
+    if (!pcmk__str_any_of(action, "status", "monitor", NULL)) {
+        if ((exit_status < 0) || (exit_status > PCMK_LSB_NOT_RUNNING)) {
+            return PCMK_OCF_UNKNOWN_ERROR;
+        }
+        return (enum ocf_exitcode) exit_status;
+    }
+
+    // LSB status actions have their own codes
+    switch (exit_status) {
+        case PCMK_LSB_STATUS_OK:
+            return PCMK_OCF_OK;
+
+        case PCMK_LSB_STATUS_NOT_INSTALLED:
+            return PCMK_OCF_NOT_INSTALLED;
+
+        case PCMK_LSB_STATUS_INSUFFICIENT_PRIV:
+            return PCMK_OCF_INSUFFICIENT_PRIV;
+
+        case PCMK_LSB_STATUS_VAR_PID:
+        case PCMK_LSB_STATUS_VAR_LOCK:
+        case PCMK_LSB_STATUS_NOT_RUNNING:
+            return PCMK_OCF_NOT_RUNNING;
+
+        default:
+            return PCMK_OCF_UNKNOWN_ERROR;
+    }
+}
+
 // Deprecated functions kept only for backward API compatibility
 // LCOV_EXCL_START
 
 #include <crm/services_compat.h>
 
 svc_action_t *
 services_action_create(const char *name, const char *action,
                        guint interval_ms, int timeout)
 {
     return resources_action_create(name, PCMK_RESOURCE_CLASS_LSB, NULL, name,
                                    action, interval_ms, timeout, NULL, 0);
 }
 
 GList *
 services_list(void)
 {
     return resources_list_agents(PCMK_RESOURCE_CLASS_LSB, NULL);
 }
 
 // LCOV_EXCL_STOP
 // End deprecated API
diff --git a/lib/services/services_lsb.h b/lib/services/services_lsb.h
index 46d23dfdad..27023f61c5 100644
--- a/lib/services/services_lsb.h
+++ b/lib/services/services_lsb.h
@@ -1,18 +1,21 @@
 /*
  * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef SERVICES_LSB__H
 #  define SERVICES_LSB__H
 
 G_GNUC_INTERNAL int services__get_lsb_metadata(const char *type, char **output);
 G_GNUC_INTERNAL GList *services__list_lsb_agents(void);
 G_GNUC_INTERNAL bool services__lsb_agent_exists(const char *agent);
 G_GNUC_INTERNAL int services__lsb_prepare(svc_action_t *op);
 
+G_GNUC_INTERNAL
+enum ocf_exitcode services__lsb2ocf(const char *action, int exit_status);
+
 #endif
diff --git a/lib/services/services_nagios.c b/lib/services/services_nagios.c
index 2699e3f664..0c032e9f60 100644
--- a/lib/services/services_nagios.c
+++ b/lib/services/services_nagios.c
@@ -1,192 +1,220 @@
 /*
- * Copyright 2010-2019 the Pacemaker project contributors
+ * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/wait.h>
 #include <errno.h>
 #include <unistd.h>
 #include <dirent.h>
 #include <grp.h>
 #include <string.h>
 #include <sys/time.h>
 #include <sys/resource.h>
 
 #include "crm/crm.h"
 #include <crm/msg_xml.h>
 #include "crm/common/mainloop.h"
 #include "crm/services.h"
 
 #include "services_private.h"
 #include "services_nagios.h"
 
 /*!
  * \internal
  * \brief Prepare a Nagios action
  *
  * \param[in] op  Action to prepare
  *
  * \return Standard Pacemaker return code
  */
 int
 services__nagios_prepare(svc_action_t *op)
 {
     op->opaque->exec = pcmk__full_path(op->agent, NAGIOS_PLUGIN_DIR);
     op->opaque->args[0] = strdup(op->opaque->exec);
     if (op->opaque->args[0] == NULL) {
         return ENOMEM;
     }
 
     if (pcmk__str_eq(op->action, "monitor", pcmk__str_casei)
         && (op->interval_ms == 0)) {
 
         // Invoke --version for a nagios probe
         op->opaque->args[1] = strdup("--version");
         if (op->opaque->args[1] == NULL) {
             return ENOMEM;
         }
 
     } else if (op->params != NULL) {
         GHashTableIter iter;
         char *key = NULL;
         char *value = NULL;
         int index = 1; // 0 is already set to executable name
 
         g_hash_table_iter_init(&iter, op->params);
 
         while (g_hash_table_iter_next(&iter, (gpointer *) & key,
                                       (gpointer *) & value)) {
 
             if (index > (PCMK__NELEM(op->opaque->args) - 2)) {
                 return E2BIG;
             }
 
             if (pcmk__str_eq(key, XML_ATTR_CRM_VERSION, pcmk__str_casei)
                 || strstr(key, CRM_META "_")) {
                 continue;
             }
 
             op->opaque->args[index++] = crm_strdup_printf("--%s", key);
             op->opaque->args[index++] = strdup(value);
             if (op->opaque->args[index - 1] == NULL) {
                 return ENOMEM;
             }
         }
     }
 
     // Nagios actions don't need to keep the parameters
     if (op->params != NULL) {
         g_hash_table_destroy(op->params);
         op->params = NULL;
     }
     return pcmk_rc_ok;
 }
 
+/*!
+ * \internal
+ * \brief Map a Nagios result to a standard OCF result
+ *
+ * \param[in] exit_status  Nagios exit status
+ *
+ * \return Standard OCF result
+ */
+enum ocf_exitcode
+services__nagios2ocf(int exit_status)
+{
+    switch (exit_status) {
+        case NAGIOS_STATE_OK:
+            return PCMK_OCF_OK;
+
+        case NAGIOS_INSUFFICIENT_PRIV:
+            return PCMK_OCF_INSUFFICIENT_PRIV;
+
+        case NAGIOS_STATE_WARNING:
+            return PCMK_OCF_DEGRADED;
+
+        case NAGIOS_STATE_CRITICAL:
+        case NAGIOS_STATE_UNKNOWN:
+        default:
+            return PCMK_OCF_UNKNOWN_ERROR;
+    }
+}
+
 static inline char *
 nagios_metadata_name(const char *plugin)
 {
     return crm_strdup_printf(NAGIOS_METADATA_DIR "/%s.xml", plugin);
 }
 
 GList *
 services__list_nagios_agents(void)
 {
     GList *plugin_list = NULL;
     GList *result = NULL;
 
     plugin_list = services_os_get_directory_list(NAGIOS_PLUGIN_DIR, TRUE, TRUE);
 
     // Return only the plugins that have metadata
     for (GList *gIter = plugin_list; gIter != NULL; gIter = gIter->next) {
         struct stat st;
         const char *plugin = gIter->data;
         char *metadata = nagios_metadata_name(plugin);
 
         if (stat(metadata, &st) == 0) {
             result = g_list_append(result, strdup(plugin));
         }
         free(metadata);
     }
     g_list_free_full(plugin_list, free);
     return result;
 }
 
 gboolean
 services__nagios_agent_exists(const char *name)
 {
     char *buf = NULL;
     gboolean rc = FALSE;
     struct stat st;
 
     if (name == NULL) {
         return rc;
     }
 
     buf = crm_strdup_printf(NAGIOS_PLUGIN_DIR "/%s", name);
     if (stat(buf, &st) == 0) {
         rc = TRUE;
     }
 
     free(buf);
     return rc;
 }
 
 int
 services__get_nagios_metadata(const char *type, char **output)
 {
     int rc = pcmk_ok;
     FILE *file_strm = NULL;
     int start = 0, length = 0, read_len = 0;
     char *metadata_file = nagios_metadata_name(type);
 
     file_strm = fopen(metadata_file, "r");
     if (file_strm == NULL) {
         crm_err("Metadata file %s does not exist", metadata_file);
         free(metadata_file);
         return -EIO;
     }
 
     /* see how big the file is */
     start = ftell(file_strm);
     fseek(file_strm, 0L, SEEK_END);
     length = ftell(file_strm);
     fseek(file_strm, 0L, start);
 
     CRM_ASSERT(length >= 0);
     CRM_ASSERT(start == ftell(file_strm));
 
     if (length <= 0) {
         crm_info("%s was not valid", metadata_file);
         free(*output);
         *output = NULL;
         rc = -EIO;
 
     } else {
         crm_trace("Reading %d bytes from file", length);
         *output = calloc(1, (length + 1));
         read_len = fread(*output, 1, length, file_strm);
         if (read_len != length) {
             crm_err("Calculated and read bytes differ: %d vs. %d",
                     length, read_len);
             free(*output);
             *output = NULL;
             rc = -EIO;
         }
     }
 
     fclose(file_strm);
     free(metadata_file);
     return rc;
 }
diff --git a/lib/services/services_nagios.h b/lib/services/services_nagios.h
index fd95cb55f7..2e447e0328 100644
--- a/lib/services/services_nagios.h
+++ b/lib/services/services_nagios.h
@@ -1,25 +1,28 @@
 /*
- * Copyright 2010-2019 the Pacemaker project contributors
+ * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef SERVICES_NAGIOS__H
 #  define SERVICES_NAGIOS__H
 
 G_GNUC_INTERNAL
 int services__nagios_prepare(svc_action_t *op);
 
+G_GNUC_INTERNAL
+enum ocf_exitcode services__nagios2ocf(int exit_status);
+
 G_GNUC_INTERNAL
 GList *services__list_nagios_agents(void);
 
 G_GNUC_INTERNAL
 gboolean services__nagios_agent_exists(const char *agent);
 
 G_GNUC_INTERNAL
 int services__get_nagios_metadata(const char *type, char **output);
 
 #endif
diff --git a/lib/services/services_ocf.c b/lib/services/services_ocf.c
index 1b7d2b50a4..e5bf2dd3cd 100644
--- a/lib/services/services_ocf.c
+++ b/lib/services/services_ocf.c
@@ -1,155 +1,179 @@
 /*
  * Copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/services_internal.h>
 
 #include "services_private.h"
 #include "services_ocf.h"
 
 GList *
 resources_os_list_ocf_providers(void)
 {
     return get_directory_list(OCF_RA_PATH, FALSE, TRUE);
 }
 
 static GList *
 services_os_get_directory_list_provider(const char *root, const char *provider,
                                         gboolean files, gboolean executable)
 {
     GList *result = NULL;
     char *dirs = strdup(root);
     char *dir = NULL;
     char buffer[PATH_MAX];
 
     if (pcmk__str_empty(dirs)) {
         free(dirs);
         return result;
     }
 
     for (dir = strtok(dirs, ":"); dir != NULL; dir = strtok(NULL, ":")) {
         GList *tmp = NULL;
 
         sprintf(buffer, "%s/%s", dir, provider);
         tmp = services_os_get_single_directory_list(buffer, files, executable);
 
         if (tmp) {
             result = g_list_concat(result, tmp);
         }
     }
 
     free(dirs);
 
     return result;
 }
 
 GList *
 resources_os_list_ocf_agents(const char *provider)
 {
     GList *gIter = NULL;
     GList *result = NULL;
     GList *providers = NULL;
 
     if (provider) {
         return services_os_get_directory_list_provider(OCF_RA_PATH, provider,
                                                        TRUE, TRUE);
     }
 
     providers = resources_os_list_ocf_providers();
     for (gIter = providers; gIter != NULL; gIter = gIter->next) {
         GList *tmp1 = result;
         GList *tmp2 = resources_os_list_ocf_agents(gIter->data);
 
         if (tmp2) {
             result = g_list_concat(tmp1, tmp2);
         }
     }
     g_list_free_full(providers, free);
     return result;
 }
 
 gboolean
 services__ocf_agent_exists(const char *provider, const char *agent)
 {
     gboolean rc = FALSE;
     struct stat st;
     char *dirs = strdup(OCF_RA_PATH);
     char *dir = NULL;
     char *buf = NULL;
 
     if (provider == NULL || agent == NULL || pcmk__str_empty(dirs)) {
         free(dirs);
         return rc;
     }
 
     for (dir = strtok(dirs, ":"); dir != NULL; dir = strtok(NULL, ":")) {
         buf = crm_strdup_printf("%s/%s/%s", dir, provider, agent);
         if (stat(buf, &st) == 0) {
             free(buf);
             rc = TRUE;
             break;
         }
 
         free(buf);
     }
 
     free(dirs);
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Prepare an OCF action
  *
  * \param[in] op  Action to prepare
  *
  * \return Standard Pacemaker return code
  */
 int
 services__ocf_prepare(svc_action_t *op)
 {
     char *dirs = strdup(OCF_RA_PATH);
     struct stat st;
 
     if (dirs == NULL) {
         return ENOMEM;
     }
 
     // Look for agent on path
     for (char *dir = strtok(dirs, ":"); dir != NULL; dir = strtok(NULL, ":")) {
         char *buf = crm_strdup_printf("%s/%s/%s", dir, op->provider, op->agent);
 
         if (stat(buf, &st) == 0) {
             op->opaque->exec = buf;
             break;
         }
         free(buf);
     }
     free(dirs);
 
     if (op->opaque->exec == NULL) {
         return ENOENT;
     }
 
     op->opaque->args[0] = strdup(op->opaque->exec);
     op->opaque->args[1] = strdup(op->action);
     if ((op->opaque->args[0] == NULL) || (op->opaque->args[1] == NULL)) {
         return ENOMEM;
     }
 
     return pcmk_rc_ok;
 }
+
+/*!
+ * \internal
+ * \brief Map an actual OCF result to a standard OCF result
+ *
+ * \param[in] exit_status  Actual OCF agent exit status
+ *
+ * \return Standard OCF result
+ */
+enum ocf_exitcode
+services__ocf2ocf(int exit_status)
+{
+    switch (exit_status) {
+        case PCMK_OCF_DEGRADED:
+        case PCMK_OCF_DEGRADED_PROMOTED:
+            break;
+        default:
+            if ((exit_status < 0) || (exit_status > PCMK_OCF_FAILED_PROMOTED)) {
+                exit_status = PCMK_OCF_UNKNOWN_ERROR;
+            }
+            break;
+    }
+    return (enum ocf_exitcode) exit_status;
+}
diff --git a/lib/services/services_ocf.h b/lib/services/services_ocf.h
index 62c344b55c..1c40552723 100644
--- a/lib/services/services_ocf.h
+++ b/lib/services/services_ocf.h
@@ -1,28 +1,31 @@
 /*
  * Copyright 2010-2011 Red Hat, Inc.
  * Later changes copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PCMK__SERVICES_OCF__H
 #define PCMK__SERVICES_OCF__H
 
 #include <glib.h>
 
 G_GNUC_INTERNAL
 GList *resources_os_list_ocf_providers(void);
 
 G_GNUC_INTERNAL
 GList *resources_os_list_ocf_agents(const char *provider);
 
 G_GNUC_INTERNAL
 gboolean services__ocf_agent_exists(const char *provider, const char *agent);
 
 G_GNUC_INTERNAL
 int services__ocf_prepare(svc_action_t *op);
 
+G_GNUC_INTERNAL
+enum ocf_exitcode services__ocf2ocf(int exit_status);
+
 #endif  // PCMK__SERVICES_OCF__H
diff --git a/lib/services/services_private.h b/lib/services/services_private.h
index a1340c71ca..bace7e1539 100644
--- a/lib/services/services_private.h
+++ b/lib/services/services_private.h
@@ -1,95 +1,98 @@
 /*
  * Copyright 2010-2011 Red Hat, Inc.
  * Later changes copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef SERVICES_PRIVATE__H
 #  define SERVICES_PRIVATE__H
 
 #  include <glib.h>
 #  include "crm/services.h"
 
 #if SUPPORT_DBUS
 #  include <dbus/dbus.h>
 #endif
 
 #define MAX_ARGC        255
 struct svc_action_private_s {
     char *exec;
     char *exit_reason;
     char *args[MAX_ARGC];
 
     uid_t uid;
     gid_t gid;
 
     guint repeat_timer;
     void (*callback) (svc_action_t * op);
     void (*fork_callback) (svc_action_t * op);
 
     int stderr_fd;
     mainloop_io_t *stderr_gsource;
 
     int stdout_fd;
     mainloop_io_t *stdout_gsource;
 
     int stdin_fd;
 #if SUPPORT_DBUS
     DBusPendingCall* pending;
     unsigned timerid;
 #endif
 };
 
 G_GNUC_INTERNAL
 GList *services_os_get_single_directory_list(const char *root, gboolean files,
                                              gboolean executable);
 
 G_GNUC_INTERNAL
 GList *services_os_get_directory_list(const char *root, gboolean files, gboolean executable);
 
 G_GNUC_INTERNAL
 int services__execute_file(svc_action_t *op);
 
 G_GNUC_INTERNAL
 gboolean cancel_recurring_action(svc_action_t * op);
 
 G_GNUC_INTERNAL
 gboolean recurring_action_timer(gpointer data);
 
 G_GNUC_INTERNAL
 int services__finalize_async_op(svc_action_t *op);
 
 G_GNUC_INTERNAL
 int services__generic_error(svc_action_t *op);
 
 G_GNUC_INTERNAL
 int services__not_installed_error(svc_action_t *op);
 
 G_GNUC_INTERNAL
 int services__authorization_error(svc_action_t *op);
 
+G_GNUC_INTERNAL
+int services__configuration_error(svc_action_t *op);
+
 G_GNUC_INTERNAL
 void services__handle_exec_error(svc_action_t * op, int error);
 
 G_GNUC_INTERNAL
 void services__set_cancelled(svc_action_t *action);
 
 G_GNUC_INTERNAL
 void services_add_inflight_op(svc_action_t *op);
 
 G_GNUC_INTERNAL
 void services_untrack_op(svc_action_t *op);
 
 G_GNUC_INTERNAL
 gboolean is_op_blocked(const char *rsc);
 
 #if SUPPORT_DBUS
 G_GNUC_INTERNAL
 void services_set_op_pending(svc_action_t *op, DBusPendingCall *pending);
 #endif
 
 #endif  /* SERVICES_PRIVATE__H */
diff --git a/lib/services/systemd.c b/lib/services/systemd.c
index f832f02494..731a4d4943 100644
--- a/lib/services/systemd.c
+++ b/lib/services/systemd.c
@@ -1,1037 +1,1052 @@
 /*
  * Copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/services_internal.h>
 #include <crm/common/mainloop.h>
 
 #include <sys/stat.h>
 #include <gio/gio.h>
 #include <services_private.h>
 #include <systemd.h>
 #include <dbus/dbus.h>
 #include <pcmk-dbus.h>
 
 static void invoke_unit_by_path(svc_action_t *op, const char *unit);
 
 #define BUS_NAME         "org.freedesktop.systemd1"
 #define BUS_NAME_MANAGER BUS_NAME ".Manager"
 #define BUS_NAME_UNIT    BUS_NAME ".Unit"
 #define BUS_PATH         "/org/freedesktop/systemd1"
 
 /*!
  * \internal
  * \brief Prepare a systemd action
  *
  * \param[in] op  Action to prepare
  *
  * \return Standard Pacemaker return code
  */
 int
 services__systemd_prepare(svc_action_t *op)
 {
     op->opaque->exec = strdup("systemd-dbus");
     if (op->opaque->exec == NULL) {
         return ENOMEM;
     }
     return pcmk_rc_ok;
 }
 
+/*!
+ * \internal
+ * \brief Map a systemd result to a standard OCF result
+ *
+ * \param[in] exit_status  Systemd result
+ *
+ * \return Standard OCF result
+ */
+enum ocf_exitcode
+services__systemd2ocf(int exit_status)
+{
+    // This library uses OCF codes for systemd actions
+    return (enum ocf_exitcode) exit_status;
+}
+
 static inline DBusMessage *
 systemd_new_method(const char *method)
 {
     crm_trace("Calling: %s on " BUS_NAME_MANAGER, method);
     return dbus_message_new_method_call(BUS_NAME, BUS_PATH, BUS_NAME_MANAGER,
                                         method);
 }
 
 /*
  * Functions to manage a static DBus connection
  */
 
 static DBusConnection* systemd_proxy = NULL;
 
 static inline DBusPendingCall *
 systemd_send(DBusMessage *msg,
              void(*done)(DBusPendingCall *pending, void *user_data),
              void *user_data, int timeout)
 {
     return pcmk_dbus_send(msg, systemd_proxy, done, user_data, timeout);
 }
 
 static inline DBusMessage *
 systemd_send_recv(DBusMessage *msg, DBusError *error, int timeout)
 {
     return pcmk_dbus_send_recv(msg, systemd_proxy, error, timeout);
 }
 
 /*!
  * \internal
  * \brief Send a method to systemd without arguments, and wait for reply
  *
  * \param[in] method  Method to send
  *
  * \return Systemd reply on success, NULL (and error will be logged) otherwise
  *
  * \note The caller must call dbus_message_unref() on the reply after
  *       handling it.
  */
 static DBusMessage *
 systemd_call_simple_method(const char *method)
 {
     DBusMessage *msg = systemd_new_method(method);
     DBusMessage *reply = NULL;
     DBusError error;
 
     /* Don't call systemd_init() here, because that calls this */
     CRM_CHECK(systemd_proxy, return NULL);
 
     if (msg == NULL) {
         crm_err("Could not create message to send %s to systemd", method);
         return NULL;
     }
 
     dbus_error_init(&error);
     reply = systemd_send_recv(msg, &error, DBUS_TIMEOUT_USE_DEFAULT);
     dbus_message_unref(msg);
 
     if (dbus_error_is_set(&error)) {
         crm_err("Could not send %s to systemd: %s (%s)",
                 method, error.message, error.name);
         dbus_error_free(&error);
         return NULL;
 
     } else if (reply == NULL) {
         crm_err("Could not send %s to systemd: no reply received", method);
         return NULL;
     }
 
     return reply;
 }
 
 static gboolean
 systemd_init(void)
 {
     static int need_init = 1;
     // https://dbus.freedesktop.org/doc/api/html/group__DBusConnection.html
 
     if (systemd_proxy
         && dbus_connection_get_is_connected(systemd_proxy) == FALSE) {
         crm_warn("Connection to System DBus is closed. Reconnecting...");
         pcmk_dbus_disconnect(systemd_proxy);
         systemd_proxy = NULL;
         need_init = 1;
     }
 
     if (need_init) {
         need_init = 0;
         systemd_proxy = pcmk_dbus_connect();
     }
     if (systemd_proxy == NULL) {
         return FALSE;
     }
     return TRUE;
 }
 
 static inline char *
 systemd_get_property(const char *unit, const char *name,
                      void (*callback)(const char *name, const char *value, void *userdata),
                      void *userdata, DBusPendingCall **pending, int timeout)
 {
     return systemd_proxy?
            pcmk_dbus_get_property(systemd_proxy, BUS_NAME, unit, BUS_NAME_UNIT,
                                   name, callback, userdata, pending, timeout)
            : NULL;
 }
 
 void
 systemd_cleanup(void)
 {
     if (systemd_proxy) {
         pcmk_dbus_disconnect(systemd_proxy);
         systemd_proxy = NULL;
     }
 }
 
 /*
  * end of systemd_proxy functions
  */
 
 /*!
  * \internal
  * \brief Check whether a file name represents a manageable systemd unit
  *
  * \param[in] name  File name to check
  *
  * \return Pointer to "dot" before filename extension if so, NULL otherwise
  */
 static const char *
 systemd_unit_extension(const char *name)
 {
     if (name) {
         const char *dot = strrchr(name, '.');
 
         if (dot && (!strcmp(dot, ".service")
                     || !strcmp(dot, ".socket")
                     || !strcmp(dot, ".mount")
                     || !strcmp(dot, ".timer")
                     || !strcmp(dot, ".path"))) {
             return dot;
         }
     }
     return NULL;
 }
 
 static char *
 systemd_service_name(const char *name)
 {
     if (name == NULL) {
         return NULL;
     }
 
     if (systemd_unit_extension(name)) {
         return strdup(name);
     }
 
     return crm_strdup_printf("%s.service", name);
 }
 
 static void
 systemd_daemon_reload_complete(DBusPendingCall *pending, void *user_data)
 {
     DBusError error;
     DBusMessage *reply = NULL;
     unsigned int reload_count = GPOINTER_TO_UINT(user_data);
 
     dbus_error_init(&error);
     if(pending) {
         reply = dbus_pending_call_steal_reply(pending);
     }
 
     if (pcmk_dbus_find_error(pending, reply, &error)) {
         crm_err("Could not issue systemd reload %d: %s", reload_count, error.message);
         dbus_error_free(&error);
 
     } else {
         crm_trace("Reload %d complete", reload_count);
     }
 
     if(pending) {
         dbus_pending_call_unref(pending);
     }
     if(reply) {
         dbus_message_unref(reply);
     }
 }
 
 static bool
 systemd_daemon_reload(int timeout)
 {
     static unsigned int reload_count = 0;
     DBusMessage *msg = systemd_new_method("Reload");
 
     reload_count++;
     CRM_ASSERT(msg != NULL);
     systemd_send(msg, systemd_daemon_reload_complete,
                  GUINT_TO_POINTER(reload_count), timeout);
     dbus_message_unref(msg);
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Set an action result based on a method error
  *
  * \param[in] op     Action to set result for
  * \param[in] error  Method error
  */
 static void
 set_result_from_method_error(svc_action_t *op, const DBusError *error)
 {
     services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                          "Unable to invoke systemd DBus method");
 
     if (strstr(error->name, "org.freedesktop.systemd1.InvalidName")
         || strstr(error->name, "org.freedesktop.systemd1.LoadFailed")
         || strstr(error->name, "org.freedesktop.systemd1.NoSuchUnit")) {
 
         if (pcmk__str_eq(op->action, "stop", pcmk__str_casei)) {
             crm_trace("Masking systemd stop failure (%s) for %s "
                       "because unknown service can be considered stopped",
                       error->name, crm_str(op->rsc));
             services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
             return;
         }
 
         services__set_result(op, PCMK_OCF_NOT_INSTALLED,
                              PCMK_EXEC_NOT_INSTALLED, "systemd unit not found");
     }
 
     crm_err("DBus request for %s of systemd unit %s for resource %s failed: %s",
             op->action, op->agent, crm_str(op->rsc), error->message);
 }
 
 /*!
  * \internal
  * \brief Extract unit path from LoadUnit reply, and execute action
  *
  * \param[in] reply  LoadUnit reply
  * \param[in] op     Action to execute (or NULL to just return path)
  *
  * \return DBus object path for specified unit if successful (only valid for
  *         lifetime of \p reply), otherwise NULL
  */
 static const char *
 execute_after_loadunit(DBusMessage *reply, svc_action_t *op)
 {
     const char *path = NULL;
     DBusError error;
 
     /* path here is not used other than as a non-NULL flag to indicate that a
      * request was indeed sent
      */
     if (pcmk_dbus_find_error((void *) &path, reply, &error)) {
         if (op != NULL) {
             set_result_from_method_error(op, &error);
         }
         dbus_error_free(&error);
 
     } else if (!pcmk_dbus_type_check(reply, NULL, DBUS_TYPE_OBJECT_PATH,
                                      __func__, __LINE__)) {
         if (op != NULL) {
             services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                  "systemd DBus method had unexpected reply");
             crm_err("Could not load systemd unit %s for %s: "
                     "DBus reply has unexpected type", op->agent, op->id);
         } else {
             crm_err("Could not load systemd unit: "
                     "DBus reply has unexpected type");
         }
 
     } else {
         dbus_message_get_args (reply, NULL,
                                DBUS_TYPE_OBJECT_PATH, &path,
                                DBUS_TYPE_INVALID);
     }
 
     if (op != NULL) {
         if (path != NULL) {
             invoke_unit_by_path(op, path);
 
         } else if (!(op->synchronous)) {
             services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                  "No DBus object found for systemd unit");
             services__finalize_async_op(op);
         }
     }
 
     return path;
 }
 
 /*!
  * \internal
  * \brief Execute a systemd action after its LoadUnit completes
  *
  * \param[in] pending    If not NULL, DBus call associated with LoadUnit request
  * \param[in] user_data  Action to execute
  */
 static void
 loadunit_completed(DBusPendingCall *pending, void *user_data)
 {
     DBusMessage *reply = NULL;
     svc_action_t *op = user_data;
 
     crm_trace("LoadUnit result for %s arrived", op->id);
 
     // Grab the reply
     if (pending != NULL) {
         reply = dbus_pending_call_steal_reply(pending);
     }
 
     // The call is no longer pending
     CRM_LOG_ASSERT(pending == op->opaque->pending);
     services_set_op_pending(op, NULL);
 
     // Execute the desired action based on the reply
     execute_after_loadunit(reply, user_data);
     if (reply != NULL) {
         dbus_message_unref(reply);
     }
 }
 
 /*!
  * \internal
  * \brief Execute a systemd action, given the unit name
  *
  * \param[in]  arg_name  Unit name (possibly shortened, i.e. without ".service")
  * \param[in]  op        Action to execute (if NULL, just get the object path)
  * \param[out] path      If non-NULL and \p op is NULL or synchronous, where to
  *                       store DBus object path for specified unit
  *
  * \return Standard Pacemaker return code (for NULL \p op, pcmk_rc_ok means unit
  *         was found; for synchronous actions, pcmk_rc_ok means unit was
  *         executed, with the actual result stored in \p op; for asynchronous
  *         actions, pcmk_rc_ok means action was initiated)
  * \note It is the caller's responsibility to free the return value if non-NULL.
  */
 static int
 invoke_unit_by_name(const char *arg_name, svc_action_t *op, char **path)
 {
     DBusMessage *msg;
     DBusMessage *reply = NULL;
     DBusPendingCall *pending = NULL;
     char *name = NULL;
 
     if (!systemd_init()) {
         if (op != NULL) {
             services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                  "No DBus connection");
         }
         return ENOTCONN;
     }
 
     /* Create a LoadUnit DBus method (equivalent to GetUnit if already loaded),
      * which makes the unit usable via further DBus methods.
      *
      * <method name="LoadUnit">
      *  <arg name="name" type="s" direction="in"/>
      *  <arg name="unit" type="o" direction="out"/>
      * </method>
      */
     msg = systemd_new_method("LoadUnit");
     CRM_ASSERT(msg != NULL);
 
     // Add the (expanded) unit name as the argument
     name = systemd_service_name(arg_name);
     CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &name,
                                             DBUS_TYPE_INVALID));
     free(name);
 
     if ((op == NULL) || op->synchronous) {
         // For synchronous ops, wait for a reply and extract the result
         const char *unit = NULL;
         int rc = pcmk_rc_ok;
 
         reply = systemd_send_recv(msg, NULL,
                                   (op? op->timeout : DBUS_TIMEOUT_USE_DEFAULT));
         dbus_message_unref(msg);
 
         unit = execute_after_loadunit(reply, op);
         if (unit == NULL) {
             rc = ENOENT;
             if (path != NULL) {
                 *path = NULL;
             }
         } else if (path != NULL) {
             *path = strdup(unit);
             if (*path == NULL) {
                 rc = ENOMEM;
             }
         }
 
         if (reply != NULL) {
             dbus_message_unref(reply);
         }
         return rc;
     }
 
     // For asynchronous ops, initiate the LoadUnit call and return
     pending = systemd_send(msg, loadunit_completed, op, op->timeout);
     if (pending == NULL) {
         services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                              "Unable to send DBus message");
         dbus_message_unref(msg);
         return ECOMM;
     }
 
     // LoadUnit was successfully initiated
     services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
     services_set_op_pending(op, pending);
     dbus_message_unref(msg);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Compare two strings alphabetically (case-insensitive)
  *
  * \param[in] a  First string to compare
  * \param[in] b  Second string to compare
  *
  * \return 0 if strings are equal, -1 if a < b, 1 if a > b
  *
  * \note Usable as a GCompareFunc with g_list_sort().
  *       NULL is considered less than non-NULL.
  */
 static gint
 sort_str(gconstpointer a, gconstpointer b)
 {
     if (!a && !b) {
         return 0;
     } else if (!a) {
         return -1;
     } else if (!b) {
         return 1;
     }
     return strcasecmp(a, b);
 }
 
 GList *
 systemd_unit_listall(void)
 {
     int nfiles = 0;
     GList *units = NULL;
     DBusMessageIter args;
     DBusMessageIter unit;
     DBusMessageIter elem;
     DBusMessage *reply = NULL;
 
     if (systemd_init() == FALSE) {
         return NULL;
     }
 
 /*
         "  <method name=\"ListUnitFiles\">\n"                               \
         "   <arg name=\"files\" type=\"a(ss)\" direction=\"out\"/>\n" \
         "  </method>\n"                                                 \
 */
 
     reply = systemd_call_simple_method("ListUnitFiles");
     if (reply == NULL) {
         return NULL;
     }
     if (!dbus_message_iter_init(reply, &args)) {
         crm_err("Could not list systemd unit files: systemd reply has no arguments");
         dbus_message_unref(reply);
         return NULL;
     }
     if (!pcmk_dbus_type_check(reply, &args, DBUS_TYPE_ARRAY,
                               __func__, __LINE__)) {
         crm_err("Could not list systemd unit files: systemd reply has invalid arguments");
         dbus_message_unref(reply);
         return NULL;
     }
 
     dbus_message_iter_recurse(&args, &unit);
     for (; dbus_message_iter_get_arg_type(&unit) != DBUS_TYPE_INVALID;
         dbus_message_iter_next(&unit)) {
 
         DBusBasicValue value;
         const char *match = NULL;
         char *unit_name = NULL;
         char *basename = NULL;
 
         if(!pcmk_dbus_type_check(reply, &unit, DBUS_TYPE_STRUCT, __func__, __LINE__)) {
             crm_warn("Skipping systemd reply argument with unexpected type");
             continue;
         }
 
         dbus_message_iter_recurse(&unit, &elem);
         if(!pcmk_dbus_type_check(reply, &elem, DBUS_TYPE_STRING, __func__, __LINE__)) {
             crm_warn("Skipping systemd reply argument with no string");
             continue;
         }
 
         dbus_message_iter_get_basic(&elem, &value);
         if (value.str == NULL) {
             crm_debug("ListUnitFiles reply did not provide a string");
             continue;
         }
         crm_trace("DBus ListUnitFiles listed: %s", value.str);
 
         match = systemd_unit_extension(value.str);
         if (match == NULL) {
             // This is not a unit file type we know how to manage
             crm_debug("ListUnitFiles entry '%s' is not supported as resource",
                       value.str);
             continue;
         }
 
         // ListUnitFiles returns full path names, we just want base name
         basename = strrchr(value.str, '/');
         if (basename) {
             basename = basename + 1;
         } else {
             basename = value.str;
         }
 
         if (!strcmp(match, ".service")) {
             // Service is the "default" unit type, so strip it
             unit_name = strndup(basename, match - basename);
         } else {
             unit_name = strdup(basename);
         }
 
         nfiles++;
         units = g_list_prepend(units, unit_name);
     }
 
     dbus_message_unref(reply);
 
     crm_trace("Found %d manageable systemd unit files", nfiles);
     units = g_list_sort(units, sort_str);
     return units;
 }
 
 gboolean
 systemd_unit_exists(const char *name)
 {
     char *path = NULL;
     char *state = NULL;
 
     /* Note: Makes a blocking dbus calls
      * Used by resources_find_service_class() when resource class=service
      */
     if ((invoke_unit_by_name(name, NULL, &path) != pcmk_rc_ok)
         || (path == NULL)) {
         return FALSE;
     }
 
     /* A successful LoadUnit is not sufficient to determine the unit's
      * existence; it merely means the LoadUnit request received a reply.
      * We must make another blocking call to check the LoadState property.
      */
     state = systemd_get_property(path, "LoadState", NULL, NULL, NULL,
                                  DBUS_TIMEOUT_USE_DEFAULT);
     if (pcmk__str_any_of(state, "loaded", "masked", NULL)) {
         free(state);
         return TRUE;
     }
     free(state);
     return FALSE;
 }
 
 #define METADATA_FORMAT                                                     \
     "<?xml version=\"1.0\"?>\n"                                             \
     "<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"                   \
     "<resource-agent name=\"%s\" version=\"" PCMK_DEFAULT_AGENT_VERSION "\">\n" \
     "  <version>1.1</version>\n"                                            \
     "  <longdesc lang=\"en\">\n"                                            \
     "    %s\n"                                                              \
     "  </longdesc>\n"                                                       \
     "  <shortdesc lang=\"en\">systemd unit file for %s</shortdesc>\n"       \
     "  <parameters/>\n"                                                     \
     "  <actions>\n"                                                         \
     "    <action name=\"start\"     timeout=\"100\" />\n"                   \
     "    <action name=\"stop\"      timeout=\"100\" />\n"                   \
     "    <action name=\"status\"    timeout=\"100\" />\n"                   \
     "    <action name=\"monitor\"   timeout=\"100\" interval=\"60\"/>\n"    \
     "    <action name=\"meta-data\" timeout=\"5\"   />\n"                   \
     "  </actions>\n"                                                        \
     "  <special tag=\"systemd\"/>\n"                                        \
     "</resource-agent>\n"
 
 static char *
 systemd_unit_metadata(const char *name, int timeout)
 {
     char *meta = NULL;
     char *desc = NULL;
     char *path = NULL;
 
     if (invoke_unit_by_name(name, NULL, &path) == pcmk_rc_ok) {
         /* TODO: Worth a making blocking call for? Probably not. Possibly if cached. */
         desc = systemd_get_property(path, "Description", NULL, NULL, NULL,
                                     timeout);
     } else {
         desc = crm_strdup_printf("Systemd unit file for %s", name);
     }
 
     meta = crm_strdup_printf(METADATA_FORMAT, name, desc, name);
     free(desc);
     free(path);
     return meta;
 }
 
 /*!
  * \internal
  * \brief Determine result of method from reply
  *
  * \param[in] reply  Reply to start, stop, or restart request
  * \param[in] op     Action that was executed
  */
 static void
 process_unit_method_reply(DBusMessage *reply, svc_action_t *op)
 {
     DBusError error;
 
     /* The first use of error here is not used other than as a non-NULL flag to
      * indicate that a request was indeed sent
      */
     if (pcmk_dbus_find_error((void *) &error, reply, &error)) {
         set_result_from_method_error(op, &error);
         dbus_error_free(&error);
 
     } else if (!pcmk_dbus_type_check(reply, NULL, DBUS_TYPE_OBJECT_PATH,
                                      __func__, __LINE__)) {
         crm_warn("DBus request for %s of %s succeeded but "
                  "return type was unexpected", op->action, crm_str(op->rsc));
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE,
                              "systemd DBus method had unexpected reply");
 
     } else {
         const char *path = NULL;
 
         dbus_message_get_args(reply, NULL,
                               DBUS_TYPE_OBJECT_PATH, &path,
                               DBUS_TYPE_INVALID);
         crm_debug("DBus request for %s of %s using %s succeeded",
                   op->action, crm_str(op->rsc), path);
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
     }
 }
 
 /*!
  * \internal
  * \brief Process the completion of an asynchronous unit start, stop, or restart
  *
  * \param[in] pending    If not NULL, DBus call associated with request
  * \param[in] user_data  Action that was executed
  */
 static void
 unit_method_complete(DBusPendingCall *pending, void *user_data)
 {
     DBusMessage *reply = NULL;
     svc_action_t *op = user_data;
 
     crm_trace("Result for %s arrived", op->id);
 
     // Grab the reply
     if (pending != NULL) {
         reply = dbus_pending_call_steal_reply(pending);
     }
 
     // The call is no longer pending
     CRM_LOG_ASSERT(pending == op->opaque->pending);
     services_set_op_pending(op, NULL);
 
     // Determine result and finalize action
     process_unit_method_reply(reply, op);
     services__finalize_async_op(op);
     if (reply != NULL) {
         dbus_message_unref(reply);
     }
 }
 
 #define SYSTEMD_OVERRIDE_ROOT "/run/systemd/system/"
 
 /* When the cluster manages a systemd resource, we create a unit file override
  * to order the service "before" pacemaker. The "before" relationship won't
  * actually be used, since systemd won't ever start the resource -- we're
  * interested in the reverse shutdown ordering it creates, to ensure that
  * systemd doesn't stop the resource at shutdown while pacemaker is still
  * running.
  *
  * @TODO Add start timeout
  */
 #define SYSTEMD_OVERRIDE_TEMPLATE                           \
     "[Unit]\n"                                              \
     "Description=Cluster Controlled %s\n"                   \
     "Before=pacemaker.service pacemaker_remote.service\n"   \
     "\n"                                                    \
     "[Service]\n"                                           \
     "Restart=no\n"
 
 // Temporarily use rwxr-xr-x umask when opening a file for writing
 static FILE *
 create_world_readable(const char *filename)
 {
     mode_t orig_umask = umask(S_IWGRP | S_IWOTH);
     FILE *fp = fopen(filename, "w");
 
     umask(orig_umask);
     return fp;
 }
 
 static void
 create_override_dir(const char *agent)
 {
     char *override_dir = crm_strdup_printf(SYSTEMD_OVERRIDE_ROOT
                                            "/%s.service.d", agent);
     int rc = pcmk__build_path(override_dir, 0755);
 
     if (rc != pcmk_rc_ok) {
         crm_warn("Could not create systemd override directory %s: %s",
                  override_dir, pcmk_rc_str(rc));
     }
     free(override_dir);
 }
 
 static char *
 get_override_filename(const char *agent)
 {
     return crm_strdup_printf(SYSTEMD_OVERRIDE_ROOT
                              "/%s.service.d/50-pacemaker.conf", agent);
 }
 
 static void
 systemd_create_override(const char *agent, int timeout)
 {
     FILE *file_strm = NULL;
     char *override_file = get_override_filename(agent);
 
     create_override_dir(agent);
 
     /* Ensure the override file is world-readable. This is not strictly
      * necessary, but it avoids a systemd warning in the logs.
      */
     file_strm = create_world_readable(override_file);
     if (file_strm == NULL) {
         crm_err("Cannot open systemd override file %s for writing",
                 override_file);
     } else {
         char *override = crm_strdup_printf(SYSTEMD_OVERRIDE_TEMPLATE, agent);
 
         int rc = fprintf(file_strm, "%s\n", override);
 
         free(override);
         if (rc < 0) {
             crm_perror(LOG_WARNING, "Cannot write to systemd override file %s",
                        override_file);
         }
         fflush(file_strm);
         fclose(file_strm);
         systemd_daemon_reload(timeout);
     }
 
     free(override_file);
 }
 
 static void
 systemd_remove_override(const char *agent, int timeout)
 {
     char *override_file = get_override_filename(agent);
     int rc = unlink(override_file);
 
     if (rc < 0) {
         // Stop may be called when already stopped, which is fine
         crm_perror(LOG_DEBUG, "Cannot remove systemd override file %s",
                    override_file);
     } else {
         systemd_daemon_reload(timeout);
     }
     free(override_file);
 }
 
 /*!
  * \internal
  * \brief Parse result of systemd status check
  *
  * Set a status action's exit status and execution status based on a DBus
  * property check result, and finalize the action if asynchronous.
  *
  * \param[in] name      DBus interface name for property that was checked
  * \param[in] state     Property value
  * \param[in] userdata  Status action that check was done for
  */
 static void
 parse_status_result(const char *name, const char *state, void *userdata)
 {
     svc_action_t *op = userdata;
 
     crm_trace("Resource %s has %s='%s'",
               crm_str(op->rsc), name, crm_str(state));
 
     if (pcmk__str_eq(state, "active", pcmk__str_none)) {
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
 
     } else if (pcmk__str_eq(state, "reloading", pcmk__str_none)) {
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
 
     } else if (pcmk__str_eq(state, "activating", pcmk__str_none)) {
         services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
 
     } else if (pcmk__str_eq(state, "deactivating", pcmk__str_none)) {
         services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
 
     } else {
         services__set_result(op, PCMK_OCF_NOT_RUNNING, PCMK_EXEC_DONE, state);
     }
 
     if (!(op->synchronous)) {
         services_set_op_pending(op, NULL);
         services__finalize_async_op(op);
     }
 }
 
 /*!
  * \internal
  * \brief Invoke a systemd unit, given its DBus object path
  *
  * \param[in] op    Action to execute
  * \param[in] unit  DBus object path of systemd unit to invoke
  */
 static void
 invoke_unit_by_path(svc_action_t *op, const char *unit)
 {
     const char *method = NULL;
     DBusMessage *msg = NULL;
     DBusMessage *reply = NULL;
 
     if (pcmk__str_any_of(op->action, "monitor", "status", NULL)) {
         DBusPendingCall *pending = NULL;
         char *state;
 
         state = systemd_get_property(unit, "ActiveState",
                                      (op->synchronous? NULL : parse_status_result),
                                      op, (op->synchronous? NULL : &pending),
                                      op->timeout);
         if (op->synchronous) {
             parse_status_result("ActiveState", state, op);
             free(state);
 
         } else if (pending == NULL) { // Could not get ActiveState property
             services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                  "Could not get unit state from DBus");
             services__finalize_async_op(op);
 
         } else {
             services_set_op_pending(op, pending);
         }
         return;
 
     } else if (pcmk__str_eq(op->action, "start", pcmk__str_none)) {
         method = "StartUnit";
         systemd_create_override(op->agent, op->timeout);
 
     } else if (pcmk__str_eq(op->action, "stop", pcmk__str_none)) {
         method = "StopUnit";
         systemd_remove_override(op->agent, op->timeout);
 
     } else if (pcmk__str_eq(op->action, "restart", pcmk__str_none)) {
         method = "RestartUnit";
 
     } else {
         services__set_result(op, PCMK_OCF_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
                              "Action not implemented for systemd resources");
         if (!(op->synchronous)) {
             services__finalize_async_op(op);
         }
         return;
     }
 
     crm_trace("Calling %s for unit path %s named %s",
               method, unit, crm_str(op->rsc));
 
     msg = systemd_new_method(method);
     CRM_ASSERT(msg != NULL);
 
     /* (ss) */
     {
         const char *replace_s = "replace";
         char *name = systemd_service_name(op->agent);
 
         CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &name, DBUS_TYPE_INVALID));
         CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &replace_s, DBUS_TYPE_INVALID));
 
         free(name);
     }
 
     if (op->synchronous) {
         reply = systemd_send_recv(msg, NULL, op->timeout);
         dbus_message_unref(msg);
         process_unit_method_reply(reply, op);
         if (reply != NULL) {
             dbus_message_unref(reply);
         }
 
     } else {
         DBusPendingCall *pending = systemd_send(msg, unit_method_complete, op,
                                                 op->timeout);
 
         dbus_message_unref(msg);
         if (pending == NULL) {
             services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                  "Unable to send DBus message");
             services__finalize_async_op(op);
 
         } else {
             services_set_op_pending(op, pending);
         }
     }
 }
 
 static gboolean
 systemd_timeout_callback(gpointer p)
 {
     svc_action_t * op = p;
 
     op->opaque->timerid = 0;
     crm_warn("%s operation on systemd unit %s named '%s' timed out", op->action, op->agent, op->rsc);
     services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_TIMEOUT,
                          "Systemd action did not complete within specified timeout");
     services__finalize_async_op(op);
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Execute a systemd action
  *
  * \param[in] op  Action to execute
  *
  * \return Standard Pacemaker return code
  * \retval EBUSY          Recurring operation could not be initiated
  * \retval pcmk_rc_error  Synchronous action failed
  * \retval pcmk_rc_ok     Synchronous action succeeded, or asynchronous action
  *                        should not be freed (because it already was or is
  *                        pending)
  *
  * \note If the return value for an asynchronous action is not pcmk_rc_ok, the
  *       caller is responsible for freeing the action.
  */
 int
 services__execute_systemd(svc_action_t *op)
 {
     CRM_ASSERT(op != NULL);
 
     if ((op->action == NULL) || (op->agent == NULL)) {
         services__set_result(op, PCMK_OCF_NOT_CONFIGURED, PCMK_EXEC_ERROR_FATAL,
                              "Bug in action caller");
         goto done;
     }
 
     if (!systemd_init()) {
         services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                              "No DBus connection");
         goto done;
     }
 
     crm_debug("Performing %ssynchronous %s op on systemd unit %s named '%s'",
               (op->synchronous? "" : "a"), op->action, op->agent,
               crm_str(op->rsc));
 
     if (pcmk__str_eq(op->action, "meta-data", pcmk__str_casei)) {
         op->stdout_data = systemd_unit_metadata(op->agent, op->timeout);
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
         goto done;
     }
 
     /* invoke_unit_by_name() should always override these values, which are here
      * just as a fail-safe in case there are any code paths that neglect to
      */
     services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                          "Bug in service library");
 
     if (invoke_unit_by_name(op->agent, op, NULL) == pcmk_rc_ok) {
         op->opaque->timerid = g_timeout_add(op->timeout + 5000,
                                             systemd_timeout_callback, op);
         services_add_inflight_op(op);
         return pcmk_rc_ok;
     }
 
 done:
     if (op->synchronous) {
         return (op->rc == PCMK_OCF_OK)? pcmk_rc_ok : pcmk_rc_error;
     } else {
         return services__finalize_async_op(op);
     }
 }
diff --git a/lib/services/systemd.h b/lib/services/systemd.h
index 6d5dea9dee..0d3dbe6d7f 100644
--- a/lib/services/systemd.h
+++ b/lib/services/systemd.h
@@ -1,27 +1,30 @@
 /*
  * Copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef SYSTEMD__H
 #  define SYSTEMD__H
 
 #  include <glib.h>
 #  include "crm/services.h"
 
 G_GNUC_INTERNAL GList *systemd_unit_listall(void);
 
 G_GNUC_INTERNAL
 int services__systemd_prepare(svc_action_t *op);
 
+G_GNUC_INTERNAL
+enum ocf_exitcode services__systemd2ocf(int exit_status);
+
 G_GNUC_INTERNAL
 int services__execute_systemd(svc_action_t *op);
 
 G_GNUC_INTERNAL gboolean systemd_unit_exists(const gchar * name);
 G_GNUC_INTERNAL void systemd_cleanup(void);
 
 #endif  /* SYSTEMD__H */
diff --git a/lib/services/upstart.c b/lib/services/upstart.c
index 3ad4275052..2fdc229adc 100644
--- a/lib/services/upstart.c
+++ b/lib/services/upstart.c
@@ -1,681 +1,696 @@
 /*
  * Original copyright 2010 Senko Rasic <senko.rasic@dobarkod.hr>
  *                         and Ante Karamatic <ivoks@init.hr>
  * Later changes copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/common/mainloop.h>
 
 #include <services_private.h>
 #include <upstart.h>
 #include <dbus/dbus.h>
 #include <pcmk-dbus.h>
 
 #include <glib.h>
 #include <gio/gio.h>
 
 #define BUS_NAME "com.ubuntu.Upstart"
 #define BUS_PATH "/com/ubuntu/Upstart"
 
 #define UPSTART_06_API     BUS_NAME"0_6"
 #define UPSTART_JOB_IFACE  UPSTART_06_API".Job"
 #define BUS_PROPERTY_IFACE "org.freedesktop.DBus.Properties"
 
 /*
   http://upstart.ubuntu.com/wiki/DBusInterface
 */
 static DBusConnection *upstart_proxy = NULL;
 
 /*!
  * \internal
  * \brief Prepare an Upstart action
  *
  * \param[in] op  Action to prepare
  *
  * \return Standard Pacemaker return code
  */
 int
 services__upstart_prepare(svc_action_t *op)
 {
     op->opaque->exec = strdup("upstart-dbus");
     if (op->opaque->exec == NULL) {
         return ENOMEM;
     }
     return pcmk_rc_ok;
 }
 
+/*!
+ * \internal
+ * \brief Map a Upstart result to a standard OCF result
+ *
+ * \param[in] exit_status  Upstart result
+ *
+ * \return Standard OCF result
+ */
+enum ocf_exitcode
+services__upstart2ocf(int exit_status)
+{
+    // This library uses OCF codes for Upstart actions
+    return (enum ocf_exitcode) exit_status;
+}
+
 static gboolean
 upstart_init(void)
 {
     static int need_init = 1;
 
     if (need_init) {
         need_init = 0;
         upstart_proxy = pcmk_dbus_connect();
     }
     if (upstart_proxy == NULL) {
         return FALSE;
     }
     return TRUE;
 }
 
 void
 upstart_cleanup(void)
 {
     if (upstart_proxy) {
         pcmk_dbus_disconnect(upstart_proxy);
         upstart_proxy = NULL;
     }
 }
 
 /*!
  * \internal
  * \brief Get the DBus object path corresponding to a job name
  *
  * \param[in]  arg_name  Name of job to get path for
  * \param[out] path      If not NULL, where to store DBus object path
  * \param[in]  timeout   Give up after this many seconds
  *
  * \return true if object path was found, false otherwise
  * \note The caller is responsible for freeing *path if it is non-NULL.
  */
 static bool
 object_path_for_job(const gchar *arg_name, char **path, int timeout)
 {
     /*
         com.ubuntu.Upstart0_6.GetJobByName (in String name, out ObjectPath job)
     */
     DBusError error;
     DBusMessage *msg;
     DBusMessage *reply = NULL;
     bool rc = false;
 
     if (path != NULL) {
         *path = NULL;
     }
 
     if (!upstart_init()) {
         return false;
     }
     msg = dbus_message_new_method_call(BUS_NAME, // target for the method call
                                        BUS_PATH, // object to call on
                                        UPSTART_06_API,  // interface to call on
                                        "GetJobByName"); // method name
 
     dbus_error_init(&error);
     CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_STRING, &arg_name,
                                             DBUS_TYPE_INVALID));
     reply = pcmk_dbus_send_recv(msg, upstart_proxy, &error, timeout);
     dbus_message_unref(msg);
 
     if (dbus_error_is_set(&error)) {
         crm_err("Could not get DBus object path for %s: %s",
                 arg_name, error.message);
         dbus_error_free(&error);
 
     } else if (!pcmk_dbus_type_check(reply, NULL, DBUS_TYPE_OBJECT_PATH,
                                      __func__, __LINE__)) {
         crm_err("Could not get DBus object path for %s: Invalid return type",
                 arg_name);
 
     } else {
         if (path != NULL) {
             dbus_message_get_args(reply, NULL, DBUS_TYPE_OBJECT_PATH, path,
                                   DBUS_TYPE_INVALID);
             if (*path != NULL) {
                 *path = strdup(*path);
             }
         }
         rc = true;
     }
 
     if (reply != NULL) {
         dbus_message_unref(reply);
     }
     return rc;
 }
 
 static void
 fix(char *input, const char *search, char replace)
 {
     char *match = NULL;
     int shuffle = strlen(search) - 1;
 
     while (TRUE) {
         int len, lpc;
 
         match = strstr(input, search);
         if (match == NULL) {
             break;
         }
         crm_trace("Found: %s", match);
         match[0] = replace;
         len = strlen(match) - shuffle;
         for (lpc = 1; lpc <= len; lpc++) {
             match[lpc] = match[lpc + shuffle];
         }
     }
 }
 
 static char *
 fix_upstart_name(const char *input)
 {
     char *output = strdup(input);
 
     fix(output, "_2b", '+');
     fix(output, "_2c", ',');
     fix(output, "_2d", '-');
     fix(output, "_2e", '.');
     fix(output, "_40", '@');
     fix(output, "_5f", '_');
     return output;
 }
 
 GList *
 upstart_job_listall(void)
 {
     GList *units = NULL;
     DBusMessageIter args;
     DBusMessageIter unit;
     DBusMessage *msg = NULL;
     DBusMessage *reply = NULL;
     const char *method = "GetAllJobs";
     DBusError error;
     int lpc = 0;
 
     if (upstart_init() == FALSE) {
         return NULL;
     }
 
 /*
   com.ubuntu.Upstart0_6.GetAllJobs (out <Array of ObjectPath> jobs)
 */
 
     dbus_error_init(&error);
     msg = dbus_message_new_method_call(BUS_NAME, // target for the method call
                                        BUS_PATH, // object to call on
                                        UPSTART_06_API, // interface to call on
                                        method); // method name
     CRM_ASSERT(msg != NULL);
 
     reply = pcmk_dbus_send_recv(msg, upstart_proxy, &error, DBUS_TIMEOUT_USE_DEFAULT);
     dbus_message_unref(msg);
 
     if (dbus_error_is_set(&error)) {
         crm_err("Call to %s failed: %s", method, error.message);
         dbus_error_free(&error);
         return NULL;
 
     } else if (!dbus_message_iter_init(reply, &args)) {
         crm_err("Call to %s failed: Message has no arguments", method);
         dbus_message_unref(reply);
         return NULL;
     }
 
     if(!pcmk_dbus_type_check(reply, &args, DBUS_TYPE_ARRAY, __func__, __LINE__)) {
         crm_err("Call to %s failed: Message has invalid arguments", method);
         dbus_message_unref(reply);
         return NULL;
     }
 
     dbus_message_iter_recurse(&args, &unit);
     while (dbus_message_iter_get_arg_type (&unit) != DBUS_TYPE_INVALID) {
         DBusBasicValue value;
         const char *job = NULL;
         char *path = NULL;
 
         if(!pcmk_dbus_type_check(reply, &unit, DBUS_TYPE_OBJECT_PATH, __func__, __LINE__)) {
             crm_warn("Skipping Upstart reply argument with unexpected type");
             continue;
         }
 
         dbus_message_iter_get_basic(&unit, &value);
 
         if(value.str) {
             int llpc = 0;
             path = value.str;
             job = value.str;
             while (path[llpc] != 0) {
                 if (path[llpc] == '/') {
                     job = path + llpc + 1;
                 }
                 llpc++;
             }
             lpc++;
             crm_trace("%s -> %s", path, job);
             units = g_list_append(units, fix_upstart_name(job));
         }
         dbus_message_iter_next (&unit);
     }
 
     dbus_message_unref(reply);
     crm_trace("Found %d upstart jobs", lpc);
     return units;
 }
 
 gboolean
 upstart_job_exists(const char *name)
 {
     return object_path_for_job(name, NULL, DBUS_TIMEOUT_USE_DEFAULT);
 }
 
 static char *
 get_first_instance(const gchar * job, int timeout)
 {
     char *instance = NULL;
     const char *method = "GetAllInstances";
     DBusError error;
     DBusMessage *msg;
     DBusMessage *reply;
     DBusMessageIter args;
     DBusMessageIter unit;
 
     dbus_error_init(&error);
     msg = dbus_message_new_method_call(BUS_NAME, // target for the method call
                                        job, // object to call on
                                        UPSTART_JOB_IFACE, // interface to call on
                                        method); // method name
     CRM_ASSERT(msg != NULL);
 
     dbus_message_append_args(msg, DBUS_TYPE_INVALID);
     reply = pcmk_dbus_send_recv(msg, upstart_proxy, &error, timeout);
     dbus_message_unref(msg);
 
     if (dbus_error_is_set(&error)) {
         crm_err("Call to %s failed: %s", method, error.message);
         dbus_error_free(&error);
         goto done;
 
     } else if(reply == NULL) {
         crm_err("Call to %s failed: no reply", method);
         goto done;
 
     } else if (!dbus_message_iter_init(reply, &args)) {
         crm_err("Call to %s failed: Message has no arguments", method);
         goto done;
     }
 
     if(!pcmk_dbus_type_check(reply, &args, DBUS_TYPE_ARRAY, __func__, __LINE__)) {
         crm_err("Call to %s failed: Message has invalid arguments", method);
         goto done;
     }
 
     dbus_message_iter_recurse(&args, &unit);
     if(pcmk_dbus_type_check(reply, &unit, DBUS_TYPE_OBJECT_PATH, __func__, __LINE__)) {
         DBusBasicValue value;
 
         dbus_message_iter_get_basic(&unit, &value);
 
         if(value.str) {
             instance = strdup(value.str);
             crm_trace("Result: %s", instance);
         }
     }
 
   done:
     if(reply) {
         dbus_message_unref(reply);
     }
     return instance;
 }
 
 /*!
  * \internal
  * \brief Parse result of Upstart status check
  *
  * \param[in] name      DBus interface name for property that was checked
  * \param[in] state     Property value
  * \param[in] userdata  Status action that check was done for
  */
 static void
 parse_status_result(const char *name, const char *state, void *userdata)
 {
     svc_action_t *op = userdata;
 
     if (pcmk__str_eq(state, "running", pcmk__str_none)) {
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
     } else {
         services__set_result(op, PCMK_OCF_NOT_RUNNING, PCMK_EXEC_DONE, state);
     }
 
     if (!(op->synchronous)) {
         services_set_op_pending(op, NULL);
         services__finalize_async_op(op);
     }
 }
 
 #define METADATA_FORMAT                                                     \
     "<?xml version=\"1.0\"?>\n"                                             \
     "<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"                   \
     "<resource-agent name=\"%s\" version=\"" PCMK_DEFAULT_AGENT_VERSION "\">\n" \
     "  <version>1.1</version>\n"                                            \
     "  <longdesc lang=\"en\">\n"                                            \
     "    Upstart agent for controlling the system %s service\n"             \
     "  </longdesc>\n"                                                       \
     "  <shortdesc lang=\"en\">Upstart job for %s</shortdesc>\n"             \
     "  <parameters/>\n"                                                     \
     "  <actions>\n"                                                         \
     "    <action name=\"start\"     timeout=\"15\" />\n"                    \
     "    <action name=\"stop\"      timeout=\"15\" />\n"                    \
     "    <action name=\"status\"    timeout=\"15\" />\n"                    \
     "    <action name=\"restart\"   timeout=\"15\" />\n"                    \
     "    <action name=\"monitor\"   timeout=\"15\" interval=\"15\" start-delay=\"15\" />\n" \
     "    <action name=\"meta-data\" timeout=\"5\" />\n"                     \
     "  </actions>\n"                                                        \
     "  <special tag=\"upstart\"/>\n"                                        \
     "</resource-agent>\n"
 
 static char *
 upstart_job_metadata(const char *name)
 {
     return crm_strdup_printf(METADATA_FORMAT, name, name, name);
 }
 
 /*!
  * \internal
  * \brief Set an action result based on a method error
  *
  * \param[in] op     Action to set result for
  * \param[in] error  Method error
  */
 static void
 set_result_from_method_error(svc_action_t *op, const DBusError *error)
 {
     services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                          "Unable to invoke Upstart DBus method");
 
     if (strstr(error->name, UPSTART_06_API ".Error.UnknownInstance")) {
 
         if (pcmk__str_eq(op->action, "stop", pcmk__str_casei)) {
             crm_trace("Masking stop failure (%s) for %s "
                       "because unknown service can be considered stopped",
                       error->name, crm_str(op->rsc));
             services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
             return;
         }
 
         services__set_result(op, PCMK_OCF_NOT_INSTALLED,
                              PCMK_EXEC_NOT_INSTALLED, "Upstart job not found");
 
     } else if (pcmk__str_eq(op->action, "start", pcmk__str_casei)
                && strstr(error->name, UPSTART_06_API ".Error.AlreadyStarted")) {
         crm_trace("Masking start failure (%s) for %s "
                   "because already started resource is OK",
                   error->name, crm_str(op->rsc));
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
         return;
     }
 
     crm_err("DBus request for %s of Upstart job %s for resource %s failed: %s",
             op->action, op->agent, crm_str(op->rsc), error->message);
 }
 
 /*!
  * \internal
  * \brief Process the completion of an asynchronous job start, stop, or restart
  *
  * \param[in] pending    If not NULL, DBus call associated with request
  * \param[in] user_data  Action that was executed
  */
 static void
 job_method_complete(DBusPendingCall *pending, void *user_data)
 {
     DBusError error;
     DBusMessage *reply = NULL;
     svc_action_t *op = user_data;
 
     // Grab the reply
     if (pending != NULL) {
         reply = dbus_pending_call_steal_reply(pending);
     }
 
     // Determine result
     dbus_error_init(&error);
     if (pcmk_dbus_find_error(pending, reply, &error)) {
         set_result_from_method_error(op, &error);
         dbus_error_free(&error);
 
     } else if (pcmk__str_eq(op->action, "stop", pcmk__str_none)) {
         // Call has no return value
         crm_debug("DBus request for stop of %s succeeded", crm_str(op->rsc));
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
 
     } else if (!pcmk_dbus_type_check(reply, NULL, DBUS_TYPE_OBJECT_PATH,
                                      __func__, __LINE__)) {
         crm_warn("DBus request for %s of %s succeeded but "
                  "return type was unexpected", op->action, crm_str(op->rsc));
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
 
     } else {
         const char *path = NULL;
 
         dbus_message_get_args(reply, NULL, DBUS_TYPE_OBJECT_PATH, &path,
                               DBUS_TYPE_INVALID);
         crm_debug("DBus request for %s of %s using %s succeeded",
                   op->action, crm_str(op->rsc), path);
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
     }
 
     // The call is no longer pending
     CRM_LOG_ASSERT(pending == op->opaque->pending);
     services_set_op_pending(op, NULL);
 
     // Finalize action
     services__finalize_async_op(op);
     if (reply != NULL) {
         dbus_message_unref(reply);
     }
 }
 
 /*!
  * \internal
  * \brief Execute an Upstart action
  *
  * \param[in] op  Action to execute
  *
  * \return Standard Pacemaker return code
  * \retval EBUSY          Recurring operation could not be initiated
  * \retval pcmk_rc_error  Synchronous action failed
  * \retval pcmk_rc_ok     Synchronous action succeeded, or asynchronous action
  *                        should not be freed (because it already was or is
  *                        pending)
  *
  * \note If the return value for an asynchronous action is not pcmk_rc_ok, the
  *       caller is responsible for freeing the action.
  */
 int
 services__execute_upstart(svc_action_t *op)
 {
     char *job = NULL;
     int arg_wait = TRUE;
     const char *arg_env = "pacemaker=1";
     const char *action = op->action;
 
     DBusError error;
     DBusMessage *msg = NULL;
     DBusMessage *reply = NULL;
     DBusMessageIter iter, array_iter;
 
     CRM_ASSERT(op != NULL);
 
     if ((op->action == NULL) || (op->agent == NULL)) {
         services__set_result(op, PCMK_OCF_NOT_CONFIGURED, PCMK_EXEC_ERROR_FATAL,
                              "Bug in action caller");
         goto cleanup;
     }
 
     if (!upstart_init()) {
         services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                              "No DBus connection");
         goto cleanup;
     }
 
     if (pcmk__str_eq(op->action, "meta-data", pcmk__str_casei)) {
         op->stdout_data = upstart_job_metadata(op->agent);
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
         goto cleanup;
     }
 
     if (!object_path_for_job(op->agent, &job, op->timeout)) {
         if (pcmk__str_eq(action, "stop", pcmk__str_none)) {
             services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
         } else {
             services__set_result(op, PCMK_OCF_NOT_INSTALLED,
                                  PCMK_EXEC_NOT_INSTALLED,
                                  "Upstart job not found");
         }
         goto cleanup;
     }
 
     if (job == NULL) {
         // Shouldn't normally be possible -- maybe a memory error
         op->rc = PCMK_OCF_UNKNOWN_ERROR;
         op->status = PCMK_EXEC_ERROR;
         goto cleanup;
     }
 
     if (pcmk__strcase_any_of(op->action, "monitor", "status", NULL)) {
         DBusPendingCall *pending = NULL;
         char *state = NULL;
         char *path = get_first_instance(job, op->timeout);
 
         services__set_result(op, PCMK_OCF_NOT_RUNNING, PCMK_EXEC_DONE,
                              "No Upstart job instances found");
         if (path == NULL) {
             goto cleanup;
         }
         state = pcmk_dbus_get_property(upstart_proxy, BUS_NAME, path,
                                        UPSTART_06_API ".Instance", "state",
                                        op->synchronous? NULL : parse_status_result,
                                        op,
                                        op->synchronous? NULL : &pending,
                                        op->timeout);
         free(path);
 
         if (op->synchronous) {
             parse_status_result("state", state, op);
             free(state);
 
         } else if (pending == NULL) {
             services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                  "Could not get job state from DBus");
 
         } else { // Successfully initiated async op
             free(job);
             services_set_op_pending(op, pending);
             services_add_inflight_op(op);
             return pcmk_rc_ok;
         }
 
         goto cleanup;
 
     } else if (pcmk__str_eq(action, "start", pcmk__str_none)) {
         action = "Start";
 
     } else if (pcmk__str_eq(action, "stop", pcmk__str_none)) {
         action = "Stop";
 
     } else if (pcmk__str_eq(action, "restart", pcmk__str_none)) {
         action = "Restart";
 
     } else {
         services__set_result(op, PCMK_OCF_UNIMPLEMENT_FEATURE,
                              PCMK_EXEC_ERROR_HARD,
                              "Action not implemented for Upstart resources");
         goto cleanup;
     }
 
     // Initialize rc/status in case called functions don't set them
     services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_DONE,
                          "Bug in service library");
 
     crm_debug("Calling %s for %s on %s", action, crm_str(op->rsc), job);
 
     msg = dbus_message_new_method_call(BUS_NAME, // target for the method call
                                        job, // object to call on
                                        UPSTART_JOB_IFACE, // interface to call on
                                        action); // method name
     CRM_ASSERT(msg != NULL);
 
     dbus_message_iter_init_append (msg, &iter);
     CRM_LOG_ASSERT(dbus_message_iter_open_container(&iter,
                                                     DBUS_TYPE_ARRAY,
                                                     DBUS_TYPE_STRING_AS_STRING,
                                                     &array_iter));
     CRM_LOG_ASSERT(dbus_message_iter_append_basic(&array_iter,
                                                   DBUS_TYPE_STRING, &arg_env));
     CRM_LOG_ASSERT(dbus_message_iter_close_container(&iter, &array_iter));
     CRM_LOG_ASSERT(dbus_message_append_args(msg, DBUS_TYPE_BOOLEAN, &arg_wait,
                                             DBUS_TYPE_INVALID));
 
     if (!(op->synchronous)) {
         DBusPendingCall *pending = pcmk_dbus_send(msg, upstart_proxy,
                                                   job_method_complete, op,
                                                   op->timeout);
 
         if (pending == NULL) {
             services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                  "Unable to send DBus message");
             goto cleanup;
 
         } else { // Successfully initiated async op
             free(job);
             services_set_op_pending(op, pending);
             services_add_inflight_op(op);
             return pcmk_rc_ok;
         }
     }
 
     // Synchronous call
 
     dbus_error_init(&error);
     reply = pcmk_dbus_send_recv(msg, upstart_proxy, &error, op->timeout);
 
     if (dbus_error_is_set(&error)) {
         set_result_from_method_error(op, &error);
         dbus_error_free(&error);
 
     } else if (pcmk__str_eq(op->action, "stop", pcmk__str_none)) {
         // DBus call does not return a value
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
 
     } else if (!pcmk_dbus_type_check(reply, NULL, DBUS_TYPE_OBJECT_PATH,
                                      __func__, __LINE__)) {
         crm_warn("Call to %s passed but return type was unexpected", op->action);
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
 
     } else {
         const char *path = NULL;
 
         dbus_message_get_args(reply, NULL, DBUS_TYPE_OBJECT_PATH, &path,
                               DBUS_TYPE_INVALID);
         crm_info("Call to %s passed: %s", op->action, path);
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
     }
 
 cleanup:
     free(job);
     if (msg != NULL) {
         dbus_message_unref(msg);
     }
     if (reply != NULL) {
         dbus_message_unref(reply);
     }
 
     if (op->synchronous) {
         return (op->rc == PCMK_OCF_OK)? pcmk_rc_ok : pcmk_rc_error;
     } else {
         return services__finalize_async_op(op);
     }
 }
diff --git a/lib/services/upstart.h b/lib/services/upstart.h
index fa68a2efea..b6c4effc07 100644
--- a/lib/services/upstart.h
+++ b/lib/services/upstart.h
@@ -1,28 +1,31 @@
 /*
  * Copyright 2010 Senko Rasic <senko.rasic@dobarkod.hr>
  * Copyright 2010 Ante Karamatic <ivoks@init.hr>
  * Later changes copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 #ifndef UPSTART__H
 #  define UPSTART__H
 
 #  include <glib.h>
 #  include "crm/services.h"
 
 G_GNUC_INTERNAL GList *upstart_job_listall(void);
 
 G_GNUC_INTERNAL
 int services__upstart_prepare(svc_action_t *op);
 
+G_GNUC_INTERNAL
+enum ocf_exitcode services__upstart2ocf(int exit_status);
+
 G_GNUC_INTERNAL
 int services__execute_upstart(svc_action_t *op);
 
 G_GNUC_INTERNAL gboolean upstart_job_exists(const gchar * name);
 G_GNUC_INTERNAL void upstart_cleanup(void);
 
 #endif  /* UPSTART__H */
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 1526ea8f63..816f5a6e8d 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -1,2003 +1,2000 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm_resource.h>
 #include <crm/common/ipc_controld.h>
 #include <crm/common/lists_internal.h>
 #include <crm/services_internal.h>
 
 resource_checks_t *
 cli_check_resource(pe_resource_t *rsc, char *role_s, char *managed)
 {
     pe_resource_t *parent = uber_parent(rsc);
     resource_checks_t *rc = calloc(1, sizeof(resource_checks_t));
 
     if (role_s) {
         enum rsc_role_e role = text2role(role_s);
 
         if (role == RSC_ROLE_STOPPED) {
             rc->flags |= rsc_remain_stopped;
         } else if (pcmk_is_set(parent->flags, pe_rsc_promotable) &&
                    (role == RSC_ROLE_UNPROMOTED)) {
             rc->flags |= rsc_unpromotable;
         }
     }
 
     if (managed && !crm_is_true(managed)) {
         rc->flags |= rsc_unmanaged;
     }
 
     if (rsc->lock_node) {
         rc->lock_node = rsc->lock_node->details->uname;
     }
 
     rc->rsc = rsc;
     return rc;
 }
 
 static GList *
 build_node_info_list(pe_resource_t *rsc)
 {
     GList *retval = NULL;
 
     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
         pe_resource_t *child = (pe_resource_t *) iter->data;
 
         for (GList *iter2 = child->running_on; iter2 != NULL; iter2 = iter2->next) {
             pe_node_t *node = (pe_node_t *) iter2->data;
             node_info_t *ni = calloc(1, sizeof(node_info_t));
             ni->node_name = node->details->uname;
             ni->promoted = pcmk_is_set(rsc->flags, pe_rsc_promotable) &&
                            child->fns->state(child, TRUE) == RSC_ROLE_PROMOTED;
 
             retval = g_list_prepend(retval, ni);
         }
     }
 
     return retval;
 }
 
 GList *
 cli_resource_search(pe_resource_t *rsc, const char *requested_name,
                     pe_working_set_t *data_set)
 {
     GList *retval = NULL;
     pe_resource_t *parent = uber_parent(rsc);
 
     if (pe_rsc_is_clone(rsc)) {
         retval = build_node_info_list(rsc);
 
     /* The anonymous clone children's common ID is supplied */
     } else if (pe_rsc_is_clone(parent)
                && !pcmk_is_set(rsc->flags, pe_rsc_unique)
                && rsc->clone_name
                && pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
                && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {
 
         retval = build_node_info_list(parent);
 
     } else if (rsc->running_on != NULL) {
         for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
             pe_node_t *node = (pe_node_t *) iter->data;
             node_info_t *ni = calloc(1, sizeof(node_info_t));
             ni->node_name = node->details->uname;
             ni->promoted = (rsc->fns->state(rsc, TRUE) == RSC_ROLE_PROMOTED);
 
             retval = g_list_prepend(retval, ni);
         }
     }
 
     return retval;
 }
 
 #define XPATH_MAX 1024
 
 // \return Standard Pacemaker return code
 static int
 find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
                    const char *rsc, const char *attr_set_type, const char *set_name,
                    const char *attr_id, const char *attr_name, char **value)
 {
     int offset = 0;
     int rc = pcmk_rc_ok;
     xmlNode *xml_search = NULL;
     char *xpath_string = NULL;
 
     if(value) {
         *value = NULL;
     }
 
     if(the_cib == NULL) {
         return ENOTCONN;
     }
 
     xpath_string = calloc(1, XPATH_MAX);
     offset +=
         snprintf(xpath_string + offset, XPATH_MAX - offset, "%s", get_object_path("resources"));
 
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//*[@id=\"%s\"]", rsc);
 
     if (attr_set_type) {
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s", attr_set_type);
         if (set_name) {
             offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "[@id=\"%s\"]", set_name);
         }
     }
 
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//nvpair[");
     if (attr_id) {
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@id=\"%s\"", attr_id);
     }
 
     if (attr_name) {
         if (attr_id) {
             offset += snprintf(xpath_string + offset, XPATH_MAX - offset, " and ");
         }
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@name=\"%s\"", attr_name);
     }
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "]");
     CRM_LOG_ASSERT(offset > 0);
 
     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
     rc = pcmk_legacy2rc(rc);
 
     if (rc != pcmk_rc_ok) {
         goto done;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_has_children(xml_search)) {
         xmlNode *child = NULL;
 
         rc = ENOTUNIQ;
         out->info(out, "Multiple attributes match name=%s", attr_name);
 
         for (child = pcmk__xml_first_child(xml_search); child != NULL;
              child = pcmk__xml_next(child)) {
             out->info(out, "  Value: %s \t(id=%s)",
                       crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
         }
 
         out->spacer(out);
 
     } else if(value) {
         const char *tmp = crm_element_value(xml_search, attr);
 
         if (tmp) {
             *value = strdup(tmp);
         }
     }
 
   done:
     free(xpath_string);
     free_xml(xml_search);
     return rc;
 }
 
 /* PRIVATE. Use the find_matching_attr_resources instead. */
 static void
 find_matching_attr_resources_recursive(pcmk__output_t *out, GList/* <pe_resource_t*> */ ** result,
                                        pe_resource_t * rsc, const char * rsc_id,
                                        const char * attr_set, const char * attr_set_type,
                                        const char * attr_id, const char * attr_name,
                                        cib_t * cib, const char * cmd, int depth)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = clone_strip(rsc->id);
     char *local_attr_id = NULL;
 
     /* visit the children */
     for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
         find_matching_attr_resources_recursive(out, result, (pe_resource_t*)gIter->data,
                                                rsc_id, attr_set, attr_set_type,
                                                attr_id, attr_name, cib, cmd, depth+1);
         /* do it only once for clones */
         if(pe_clone == rsc->variant) {
             break;
         }
     }
 
     rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
                             attr_set, attr_id, attr_name, &local_attr_id);
     /* Post-order traversal. 
      * The root is always on the list and it is the last item. */
     if((0 == depth) || (pcmk_rc_ok == rc)) {
         /* push the head */
         *result = g_list_append(*result, rsc);
     }
 
     free(local_attr_id);
     free(lookup_id);
 }
 
 
 /* The result is a linearized pre-ordered tree of resources. */
 static GList/*<pe_resource_t*>*/ *
 find_matching_attr_resources(pcmk__output_t *out, pe_resource_t * rsc,
                              const char * rsc_id, const char * attr_set,
                              const char * attr_set_type, const char * attr_id,
                              const char * attr_name, cib_t * cib, const char * cmd,
                              gboolean force)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = NULL;
     char *local_attr_id = NULL;
     GList * result = NULL;
     /* If --force is used, update only the requested resource (clone or primitive).
      * Otherwise, if the primitive has the attribute, use that.
      * Otherwise use the clone. */
     if(force == TRUE) {
         return g_list_append(result, rsc);
     }
     if(rsc->parent && pe_clone == rsc->parent->variant) {
         int rc = pcmk_rc_ok;
         char *local_attr_id = NULL;
         rc = find_resource_attr(out, cib, XML_ATTR_ID, rsc_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &local_attr_id);
         free(local_attr_id);
 
         if(rc != pcmk_rc_ok) {
             rsc = rsc->parent;
             out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
                       cmd, attr_name, rsc->id, rsc_id);
         }
         return g_list_append(result, rsc);
     } else if(rsc->parent == NULL && rsc->children && pe_clone == rsc->variant) {
         pe_resource_t *child = rsc->children->data;
 
         if(child->variant == pe_native) {
             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
             rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
                                     attr_set, attr_id, attr_name, &local_attr_id);
 
             if(rc == pcmk_rc_ok) {
                 rsc = child;
                 out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
                           attr_name, lookup_id, cmd, rsc_id);
             }
 
             free(local_attr_id);
             free(lookup_id);
         }
         return g_list_append(result, rsc);
     }
     /* If the resource is a group ==> children inherit the attribute if defined. */
     find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
                                            attr_set_type, attr_id, attr_name,
                                            cib, cmd, 0);
     return result;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_set_type,
                               const char *attr_id, const char *attr_name,
                               const char *attr_value, gboolean recursive,
                               cib_t *cib, int cib_options,
                               pe_working_set_t *data_set, gboolean force)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     static bool need_init = TRUE;
 
     char *local_attr_id = NULL;
     char *local_attr_set = NULL;
 
     GList/*<pe_resource_t*>*/ *resources = NULL;
     const char *common_attr_id = attr_id;
 
     if (attr_id == NULL && force == FALSE) {
         find_resource_attr (out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
                             NULL, NULL, attr_name, NULL);
     }
 
     if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
         if (force == FALSE) {
             rc = find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id,
                                     XML_TAG_META_SETS, attr_set, attr_id,
                                     attr_name, &local_attr_id);
             if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
                 out->err(out, "WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)",
                          uber_parent(rsc)->id, attr_name, local_attr_id);
                 out->err(out, "         Delete '%s' first or use the force option to override",
                          local_attr_id);
             }
             free(local_attr_id);
             if (rc == pcmk_rc_ok) {
                 return ENOTUNIQ;
             }
         }
         resources = g_list_append(resources, rsc);
 
     } else {
         resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
                                                  attr_id, attr_name, cib, "update", force);
     }
 
     /* If either attr_set or attr_id is specified,
      * one clearly intends to modify a single resource.
      * It is the last item on the resource list.*/
     for(GList *gIter = (attr_set||attr_id) ? g_list_last(resources) : resources
             ; gIter; gIter = gIter->next) {
         char *lookup_id = NULL;
 
         xmlNode *xml_top = NULL;
         xmlNode *xml_obj = NULL;
         local_attr_id = NULL;
         local_attr_set = NULL;
 
         rsc = (pe_resource_t*)gIter->data;
         attr_id = common_attr_id;
 
         lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
         rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &local_attr_id);
 
         if (rc == pcmk_rc_ok) {
             crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
             attr_id = local_attr_id;
 
         } else if (rc != ENXIO) {
             free(lookup_id);
             free(local_attr_id);
             g_list_free(resources);
             return rc;
 
         } else {
             const char *tag = crm_element_name(rsc->xml);
 
             if (attr_set == NULL) {
                 local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
                                                    attr_set_type);
                 attr_set = local_attr_set;
             }
             if (attr_id == NULL) {
                 local_attr_id = crm_strdup_printf("%s-%s", attr_set, attr_name);
                 attr_id = local_attr_id;
             }
 
             xml_top = create_xml_node(NULL, tag);
             crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);
 
             xml_obj = create_xml_node(xml_top, attr_set_type);
             crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
         }
 
         xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value);
         if (xml_top == NULL) {
             xml_top = xml_obj;
         }
 
         crm_log_xml_debug(xml_top, "Update");
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
         rc = pcmk_legacy2rc(rc);
 
         if (rc == pcmk_rc_ok) {
             out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s", lookup_id, local_attr_id,
                       attr_set ? " set=" : "", attr_set ? attr_set : "",
                       attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
         }
 
         free_xml(xml_top);
 
         free(lookup_id);
         free(local_attr_id);
         free(local_attr_set);
 
         if(recursive && pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
             GList *lpc = NULL;
 
             if(need_init) {
                 need_init = FALSE;
                 pcmk__unpack_constraints(data_set);
                 pe__clear_resource_flags_on_all(data_set, pe_rsc_allocating);
             }
 
             crm_debug("Looking for dependencies %p", rsc->rsc_cons_lhs);
             pe__set_resource_flags(rsc, pe_rsc_allocating);
             for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
                 pcmk__colocation_t *cons = (pcmk__colocation_t *) lpc->data;
                 pe_resource_t *peer = cons->rsc_lh;
 
                 crm_debug("Checking %s %d", cons->id, cons->score);
                 if (cons->score > 0 && !pcmk_is_set(peer->flags, pe_rsc_allocating)) {
                     /* Don't get into colocation loops */
                     crm_debug("Setting %s=%s for dependent resource %s", attr_name, attr_value, peer->id);
                     cli_resource_update_attribute(peer, peer->id, NULL, attr_set_type,
                                                   NULL, attr_name, attr_value, recursive,
                                                   cib, cib_options, data_set, force);
                 }
             }
         }
     }
     g_list_free(resources);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_set_type,
                               const char *attr_id, const char *attr_name,
                               cib_t *cib, int cib_options,
                               pe_working_set_t *data_set, gboolean force)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     GList/*<pe_resource_t*>*/ *resources = NULL;
 
     if (attr_id == NULL && force == FALSE) {
         find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
                            NULL, NULL, attr_name, NULL);
     }
 
     if(pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
         resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
                                                  attr_id, attr_name, cib, "delete", force);
     } else {
         resources = g_list_append(resources, rsc);
     }
 
     for(GList *gIter = resources; gIter; gIter = gIter->next) {
         char *lookup_id = NULL;
         xmlNode *xml_obj = NULL;
         char *local_attr_id = NULL;
 
         rsc = (pe_resource_t*)gIter->data;
 
         lookup_id = clone_strip(rsc->id);
         rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &local_attr_id);
 
         if (rc == ENXIO) {
             free(lookup_id);
             rc = pcmk_rc_ok;
             continue;
 
         } else if (rc != pcmk_rc_ok) {
             free(lookup_id);
             g_list_free(resources);
             return rc;
         }
 
         if (attr_id == NULL) {
             attr_id = local_attr_id;
         }
 
         xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, NULL);
         crm_log_xml_debug(xml_obj, "Delete");
 
         CRM_ASSERT(cib);
         rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
         rc = pcmk_legacy2rc(rc);
 
         if (rc == pcmk_rc_ok) {
             out->info(out, "Deleted '%s' option: id=%s%s%s%s%s", lookup_id, local_attr_id,
                       attr_set ? " set=" : "", attr_set ? attr_set : "",
                       attr_name ? " name=" : "", attr_name ? attr_name : "");
         }
 
         free(lookup_id);
         free_xml(xml_obj);
         free(local_attr_id);
     }
     g_list_free(resources);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
                 const char *host_uname, const char *rsc_id, pe_working_set_t *data_set)
 {
     pcmk__output_t *out = data_set->priv;
     const char *router_node = host_uname;
     const char *rsc_api_id = NULL;
     const char *rsc_long_id = NULL;
     const char *rsc_class = NULL;
     const char *rsc_provider = NULL;
     const char *rsc_type = NULL;
     bool cib_only = false;
     pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
         out->err(out, "Resource %s not found", rsc_id);
         return ENXIO;
 
     } else if (rsc->variant != pe_native) {
         out->err(out, "We can only process primitive resources, not %s", rsc_id);
         return EINVAL;
     }
 
     rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
     rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
     if ((rsc_class == NULL) || (rsc_type == NULL)) {
         out->err(out, "Resource %s does not have a class and type", rsc_id);
         return EINVAL;
     }
 
     {
         pe_node_t *node = pe_find_node(data_set->nodes, host_uname);
 
         if (node == NULL) {
             out->err(out, "Node %s not found", host_uname);
             return pcmk_rc_node_unknown;
         }
 
         if (!(node->details->online)) {
             if (do_fail_resource) {
                 out->err(out, "Node %s is not online", host_uname);
                 return ENOTCONN;
             } else {
                 cib_only = true;
             }
         }
         if (!cib_only && pe__is_guest_or_remote_node(node)) {
             node = pe__current_node(node->details->remote_rsc);
             if (node == NULL) {
                 out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
                          host_uname);
                 return ENOTCONN;
             }
             router_node = node->details->uname;
         }
     }
 
     if (rsc->clone_name) {
         rsc_api_id = rsc->clone_name;
         rsc_long_id = rsc->id;
     } else {
         rsc_api_id = rsc->id;
     }
     if (do_fail_resource) {
         return pcmk_controld_api_fail(controld_api, host_uname, router_node,
                                       rsc_api_id, rsc_long_id,
                                       rsc_class, rsc_provider, rsc_type);
     } else {
         return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
                                          rsc_api_id, rsc_long_id, rsc_class,
                                          rsc_provider, rsc_type, cib_only);
     }
 }
 
 /*!
  * \internal
  * \brief Get resource name as used in failure-related node attributes
  *
  * \param[in] rsc  Resource to check
  *
  * \return Newly allocated string containing resource's fail name
  * \note The caller is responsible for freeing the result.
  */
 static inline char *
 rsc_fail_name(pe_resource_t *rsc)
 {
     const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
 
     return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
                   const char *rsc_id, pe_working_set_t *data_set)
 {
     int rc = pcmk_rc_ok;
 
     /* Erase the resource's entire LRM history in the CIB, even if we're only
      * clearing a single operation's fail count. If we erased only entries for a
      * single operation, we might wind up with a wrong idea of the current
      * resource state, and we might not re-probe the resource.
      */
     rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, data_set);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     crm_trace("Processing %d mainloop inputs",
               pcmk_controld_api_replies_expected(controld_api));
     while (g_main_context_iteration(NULL, FALSE)) {
         crm_trace("Processed mainloop input, %d still remaining",
                   pcmk_controld_api_replies_expected(controld_api));
     }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
                    const char *node_name, const char *rsc_id, const char *operation,
                    const char *interval_spec, pe_working_set_t *data_set)
 {
     int rc = pcmk_rc_ok;
     const char *failed_value = NULL;
     const char *failed_id = NULL;
     const char *interval_ms_s = NULL;
     GHashTable *rscs = NULL;
     GHashTableIter iter;
 
     /* Create a hash table to use as a set of resources to clean. This lets us
      * clean each resource only once (per node) regardless of how many failed
      * operations it has.
      */
     rscs = pcmk__strkey_table(NULL, NULL);
 
     // Normalize interval to milliseconds for comparison to history entry
     if (operation) {
         interval_ms_s = crm_strdup_printf("%u",
                                           crm_parse_interval_spec(interval_spec));
     }
 
     for (xmlNode *xml_op = pcmk__xml_first_child(data_set->failed);
          xml_op != NULL;
          xml_op = pcmk__xml_next(xml_op)) {
 
         failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
         if (failed_id == NULL) {
             // Malformed history entry, should never happen
             continue;
         }
 
         // No resource specified means all resources match
         if (rsc_id) {
             pe_resource_t *fail_rsc = pe_find_resource_with_flags(data_set->resources,
                                                                   failed_id,
                                                                   pe_find_renamed|pe_find_anon);
 
             if (!fail_rsc || !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
                 continue;
             }
         }
 
         // Host name should always have been provided by this point
         failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
         if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
             continue;
         }
 
         // No operation specified means all operations match
         if (operation) {
             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
             if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
                 continue;
             }
 
             // Interval (if operation was specified) defaults to 0 (not all)
             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
             if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
                 continue;
             }
         }
 
         g_hash_table_add(rscs, (gpointer) failed_id);
     }
 
     g_hash_table_iter_init(&iter, rscs);
     while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
         crm_debug("Erasing failures of %s on %s", failed_id, node_name);
         rc = clear_rsc_history(controld_api, node_name, failed_id, data_set);
         if (rc != pcmk_rc_ok) {
             return rc;
         }
     }
     g_hash_table_destroy(rscs);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_fail_attrs(pe_resource_t *rsc, const char *operation,
                      const char *interval_spec, pe_node_t *node)
 {
     int rc = pcmk_rc_ok;
     int attr_options = pcmk__node_attr_none;
     char *rsc_name = rsc_fail_name(rsc);
 
     if (pe__is_guest_or_remote_node(node)) {
         attr_options |= pcmk__node_attr_remote;
     }
     rc = pcmk__node_attr_request_clear(NULL, node->details->uname, rsc_name,
                                        operation, interval_spec, NULL,
                                        attr_options);
     free(rsc_name);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
                     pe_resource_t *rsc, const char *operation,
                     const char *interval_spec, bool just_failures,
                     pe_working_set_t *data_set, gboolean force)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     pe_node_t *node = NULL;
 
     if (rsc == NULL) {
         return ENXIO;
 
     } else if (rsc->children) {
         GList *lpc = NULL;
 
         for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
             pe_resource_t *child = (pe_resource_t *) lpc->data;
 
             rc = cli_resource_delete(controld_api, host_uname, child, operation,
                                      interval_spec, just_failures, data_set,
                                      force);
             if (rc != pcmk_rc_ok) {
                 return rc;
             }
         }
         return pcmk_rc_ok;
 
     } else if (host_uname == NULL) {
         GList *lpc = NULL;
         GList *nodes = g_hash_table_get_values(rsc->known_on);
 
         if(nodes == NULL && force) {
             nodes = pcmk__copy_node_list(data_set->nodes, false);
 
         } else if(nodes == NULL && rsc->exclusive_discover) {
             GHashTableIter iter;
             pe_node_t *node = NULL;
 
             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
             while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
                 if(node->weight >= 0) {
                     nodes = g_list_prepend(nodes, node);
                 }
             }
 
         } else if(nodes == NULL) {
             nodes = g_hash_table_get_values(rsc->allowed_nodes);
         }
 
         for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
             node = (pe_node_t *) lpc->data;
 
             if (node->details->online) {
                 rc = cli_resource_delete(controld_api, node->details->uname,
                                          rsc, operation, interval_spec,
                                          just_failures, data_set, force);
             }
             if (rc != pcmk_rc_ok) {
                 g_list_free(nodes);
                 return rc;
             }
         }
 
         g_list_free(nodes);
         return pcmk_rc_ok;
     }
 
     node = pe_find_node(data_set->nodes, host_uname);
 
     if (node == NULL) {
         out->err(out, "Unable to clean up %s because node %s not found",
                  rsc->id, host_uname);
         return ENODEV;
     }
 
     if (!node->details->rsc_discovery_enabled) {
         out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
                  rsc->id, host_uname);
         return EOPNOTSUPP;
     }
 
     if (controld_api == NULL) {
         out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
                  rsc->id, host_uname);
         return pcmk_rc_ok;
     }
 
     rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Unable to clean up %s failures on %s: %s",
                  rsc->id, host_uname, pcmk_rc_str(rc));
         return rc;
     }
 
     if (just_failures) {
         rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
                                 interval_spec, data_set);
     } else {
         rc = clear_rsc_history(controld_api, host_uname, rsc->id, data_set);
     }
     if (rc != pcmk_rc_ok) {
         out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
                  rsc->id, host_uname, pcmk_strerror(rc));
     } else {
         out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
     }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
                 const char *operation, const char *interval_spec,
                 pe_working_set_t *data_set)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     int attr_options = pcmk__node_attr_none;
     const char *display_name = node_name? node_name : "all nodes";
 
     if (controld_api == NULL) {
         out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
                   display_name);
         return rc;
     }
 
     if (node_name) {
         pe_node_t *node = pe_find_node(data_set->nodes, node_name);
 
         if (node == NULL) {
             out->err(out, "Unknown node: %s", node_name);
             return ENXIO;
         }
         if (pe__is_guest_or_remote_node(node)) {
             attr_options |= pcmk__node_attr_remote;
         }
     }
 
     rc = pcmk__node_attr_request_clear(NULL, node_name, NULL, operation,
                                        interval_spec, NULL, attr_options);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Unable to clean up all failures on %s: %s",
                  display_name, pcmk_rc_str(rc));
         return rc;
     }
 
     if (node_name) {
         rc = clear_rsc_failures(out, controld_api, node_name, NULL,
                                 operation, interval_spec, data_set);
         if (rc != pcmk_rc_ok) {
             out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
                      node_name, pcmk_strerror(rc));
             return rc;
         }
     } else {
         for (GList *iter = data_set->nodes; iter; iter = iter->next) {
             pe_node_t *node = (pe_node_t *) iter->data;
 
             rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
                                     operation, interval_spec, data_set);
             if (rc != pcmk_rc_ok) {
                 out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
                          pcmk_strerror(rc));
                 return rc;
             }
         }
     }
 
     out->info(out, "Cleaned up all resources on %s", display_name);
     return rc;
 }
 
 int
 cli_resource_check(pcmk__output_t *out, cib_t * cib_conn, pe_resource_t *rsc)
 {
     char *role_s = NULL;
     char *managed = NULL;
     pe_resource_t *parent = uber_parent(rsc);
     int rc = pcmk_rc_no_output;
     resource_checks_t *checks = NULL;
 
     find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);
 
     find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);
 
     checks = cli_check_resource(rsc, role_s, managed);
 
     if (checks->flags != 0 || checks->lock_node != NULL) {
         rc = out->message(out, "resource-check-list", checks);
     }
 
     free(role_s);
     free(managed);
     free(checks);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
                   const char *rsc_id, pe_working_set_t *data_set)
 {
     crm_notice("Failing %s on %s", rsc_id, host_uname);
     return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, data_set);
 }
 
 static GHashTable *
 generate_resource_params(pe_resource_t *rsc, pe_node_t *node,
                          pe_working_set_t *data_set)
 {
     GHashTable *params = NULL;
     GHashTable *meta = NULL;
     GHashTable *combined = NULL;
     GHashTableIter iter;
     char *key = NULL;
     char *value = NULL;
 
     combined = pcmk__strkey_table(free, free);
 
     params = pe_rsc_params(rsc, node, data_set);
     if (params != NULL) {
         g_hash_table_iter_init(&iter, params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             g_hash_table_insert(combined, strdup(key), strdup(value));
         }
     }
 
     meta = pcmk__strkey_table(free, free);
     get_meta_attributes(meta, rsc, node, data_set);
     if (meta != NULL) {
         g_hash_table_iter_init(&iter, meta);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             char *crm_name = crm_meta_name(key);
 
             g_hash_table_insert(combined, crm_name, strdup(value));
         }
         g_hash_table_destroy(meta);
     }
 
     return combined;
 }
 
 bool resource_is_running_on(pe_resource_t *rsc, const char *host)
 {
     bool found = TRUE;
     GList *hIter = NULL;
     GList *hosts = NULL;
 
     if(rsc == NULL) {
         return FALSE;
     }
 
     rsc->fns->location(rsc, &hosts, TRUE);
     for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
         pe_node_t *node = (pe_node_t *) hIter->data;
 
         if(strcmp(host, node->details->uname) == 0) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         } else if(strcmp(host, node->details->id) == 0) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         }
     }
 
     if(host != NULL) {
         crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
         found = FALSE;
 
     } else if(host == NULL && hosts == NULL) {
         crm_trace("Resource %s is not running\n", rsc->id);
         found = FALSE;
     }
 
   done:
     g_list_free(hosts);
     return found;
 }
 
 /*!
  * \internal
  * \brief Create a list of all resources active on host from a given list
  *
  * \param[in] host      Name of host to check whether resources are active
  * \param[in] rsc_list  List of resources to check
  *
  * \return New list of resources from list that are active on host
  */
 static GList *
 get_active_resources(const char *host, GList *rsc_list)
 {
     GList *rIter = NULL;
     GList *active = NULL;
 
     for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) rIter->data;
 
         /* Expand groups to their members, because if we're restarting a member
          * other than the first, we can't otherwise tell which resources are
          * stopping and starting.
          */
         if (rsc->variant == pe_group) {
             active = g_list_concat(active,
                                    get_active_resources(host, rsc->children));
         } else if (resource_is_running_on(rsc, host)) {
             active = g_list_append(active, strdup(rsc->id));
         }
     }
     return active;
 }
 
 static void dump_list(GList *items, const char *tag) 
 {
     int lpc = 0;
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
         lpc++;
     }
 }
 
 static void display_list(pcmk__output_t *out, GList *items, const char *tag)
 {
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         out->info(out, "%s%s", tag, (const char *)item->data);
     }
 }
 
 /*!
  * \internal
  * \brief Upgrade XML to latest schema version and use it as working set input
  *
  * This also updates the working set timestamp to the current time.
  *
  * \param[in] data_set   Working set instance to update
  * \param[in] xml        XML to use as input
  *
  * \return Standard Pacemaker return code
  * \note On success, caller is responsible for freeing memory allocated for
  *       data_set->now.
  * \todo This follows the example of other callers of cli_config_update()
  *       and returns ENOKEY ("Required key not available") if that fails,
  *       but perhaps pcmk_rc_schema_validation would be better in that case.
  */
 int
 update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml)
 {
     if (cli_config_update(xml, NULL, FALSE) == FALSE) {
         return ENOKEY;
     }
     data_set->input = *xml;
     data_set->now = crm_time_new(NULL);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Update a working set's XML input based on a CIB query
  *
  * \param[in] data_set   Data set instance to initialize
  * \param[in] cib        Connection to the CIB manager
  *
  * \return Standard Pacemaker return code
  * \note On success, caller is responsible for freeing memory allocated for
  *       data_set->input and data_set->now.
  */
 static int
 update_working_set_from_cib(pcmk__output_t *out, pe_working_set_t * data_set,
                             cib_t *cib)
 {
     xmlNode *cib_xml_copy = NULL;
     int rc = pcmk_rc_ok;
 
     rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
     rc = pcmk_legacy2rc(rc);
 
     if (rc != pcmk_rc_ok) {
         out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_strerror(rc), rc);
         return rc;
     }
     rc = update_working_set_xml(data_set, &cib_xml_copy);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Could not upgrade the current CIB XML");
         free_xml(cib_xml_copy);
         return rc;
     }
 
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
 {
     char *pid = NULL;
     char *shadow_file = NULL;
     cib_t *shadow_cib = NULL;
     int rc = pcmk_rc_ok;
 
     pcmk__output_t *out = data_set->priv;
 
     pe_reset_working_set(data_set);
     rc = update_working_set_from_cib(out, data_set, cib);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     if(simulate) {
         bool prev_quiet = false;
 
         pid = pcmk__getpid_s();
         shadow_cib = cib_shadow_new(pid);
         shadow_file = get_shadow_file(pid);
 
         if (shadow_cib == NULL) {
             out->err(out, "Could not create shadow cib: '%s'", pid);
             rc = ENXIO;
             goto done;
         }
 
         rc = write_xml_file(data_set->input, shadow_file, FALSE);
 
         if (rc < 0) {
             out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
             goto done;
         }
 
         rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
         rc = pcmk_legacy2rc(rc);
 
         if (rc != pcmk_rc_ok) {
             out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_strerror(rc), rc);
             goto done;
         }
 
         pcmk__schedule_actions(data_set, data_set->input, NULL);
 
         prev_quiet = out->is_quiet(out);
         out->quiet = true;
         run_simulation(data_set, shadow_cib, NULL);
         out->quiet = prev_quiet;
 
         rc = update_dataset(shadow_cib, data_set, FALSE);
 
     } else {
         cluster_status(data_set);
     }
 
   done:
     /* Do not free data_set->input here, we need rsc->xml to be valid later on */
     cib_delete(shadow_cib);
     free(pid);
 
     if(shadow_file) {
         unlink(shadow_file);
         free(shadow_file);
     }
 
     return rc;
 }
 
 static int
 max_delay_for_resource(pe_working_set_t * data_set, pe_resource_t *rsc) 
 {
     int delay = 0;
     int max_delay = 0;
 
     if(rsc && rsc->children) {
         GList *iter = NULL;
 
         for(iter = rsc->children; iter; iter = iter->next) {
             pe_resource_t *child = (pe_resource_t *)iter->data;
 
             delay = max_delay_for_resource(data_set, child);
             if(delay > max_delay) {
                 double seconds = delay / 1000.0;
                 crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
                 max_delay = delay;
             }
         }
 
     } else if(rsc) {
         char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
         pe_action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
         const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);
         long long result_ll;
 
         if ((pcmk__scan_ll(value, &result_ll, -1LL) == pcmk_rc_ok)
             && (result_ll >= 0) && (result_ll <= INT_MAX)) {
             max_delay = (int) result_ll;
         } else {
             max_delay = -1;
         }
         pe_free_action(stop);
     }
 
     return max_delay;
 }
 
 static int
 max_delay_in(pe_working_set_t * data_set, GList *resources) 
 {
     int max_delay = 0;
     GList *item = NULL;
 
     for (item = resources; item != NULL; item = item->next) {
         int delay = 0;
         pe_resource_t *rsc = pe_find_resource(data_set->resources, (const char *)item->data);
 
         delay = max_delay_for_resource(data_set, rsc);
 
         if(delay > max_delay) {
             double seconds = delay / 1000.0;
             crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
             max_delay = delay;
         }
     }
 
     return 5 + (max_delay / 1000);
 }
 
 #define waiting_for_starts(d, r, h) ((d != NULL) || \
                                     (!resource_is_running_on((r), (h))))
 
 /*!
  * \internal
  * \brief Restart a resource (on a particular host if requested).
  *
  * \param[in] rsc        The resource to restart
  * \param[in] host       The host to restart the resource on (or NULL for all)
  * \param[in] timeout_ms Consider failed if actions do not complete in this time
  *                       (specified in milliseconds, but a two-second
  *                       granularity is actually used; if 0, a timeout will be
  *                       calculated based on the resource timeout)
  * \param[in] cib        Connection to the CIB manager
  *
  * \return Standard Pacemaker return code (exits on certain failures)
  */
 int
 cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
                      const char *move_lifetime, int timeout_ms, cib_t *cib,
                      int cib_options, gboolean promoted_role_only, gboolean force)
 {
     int rc = pcmk_rc_ok;
     int lpc = 0;
     int before = 0;
     int step_timeout_s = 0;
     int sleep_interval = 2;
     int timeout = timeout_ms / 1000;
 
     bool stop_via_ban = FALSE;
     char *rsc_id = NULL;
     char *orig_target_role = NULL;
 
     GList *list_delta = NULL;
     GList *target_active = NULL;
     GList *current_active = NULL;
     GList *restart_target_active = NULL;
 
     pe_working_set_t *data_set = NULL;
 
     if (!resource_is_running_on(rsc, host)) {
         const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
         if(host) {
             out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
         } else {
             out->err(out, "%s is not running anywhere and so cannot be restarted", id);
         }
         return ENXIO;
     }
 
     rsc_id = strdup(rsc->id);
     if ((pe_rsc_is_clone(rsc) || pe_bundle_replicas(rsc)) && host) {
         stop_via_ban = TRUE;
     }
 
     /*
       grab full cib
       determine originally active resources
       disable or ban
       poll cib and watch for affected resources to get stopped
       without --timeout, calculate the stop timeout for each step and wait for that
       if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
       if everything stopped, re-enable or un-ban
       poll cib and watch for affected resources to get started
       without --timeout, calculate the start timeout for each step and wait for that
       if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
       report success
 
       Optimizations:
       - use constraints to determine ordered list of affected resources
       - Allow a --no-deps option (aka. --force-restart)
     */
 
     data_set = pe_new_working_set();
     if (data_set == NULL) {
         crm_perror(LOG_ERR, "Could not allocate working set");
         rc = ENOMEM;
         goto done;
     }
 
     data_set->priv = out;
     pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
     rc = update_dataset(cib, data_set, FALSE);
 
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not get new resource list: %s (%d)", pcmk_strerror(rc), rc);
         goto done;
     }
 
     restart_target_active = get_active_resources(host, data_set->resources);
     current_active = get_active_resources(host, data_set->resources);
 
     dump_list(current_active, "Origin");
 
     if (stop_via_ban) {
         /* Stop the clone or bundle instance by banning it from the host */
         out->quiet = true;
         rc = cli_resource_ban(out, rsc_id, host, move_lifetime, NULL, cib,
                               cib_options, promoted_role_only);
 
     } else {
         /* Stop the resource by setting target-role to Stopped.
          * Remember any existing target-role so we can restore it later
          * (though it only makes any difference if it's Unpromoted).
          */
         char *lookup_id = clone_strip(rsc->id);
 
         find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
                            NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
         free(lookup_id);
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, XML_RSC_ATTR_TARGET_ROLE,
                                            RSC_STOPPED, FALSE, cib, cib_options,
                                            data_set, force);
     }
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not set target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
         if (current_active) {
             g_list_free_full(current_active, free);
         }
         if (restart_target_active) {
             g_list_free_full(restart_target_active, free);
         }
         goto done;
     }
 
     rc = update_dataset(cib, data_set, TRUE);
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not determine which resources would be stopped");
         goto failure;
     }
 
     target_active = get_active_resources(host, data_set->resources);
     dump_list(target_active, "Target");
 
     list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
     out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (list_delta != NULL) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
             rc = update_dataset(cib, data_set, FALSE);
             if(rc != pcmk_rc_ok) {
                 out->err(out, "Could not determine which resources were stopped");
                 goto failure;
             }
 
             if (current_active) {
                 g_list_free_full(current_active, free);
             }
             current_active = get_active_resources(host, data_set->resources);
             g_list_free(list_delta);
             list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
         if(before == g_list_length(list_delta)) {
             /* aborted during stop phase, print the contents of list_delta */
             out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
 
     }
 
     if (stop_via_ban) {
         rc = cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
 
     } else if (orig_target_role) {
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, XML_RSC_ATTR_TARGET_ROLE,
                                            orig_target_role, FALSE, cib,
                                            cib_options, data_set, force);
         free(orig_target_role);
         orig_target_role = NULL;
     } else {
         rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
                                            cib_options, data_set, force);
     }
 
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not unset target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
         goto done;
     }
 
     if (target_active) {
         g_list_free_full(target_active, free);
     }
     target_active = restart_target_active;
     list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
     out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (waiting_for_starts(list_delta, rsc, host)) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
 
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
 
             rc = update_dataset(cib, data_set, FALSE);
             if(rc != pcmk_rc_ok) {
                 out->err(out, "Could not determine which resources were started");
                 goto failure;
             }
 
             if (current_active) {
                 g_list_free_full(current_active, free);
             }
 
             /* It's OK if dependent resources moved to a different node,
              * so we check active resources on all nodes.
              */
             current_active = get_active_resources(NULL, data_set->resources);
             g_list_free(list_delta);
             list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         if(before == g_list_length(list_delta)) {
             /* aborted during start phase, print the contents of list_delta */
             out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
 
     }
 
     rc = pcmk_rc_ok;
     goto done;
 
   failure:
     if (stop_via_ban) {
         cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
     } else if (orig_target_role) {
         cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
                                       XML_RSC_ATTR_TARGET_ROLE, orig_target_role,
                                       FALSE, cib, cib_options, data_set, force);
         free(orig_target_role);
     } else {
         cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
                                       XML_RSC_ATTR_TARGET_ROLE, cib, cib_options,
                                       data_set, force);
     }
 
 done:
     if (list_delta) {
         g_list_free(list_delta);
     }
     if (current_active) {
         g_list_free_full(current_active, free);
     }
     if (target_active && (target_active != restart_target_active)) {
         g_list_free_full(target_active, free);
     }
     if (restart_target_active) {
         g_list_free_full(restart_target_active, free);
     }
     free(rsc_id);
     pe_free_working_set(data_set);
     return rc;
 }
 
 static inline bool action_is_pending(pe_action_t *action)
 {
     if (pcmk_any_flags_set(action->flags, pe_action_optional|pe_action_pseudo)
         || !pcmk_is_set(action->flags, pe_action_runnable)
         || pcmk__str_eq("notify", action->task, pcmk__str_casei)) {
         return false;
     }
     return true;
 }
 
 /*!
  * \internal
  * \brief Return TRUE if any actions in a list are pending
  *
  * \param[in] actions   List of actions to check
  *
  * \return TRUE if any actions in the list are pending, FALSE otherwise
  */
 static bool
 actions_are_pending(GList *actions)
 {
     GList *action;
 
     for (action = actions; action != NULL; action = action->next) {
         pe_action_t *a = (pe_action_t *)action->data;
         if (action_is_pending(a)) {
             crm_notice("Waiting for %s (flags=0x%.8x)", a->uuid, a->flags);
             return TRUE;
         }
     }
     return FALSE;
 }
 
 static void
 print_pending_actions(pcmk__output_t *out, GList *actions)
 {
     GList *action;
 
     out->info(out, "Pending actions:");
     for (action = actions; action != NULL; action = action->next) {
         pe_action_t *a = (pe_action_t *) action->data;
 
         if (!action_is_pending(a)) {
             continue;
         }
 
         if (a->node) {
             out->info(out, "\tAction %d: %s\ton %s", a->id, a->uuid, a->node->details->uname);
         } else {
             out->info(out, "\tAction %d: %s", a->id, a->uuid);
         }
     }
 }
 
 /* For --wait, timeout (in seconds) to use if caller doesn't specify one */
 #define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
 
 /* For --wait, how long to sleep between cluster state checks */
 #define WAIT_SLEEP_S (2)
 
 /*!
  * \internal
  * \brief Wait until all pending cluster actions are complete
  *
  * This waits until either the CIB's transition graph is idle or a timeout is
  * reached.
  *
  * \param[in] timeout_ms Consider failed if actions do not complete in this time
  *                       (specified in milliseconds, but one-second granularity
  *                       is actually used; if 0, a default will be used)
  * \param[in] cib        Connection to the CIB manager
  *
  * \return Standard Pacemaker return code
  */
 int
 wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
 {
     pe_working_set_t *data_set = NULL;
     int rc = pcmk_rc_ok;
     int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
     time_t expire_time = time(NULL) + timeout_s;
     time_t time_diff;
     bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
 
     data_set = pe_new_working_set();
     if (data_set == NULL) {
         return ENOMEM;
     }
     pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
 
     do {
 
         /* Abort if timeout is reached */
         time_diff = expire_time - time(NULL);
         if (time_diff > 0) {
             crm_info("Waiting up to %ld seconds for cluster actions to complete", time_diff);
         } else {
             print_pending_actions(out, data_set->actions);
             pe_free_working_set(data_set);
             return ETIME;
         }
         if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
             sleep(WAIT_SLEEP_S);
         }
 
         /* Get latest transition graph */
         pe_reset_working_set(data_set);
         rc = update_working_set_from_cib(out, data_set, cib);
         if (rc != pcmk_rc_ok) {
             pe_free_working_set(data_set);
             return rc;
         }
         pcmk__schedule_actions(data_set, data_set->input, NULL);
 
         if (!printed_version_warning) {
             /* If the DC has a different version than the local node, the two
              * could come to different conclusions about what actions need to be
              * done. Warn the user in this case.
              *
              * @TODO A possible long-term solution would be to reimplement the
              * wait as a new controller operation that would be forwarded to the
              * DC. However, that would have potential problems of its own.
              */
             const char *dc_version = g_hash_table_lookup(data_set->config_hash,
                                                          "dc-version");
 
             if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
                 out->info(out, "warning: wait option may not work properly in "
                           "mixed-version cluster");
                 printed_version_warning = TRUE;
             }
         }
 
     } while (actions_are_pending(data_set->actions));
 
     pe_free_working_set(data_set);
     return rc;
 }
 
 static const char *
 get_action(const char *rsc_action) {
     const char *action = NULL;
 
     if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
         action = "validate-all";
 
     } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
         action = "monitor";
 
     } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
                                     "force-demote", "force-promote", NULL)) {
         action = rsc_action+6;
     } else {
         action = rsc_action;
     }
 
     return action;
 }
 
 /*!
  * \brief Set up environment variables as expected by resource agents
  *
  * When the cluster executes resource agents, it adds certain environment
  * variables (directly or via resource meta-attributes) expected by some
  * resource agents. Add the essential ones that many resource agents expect, so
  * the behavior is the same for command-line execution.
  *
  * \param[in] params       Resource parameters that will be passed to agent
  * \param[in] timeout_ms   Action timeout (in milliseconds)
  * \param[in] check_level  OCF check level
  * \param[in] verbosity    Verbosity level
  */
 static void
 set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
                       int verbosity)
 {
     g_hash_table_insert(params, strdup("CRM_meta_timeout"),
                         crm_strdup_printf("%d", timeout_ms));
 
     g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION),
                         strdup(CRM_FEATURE_SET));
 
     if (check_level >= 0) {
         char *level = crm_strdup_printf("%d", check_level);
 
         setenv("OCF_CHECK_LEVEL", level, 1);
         free(level);
     }
 
     setenv("HA_debug", (verbosity > 0)? "1" : "0", 1);
     if (verbosity > 1) {
         setenv("OCF_TRACE_RA", "1", 1);
     }
 
     /* A resource agent using the standard ocf-shellfuncs library will not print
      * messages to stderr if it doesn't have a controlling terminal (e.g. if
      * crm_resource is called via script or ssh). This forces it to do so.
      */
     setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
 }
 
 /*!
  * \internal
  * \brief Apply command-line overrides to resource parameters
  *
  * \param[in] params     Parameters to be passed to agent
  * \param[in] overrides  Parameters to override (or NULL if none)
  */
 static void
 apply_overrides(GHashTable *params, GHashTable *overrides)
 {
     if (overrides != NULL) {
         GHashTableIter iter;
         char *name = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, overrides);
         while (g_hash_table_iter_next(&iter, (gpointer *) &name,
                                       (gpointer *) &value)) {
             g_hash_table_replace(params, strdup(name), strdup(value));
         }
     }
 }
 
 crm_exit_t
 cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
                                  const char *rsc_class, const char *rsc_prov,
                                  const char *rsc_type, const char *rsc_action,
                                  GHashTable *params, GHashTable *override_hash,
                                  int timeout_ms, int resource_verbose, gboolean force,
                                  int check_level)
 {
     const char *class = rsc_class;
     const char *action = get_action(rsc_action);
     crm_exit_t exit_code = CRM_EX_OK;
     svc_action_t *op = NULL;
 
     // If no timeout was provided, use the same default as the cluster
     if (timeout_ms == 0) {
         timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
     }
 
     set_agent_environment(params, timeout_ms, check_level, resource_verbose);
     apply_overrides(params, override_hash);
 
     op = services__create_resource_action(rsc_name? rsc_name : "test",
                                           rsc_class, rsc_prov, rsc_type, action,
                                           0, timeout_ms, params, 0);
     if (op == NULL) {
         out->err(out, "Could not execute %s using %s%s%s:%s: %s",
                  action, rsc_class, (rsc_prov? ":" : ""),
                  (rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
         g_hash_table_destroy(params);
         return CRM_EX_OSERR;
     }
 
     if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
         class = resources_find_service_class(rsc_type);
     }
     if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
                               PCMK_RESOURCE_CLASS_LSB, NULL)) {
         services__set_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
                              "Manual execution of this standard is unsupported");
     }
 
     if (op->rc != PCMK_OCF_UNKNOWN) {
         exit_code = op->rc;
         goto done;
     }
 
     services_action_sync(op);
-    exit_code = op->rc;
 
-    // Map LSB status results to OCF results for consistent reporting to user
-    if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
-        && pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
+    // Map results to OCF codes for consistent reporting to user
+    {
+        enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);
 
-        /* A simple cast is sufficient because services_get_ocf_exitcode()
-         * will only return OCF codes that overlap with crm_exit_t.
-         */
-        exit_code = (crm_exit_t) services_get_ocf_exitcode(action, exit_code);
+        // Cast variable instead of function return to keep compilers happy
+        exit_code = (crm_exit_t) ocf_code;
     }
 
 done:
     out->message(out, "resource-agent-action", resource_verbose, rsc_class,
                  rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
                  exit_code, op->status, services__exit_reason(op),
                  op->stdout_data, op->stderr_data);
     services_action_free(op);
     return exit_code;
 }
 
 crm_exit_t
 cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
                      const char *rsc_action, GHashTable *override_hash,
                      int timeout_ms, cib_t * cib, pe_working_set_t *data_set,
                      int resource_verbose, gboolean force, int check_level)
 {
     pcmk__output_t *out = data_set->priv;
     crm_exit_t exit_code = CRM_EX_OK;
     const char *rid = NULL;
     const char *rtype = NULL;
     const char *rprov = NULL;
     const char *rclass = NULL;
     GHashTable *params = NULL;
 
     if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
                                     "force-promote", NULL)) {
         if(pe_rsc_is_clone(rsc)) {
             GList *nodes = cli_resource_search(rsc, requested_name, data_set);
             if(nodes != NULL && force == FALSE) {
                 out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
                          rsc_action, rsc->id);
                 out->err(out, "Try setting target-role=Stopped first or specifying "
                          "the force option");
                 return CRM_EX_UNSAFE;
             }
 
             g_list_free_full(nodes, free);
         }
     }
 
     if(pe_rsc_is_clone(rsc)) {
         /* Grab the first child resource in the hope it's not a group */
         rsc = rsc->children->data;
     }
 
     if(rsc->variant == pe_group) {
         out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
         return CRM_EX_UNIMPLEMENT_FEATURE;
     } else if (rsc->variant == pe_container || pe_rsc_is_bundled(rsc)) {
         out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
         return CRM_EX_UNIMPLEMENT_FEATURE;
     }
 
     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
     rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 
     params = generate_resource_params(rsc, NULL /* @TODO use local node */,
                                       data_set);
 
     if (timeout_ms == 0) {
         timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action), data_set);
     }
 
     rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;
 
     exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
                                                  params, override_hash, timeout_ms,
                                                  resource_verbose, force, check_level);
     return exit_code;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_move(pe_resource_t *rsc, const char *rsc_id, const char *host_name,
                   const char *move_lifetime, cib_t *cib, int cib_options,
                   pe_working_set_t *data_set, gboolean promoted_role_only,
                   gboolean force)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     unsigned int count = 0;
     pe_node_t *current = NULL;
     pe_node_t *dest = pe_find_node(data_set->nodes, host_name);
     bool cur_is_dest = FALSE;
 
     if (dest == NULL) {
         return pcmk_rc_node_unknown;
     }
 
     if (promoted_role_only && !pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         pe_resource_t *p = uber_parent(rsc);
 
         if (pcmk_is_set(p->flags, pe_rsc_promotable)) {
             out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
             rsc_id = p->id;
             rsc = p;
 
         } else {
             out->info(out, "Ignoring master option: %s is not promotable", rsc_id);
             promoted_role_only = FALSE;
         }
     }
 
     current = pe__find_active_requires(rsc, &count);
 
     if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         GList *iter = NULL;
         unsigned int promoted_count = 0;
         pe_node_t *promoted_node = NULL;
 
         for(iter = rsc->children; iter; iter = iter->next) {
             pe_resource_t *child = (pe_resource_t *)iter->data;
             enum rsc_role_e child_role = child->fns->state(child, TRUE);
 
             if (child_role == RSC_ROLE_PROMOTED) {
                 rsc = child;
                 promoted_node = pe__current_node(child);
                 promoted_count++;
             }
         }
         if (promoted_role_only || (promoted_count != 0)) {
             count = promoted_count;
             current = promoted_node;
         }
 
     }
 
     if (count > 1) {
         if (pe_rsc_is_clone(rsc)) {
             current = NULL;
         } else {
             return pcmk_rc_multiple;
         }
     }
 
     if (current && (current->details == dest->details)) {
         cur_is_dest = TRUE;
         if (force) {
             crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
                      rsc_id, promoted_role_only?"promoted":"active", dest->details->uname);
         } else {
             return pcmk_rc_already;
         }
     }
 
     /* Clear any previous prefer constraints across all nodes. */
     cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, cib_options, FALSE, force);
 
     /* Clear any previous ban constraints on 'dest'. */
     cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib,
                        cib_options, TRUE, force);
 
     /* Record an explicit preference for 'dest' */
     rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
                              cib, cib_options, promoted_role_only);
 
     crm_trace("%s%s now prefers node %s%s",
               rsc->id, (promoted_role_only? " (promoted)" : ""),
               dest->details->uname, force?"(forced)":"");
 
     /* only ban the previous location if current location != destination location.
      * it is possible to use -M to enforce a location without regard of where the
      * resource is currently located */
     if(force && (cur_is_dest == FALSE)) {
         /* Ban the original location if possible */
         if(current) {
             (void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
                                    NULL, cib, cib_options, promoted_role_only);
 
         } else if(count > 1) {
             out->info(out, "Resource '%s' is currently %s in %d locations. "
                       "One may now move to %s",
                       rsc_id, (promoted_role_only? "promoted" : "active"),
                       count, dest->details->uname);
             out->info(out, "To prevent '%s' from being %s at a specific location, "
                       "specify a node.",
                       rsc_id, (promoted_role_only? "promoted" : "active"));
 
         } else {
             crm_trace("Not banning %s from its current location: not active", rsc_id);
         }
     }
 
     return rc;
 }