diff --git a/configure.ac b/configure.ac
index e2386c12c1..9165720d1d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,2112 +1,2113 @@
 dnl
 dnl autoconf for Pacemaker
 dnl
 dnl Copyright 2009-2021 the Pacemaker project contributors
 dnl
 dnl The version control history for this file may have further details.
 dnl
 dnl This source code is licensed under the GNU General Public License version 2
 dnl or later (GPLv2+) WITHOUT ANY WARRANTY.
 
 dnl ===============================================
 dnl Bootstrap
 dnl ===============================================
 AC_PREREQ(2.64)
 
 dnl AC_CONFIG_MACRO_DIR is deprecated as of autoconf 2.70 (2020-12-08).
 dnl Once we can require that version, we can simplify this, and no longer
 dnl need ACLOCAL_AMFLAGS in Makefile.am.
 m4_ifdef([AC_CONFIG_MACRO_DIRS],
          [AC_CONFIG_MACRO_DIRS([m4])],
          [AC_CONFIG_MACRO_DIR([m4])])
 
 AC_DEFUN([AC_DATAROOTDIR_CHECKED])
 
 dnl Suggested structure:
 dnl     information on the package
 dnl     checks for programs
 dnl     checks for libraries
 dnl     checks for header files
 dnl     checks for types
 dnl     checks for structures
 dnl     checks for compiler characteristics
 dnl     checks for library functions
 dnl     checks for system services
 
 m4_include([m4/version.m4])
 AC_INIT([pacemaker], VERSION_NUMBER, [users@clusterlabs.org], [pacemaker],
         PCMK_URL)
 
 PCMK_FEATURES=""
 
 LT_CONFIG_LTDL_DIR([libltdl])
 AC_CONFIG_AUX_DIR([libltdl/config])
 AC_CANONICAL_HOST
 
 dnl Where #defines that autoconf makes (e.g. HAVE_whatever) go
 dnl
 dnl Internal header: include/config.h
 dnl   - Contains ALL defines
 dnl   - include/config.h.in is generated automatically by autoheader
 dnl   - NOT to be included in any header files except crm_internal.h
 dnl     (which is also not to be included in any other header files)
 dnl
 dnl External header: include/crm_config.h
 dnl   - Contains a subset of defines checked here
 dnl   - Manually edit include/crm_config.h.in to have configure include
 dnl     new defines
 dnl   - Should not include HAVE_* defines
 dnl   - Safe to include anywhere
 AC_CONFIG_HEADERS([include/config.h include/crm_config.h])
 
 dnl 1.13:           minimum automake version required
 dnl foreign:        don't require GNU-standard top-level files
 dnl tar-ustar:      use (older) POSIX variant of generated tar rather than v7
 dnl subdir-objects: keep .o's with their .c's (no-op in 2.0+)
 AM_INIT_AUTOMAKE([1.13 foreign tar-ustar subdir-objects])
 
 dnl Require minimum version of pkg-config
 PKG_PROG_PKG_CONFIG(0.27)
 AS_IF([test "x${PKG_CONFIG}" != x], [],
       [AC_MSG_FAILURE([Could not find required build tool pkg-config (0.27 or later)])])
 PKG_INSTALLDIR
 PKG_NOARCH_INSTALLDIR
 
 dnl Example 2.4. Silent Custom Rule to Generate a File
 dnl %-bar.pc: %.pc
 dnl	$(AM_V_GEN)$(LN_S) $(notdir $^) $@
 
 dnl Versioned attributes implementation is not yet production-ready
 AC_DEFINE_UNQUOTED(ENABLE_VERSIONED_ATTRS, 0, [Enable versioned attributes])
 
 CC_IN_CONFIGURE=yes
 export CC_IN_CONFIGURE
 
 LDD=ldd
 
 GLIB_TESTS
 
 dnl ========================================================================
 dnl Compiler characteristics
 dnl ========================================================================
 
 dnl A particular compiler can be forced by setting the CC environment variable
 AC_PROG_CC
 
 dnl Use at least C99 if possible. This will generate an "obsolete" warning
 dnl since autoconf 2.70, but is needed for older versions.
 AC_PROG_CC_STDC
 
 dnl C++ is not needed for build, just maintainer utilities
 AC_PROG_CXX
 
 dnl We use md5.c from gnulib, which has its own m4 macros. Per its docs:
 dnl "The macro gl_EARLY must be called as soon as possible after verifying that
 dnl the C compiler is working. ... The core part of the gnulib checks are done
 dnl by the macro gl_INIT." In addition, prevent gnulib from introducing OpenSSL
 dnl as a dependency.
 gl_EARLY
 gl_SET_CRYPTO_CHECK_DEFAULT([no])
 gl_INIT
 
 # --enable-new-dtags: Use RUNPATH instead of RPATH.
 # It is necessary to have this done before libtool does linker detection.
 # See also: https://github.com/kronosnet/kronosnet/issues/107
 AX_CHECK_LINK_FLAG([-Wl,--enable-new-dtags],
                   [AM_LDFLAGS=-Wl,--enable-new-dtags],
                   [AC_MSG_ERROR(["Linker support for --enable-new-dtags is required"])])
 AC_SUBST([AM_LDFLAGS])
 
 saved_LDFLAGS="$LDFLAGS"
 LDFLAGS="$AM_LDFLAGS $LDFLAGS"
 LT_INIT([dlopen])
 LDFLAGS="$saved_LDFLAGS"
 LTDL_INIT([convenience])
 
 AC_TYPE_SIZE_T
 AC_CHECK_SIZEOF(char)
 AC_CHECK_SIZEOF(short)
 AC_CHECK_SIZEOF(int)
 AC_CHECK_SIZEOF(long)
 AC_CHECK_SIZEOF(long long)
 
 dnl ===============================================
 dnl Helpers
 dnl ===============================================
 cc_supports_flag() {
     local CFLAGS="-Werror $@"
     AC_MSG_CHECKING([whether $CC supports $@])
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ ]])],
                       [RC=0; AC_MSG_RESULT([yes])],
                       [RC=1; AC_MSG_RESULT([no])])
     return $RC
 }
 
 # Some tests need to use their own CFLAGS
 
 cc_temp_flags() {
     ac_save_CFLAGS="$CFLAGS"
     CFLAGS="$*"
 }
 
 cc_restore_flags() {
     CFLAGS=$ac_save_CFLAGS
 }
 
 # yes_no_try $user_response $default
 DISABLED=0
 REQUIRED=1
 OPTIONAL=2
 yes_no_try() {
     local value
     AS_IF([test x"$1" = x""], [value="$2"], [value="$1"])
     AS_CASE(["`echo "$value" | tr '[A-Z]' '[a-z]'`"],
             [0|no|false|disable], [return $DISABLED],
             [1|yes|true|enable], [return $REQUIRED],
             [try|check], [return $OPTIONAL]
     )
     AC_MSG_ERROR([Invalid option value "$value"])
 }
 
 check_systemdsystemunitdir() {
     AC_MSG_CHECKING([which system unit file directory to use])
     PKG_CHECK_VAR([systemdsystemunitdir], [systemd], [systemdsystemunitdir])
     AC_MSG_RESULT([${systemdsystemunitdir}])
     test x"$systemdsystemunitdir" != x""
     return $?
 }
 
 dnl ===============================================
 dnl Configure Options
 dnl ===============================================
 
 dnl Actual library checks come later, but pkg-config can be used here to grab
 dnl external values to use as defaults for configure options
 
 dnl --enable-* options: build process
 
 AC_ARG_ENABLE([quiet],
     [AS_HELP_STRING([--enable-quiet],
         [suppress make output unless there is an error @<:@no@:>@])]
 )
 yes_no_try "$enable_quiet" "no"
 enable_quiet=$?
 
 AC_ARG_ENABLE([fatal-warnings],
     [AS_HELP_STRING([--enable-fatal-warnings],
         [enable pedantic and fatal warnings for gcc @<:@try@:>@])],
 )
 yes_no_try "$enable_fatal_warnings" "try"
 enable_fatal_warnings=$?
 
 AC_ARG_ENABLE([hardening],
     [AS_HELP_STRING([--enable-hardening],
         [harden the resulting executables/libraries @<:@try@:>@])]
 )
 yes_no_try "$enable_hardening" "try"
 enable_hardening=$?
 
 dnl --enable-* options: features
 
 AC_ARG_ENABLE([systemd],
     [AS_HELP_STRING([--enable-systemd],
         [enable support for managing resources via systemd @<:@try@:>@])]
 )
 yes_no_try "$enable_systemd" "try"
 enable_systemd=$?
 
 AC_ARG_ENABLE([upstart],
     [AS_HELP_STRING([--enable-upstart],
         [enable support for managing resources via Upstart (deprecated) @<:@try@:>@])]
 )
 yes_no_try "$enable_upstart" "try"
 enable_upstart=$?
 
 dnl --enable-* options: compatibility
 
 AC_ARG_ENABLE([compat-2.0],
     [AS_HELP_STRING([--enable-compat-2.0], m4_normalize([
         preserve certain output as it was in 2.0; this option will be
         available only for the lifetime of the 2.1 series @<:@no@:>@]))]
 )
 yes_no_try "$enable_compat_2_0" "no"
 enable_compat_2_0=$?
 AS_IF([test $enable_compat_2_0 -ne $DISABLED],
       [
           AC_DEFINE_UNQUOTED([PCMK__COMPAT_2_0], [1],
                              [Keep certain output compatible with 2.0 release series])
           PCMK_FEATURES="$PCMK_FEATURES compat-2.0"
       ]
 )
 
 # Add an option to create symlinks at the pre-2.0.0 daemon name locations, so
 # that users and tools can continue to invoke those names directly (e.g., for
 # meta-data). This option will be removed in a future release.
 AC_ARG_ENABLE([legacy-links],
     [AS_HELP_STRING([--enable-legacy-links],
         [add symlinks for old daemon names (deprecated) @<:@no@:>@])]
 )
 yes_no_try "$enable_legacy_links" "no"
 enable_legacy_links=$?
 AM_CONDITIONAL([BUILD_LEGACY_LINKS], [test $enable_legacy_links -ne $DISABLED])
 
 dnl --with-* options: basic parameters
 
 dnl This argument is defined via an M4 macro so default can be a variable
 AC_DEFUN([VERSION_ARG],
     [AC_ARG_WITH([version],
         [AS_HELP_STRING([--with-version=VERSION],
             [override package version @<:@$1@:>@])],
         [ PACEMAKER_VERSION="$withval" ],
         [ PACEMAKER_VERSION="$PACKAGE_VERSION" ])]
 )
 VERSION_ARG(VERSION_NUMBER)
 
 # Redefine PACKAGE_VERSION and VERSION according to PACEMAKER_VERSION in case
 # the user used --with-version. Unfortunately, this can only affect the
 # substitution variables and later uses in this file, not the config.h
 # constants, so we have to be careful to use only PACEMAKER_VERSION in C code.
 PACKAGE_VERSION=$PACEMAKER_VERSION
 VERSION=$PACEMAKER_VERSION
 
 CRM_DAEMON_USER=""
 AC_ARG_WITH([daemon-user],
     [AS_HELP_STRING([--with-daemon-user=USER],
         [user to run unprivileged Pacemaker daemons as (advanced option: changing this may break other cluster components unless similarly configured) @<:@hacluster@:>@])],
     [ CRM_DAEMON_USER="$withval" ]
 )
 
 CRM_DAEMON_GROUP=""
 AC_ARG_WITH([daemon-group],
     [AS_HELP_STRING([--with-daemon-group=GROUP],
         [group to run unprivileged Pacemaker daemons as (advanced option: changing this may break other cluster components unless similarly configured) @<:@haclient@:>@])],
     [ CRM_DAEMON_GROUP="$withval" ]
 )
 
 BUG_URL=""
 AC_ARG_WITH([bug-url],
     [AS_HELP_STRING([--with-bug-url=DIR], m4_normalize([
         address where users should submit bug reports
         @<:@https://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker@:>@]))],
     [ BUG_URL="$withval" ]
 )
 
 dnl --with-* options: features
 
 AC_ARG_WITH([cibsecrets],
     [AS_HELP_STRING([--with-cibsecrets],
         [support separate file for CIB secrets @<:@no@:>@])]
 )
 yes_no_try "$with_cibsecrets" "no"
 with_cibsecrets=$?
 
 AC_ARG_WITH([gnutls],
     [AS_HELP_STRING([--with-gnutls],
         [support Pacemaker Remote and remote-tls-port using GnuTLS @<:@try@:>@])]
 )
 yes_no_try "$with_gnutls" "try"
 with_gnutls=$?
 
 PCMK_GNUTLS_PRIORITIES="NORMAL"
 AC_ARG_WITH([gnutls-priorities],
     [AS_HELP_STRING([--with-gnutls-priorities],
         [default GnuTLS cipher priorities @<:@NORMAL@:>@])],
     [ test x"$withval" = x"no" || PCMK_GNUTLS_PRIORITIES="$withval" ]
 )
 
 AC_ARG_WITH([concurrent-fencing-default],
     [AS_HELP_STRING([--with-concurrent-fencing-default],
         [default value for concurrent-fencing cluster option @<:@false@:>@])],
 )
 AS_CASE([$with_concurrent_fencing_default],
         [""], [with_concurrent_fencing_default="false"],
         [false], [],
         [true], [PCMK_FEATURES="$PCMK_FEATURES default-concurrent-fencing"],
         [AC_MSG_ERROR([Invalid value "$with_concurrent_fencing_default" for --with-concurrent-fencing-default])]
 )
 AC_DEFINE_UNQUOTED([PCMK__CONCURRENT_FENCING_DEFAULT],
                    ["$with_concurrent_fencing_default"],
                    [Default value for concurrent-fencing cluster option])
 
 AC_ARG_WITH([sbd-sync-default],
     [AS_HELP_STRING([--with-sbd-sync-default], m4_normalize([
         default value used by sbd if SBD_SYNC_RESOURCE_STARTUP
         environment variable is not set @<:@false@:>@]))],
 )
 AS_CASE([$with_sbd_sync_default],
         [""], [with_sbd_sync_default=false],
         [false], [],
         [true], [PCMK_FEATURES="$PCMK_FEATURES default-sbd-sync"],
         [AC_MSG_ERROR([Invalid value "$with_sbd_sync_default" for --with-sbd-sync-default])]
 )
 AC_DEFINE_UNQUOTED([PCMK__SBD_SYNC_DEFAULT],
                    [$with_sbd_sync_default],
                    [Default value for SBD_SYNC_RESOURCE_STARTUP environment variable])
 
 AC_ARG_WITH([resource-stickiness-default],
     [AS_HELP_STRING([--with-resource-stickiness-default],
         [If positive, value to add to new CIBs as explicit resource default for resource-stickiness @<:@0@:>@])],
 )
 errmsg="Invalid value \"$with_resource_stickiness_default\" for --with-resource-stickiness-default"
 AS_CASE([$with_resource_stickiness_default],
         [0|""], [with_resource_stickiness_default="0"],
         [*[[!0-9]]*], [AC_MSG_ERROR([$errmsg])],
         [PCMK_FEATURES="$PCMK_FEATURES default-resource-stickiness"]
 )
 AC_DEFINE_UNQUOTED([PCMK__RESOURCE_STICKINESS_DEFAULT],
                    [$with_resource_stickiness_default],
                    [Default value for resource-stickiness resource meta-attribute])
 
 AC_ARG_WITH([corosync],
     [AS_HELP_STRING([--with-corosync],
         [support the Corosync messaging and membership layer @<:@try@:>@])]
 )
 yes_no_try "$with_corosync" "try"
 with_corosync=$?
 
 AC_ARG_WITH([nagios],
     [AS_HELP_STRING([--with-nagios], [support nagios resources])]
 )
 yes_no_try "$with_nagios" "try"
 with_nagios=$?
 
 dnl --with-* options: directory locations
 
 AC_ARG_WITH([nagios-plugin-dir],
     [AS_HELP_STRING([--with-nagios-plugin-dir=DIR],
         [directory for nagios plugins @<:@LIBEXECDIR/nagios/plugins@:>@])],
     [ NAGIOS_PLUGIN_DIR="$withval" ]
 )
 
 AC_ARG_WITH([nagios-metadata-dir],
     [AS_HELP_STRING([--with-nagios-metadata-dir=DIR],
         [directory for nagios plugins metadata @<:@DATADIR/nagios/plugins-metadata@:>@])],
     [ NAGIOS_METADATA_DIR="$withval" ]
 )
 
 INITDIR=""
 AC_ARG_WITH([initdir],
     [AS_HELP_STRING([--with-initdir=DIR],
         [directory for init (rc) scripts])],
     [ INITDIR="$withval" ]
 )
 
 systemdsystemunitdir="${systemdsystemunitdir-}"
 AC_ARG_WITH([systemdsystemunitdir],
     [AS_HELP_STRING([--with-systemdsystemunitdir=DIR],
         [directory for systemd unit files (advanced option: must match what systemd uses)])],
     [ systemdsystemunitdir="$withval" ]
 )
 
 CONFIGDIR=""
 AC_ARG_WITH([configdir],
     [AS_HELP_STRING([--with-configdir=DIR],
         [directory for Pacemaker configuration file @<:@SYSCONFDIR/sysconfig@:>@])],
     [ CONFIGDIR="$withval" ]
 )
 
 dnl --runstatedir is available as of autoconf 2.70 (2020-12-08). When users
 dnl have an older version, they can use our --with-runstatedir.
 pcmk_runstatedir=""
 AC_ARG_WITH([runstatedir],
     [AS_HELP_STRING([--with-runstatedir=DIR],
         [modifiable per-process data @<:@LOCALSTATEDIR/run@:>@ (ignored if --runstatedir is available)])],
     [ pcmk_runstatedir="$withval" ]
 )
 
 CRM_LOG_DIR=""
 AC_ARG_WITH([logdir],
     [AS_HELP_STRING([--with-logdir=DIR],
         [directory for Pacemaker log file @<:@LOCALSTATEDIR/log/pacemaker@:>@])],
     [ CRM_LOG_DIR="$withval" ]
 )
 
 CRM_BUNDLE_DIR=""
 AC_ARG_WITH([bundledir],
     [AS_HELP_STRING([--with-bundledir=DIR],
         [directory for Pacemaker bundle logs @<:@LOCALSTATEDIR/log/pacemaker/bundles@:>@])],
     [ CRM_BUNDLE_DIR="$withval" ]
 )
 
 dnl Get default from resource-agents if possible. Otherwise, the default uses
 dnl /usr/lib rather than libdir because it's determined by the OCF project and
 dnl not Pacemaker. Even if a user wants to install Pacemaker to /usr/local or
 dnl such, the OCF agents will be expected in their usual location. However, we
 dnl do give the user the option to override it.
 PKG_CHECK_VAR([OCF_ROOT_DIR], [resource-agents], [ocfrootdir], [],
               [OCF_ROOT_DIR="/usr/lib/ocf"])
 AC_ARG_WITH([ocfdir],
     [AS_HELP_STRING([--with-ocfdir=DIR], m4_normalize([
         OCF resource agent root directory (advanced option: changing this
         may break other cluster components unless similarly configured)
         @<:@value from resource-agents package if available otherwise
         /usr/lib/ocf@:>@]))],
     [ OCF_ROOT_DIR="$withval" ]
 )
 AC_SUBST(OCF_ROOT_DIR)
 AC_DEFINE_UNQUOTED([OCF_ROOT_DIR], ["$OCF_ROOT_DIR"],
                    [OCF root directory for resource agents and libraries])
 
 PKG_CHECK_VAR([OCF_RA_PATH], [resource-agents], [ocfrapath], [],
               [OCF_RA_PATH="$OCF_ROOT_DIR/resource.d"])
 AC_ARG_WITH([ocfrapath],
     [AS_HELP_STRING([--with-ocfrapath=DIR], m4_normalize([
         OCF resource agent directories (colon-separated) to search
         @<:@value from resource-agents package if available otherwise
         OCFDIR/resource.d@:>@]))],
     [ OCF_RA_PATH="$withval" ]
 )
 AC_SUBST(OCF_RA_PATH)
 AC_DEFINE_UNQUOTED([OCF_RA_PATH], ["$OCF_RA_PATH"],
                    [OCF directories to search for resource agents ])
 
 OCF_RA_INSTALL_DIR="$OCF_ROOT_DIR/resource.d"
 AC_ARG_WITH([ocfrainstalldir],
     [AS_HELP_STRING([--with-ocfrainstalldir=DIR], m4_normalize([
         OCF installation directory for Pacemakers resource agents
         @<:@OCFDIR/resource.d@:>@]))],
     [ OCF_RA_INSTALL_DIR="$withval" ]
 )
 AC_SUBST(OCF_RA_INSTALL_DIR)
 
 dnl Get default from fence-agents if available
 PKG_CHECK_VAR([FA_PREFIX], [fence-agents], [prefix],
               [PCMK__FENCE_BINDIR="${FA_PREFIX}/sbin"],
               [PCMK__FENCE_BINDIR="$sbindir"])
 AC_ARG_WITH([fence-bindir],
     [AS_HELP_STRING([--with-fence-bindir=DIR], m4_normalize([
         directory for executable fence agents @<:@value from fence-agents
         package if available otherwise SBINDIR@:>@]))],
     [ PCMK__FENCE_BINDIR="$withval" ]
 )
 AC_SUBST(PCMK__FENCE_BINDIR)
 
 dnl --with-* options: non-production testing
 
 AC_ARG_WITH([profiling],
     [AS_HELP_STRING([--with-profiling],
         [disable optimizations, for effective profiling @<:@no@:>@])]
 )
 yes_no_try "$with_profiling" "no"
 with_profiling=$?
 
 AC_ARG_WITH([coverage],
     [AS_HELP_STRING([--with-coverage],
         [disable optimizations, for effective profiling and coverage testing @<:@no@:>@])]
 )
 yes_no_try "$with_coverage" "no"
 with_coverage=$?
 
 AC_ARG_WITH([sanitizers],
   [AS_HELP_STRING([--with-sanitizers=...,...],
     [enable SANitizer build, do *NOT* use for production. Only ASAN/UBSAN/TSAN are currently supported])],
   [ SANITIZERS="$withval" ],
   [ SANITIZERS="" ])
 
 
 dnl ===============================================
 dnl General Processing
 dnl ===============================================
 
 AC_DEFINE_UNQUOTED(PACEMAKER_VERSION, "$VERSION",
                    [Version number of this Pacemaker build])
 
 PACKAGE_SERIES=`echo $VERSION | awk -F. '{ print $1"."$2 }'`
 AC_SUBST(PACKAGE_SERIES)
 
 AC_PROG_LN_S
 AC_PROG_MKDIR_P
 
 # Check for fatal warning support
 AS_IF([test $enable_fatal_warnings -ne $DISABLED && test "$GCC" = "yes" && cc_supports_flag -Werror],
       [WERROR="-Werror"],
       [
           WERROR=""
           AS_CASE([$enable_fatal_warnings],
                   [$REQUIRED], [AC_MSG_ERROR([Compiler does not support fatal warnings])],
                   [$OPTIONAL], [
                       AC_MSG_NOTICE([Compiler does not support fatal warnings])
                       enable_fatal_warnings=$DISABLED
                   ])
       ])
 
 AC_MSG_NOTICE([Sanitizing prefix: ${prefix}])
 AS_IF([test "$prefix" = "NONE"],
       [
           prefix=/usr
           dnl Fix default variables - "prefix" variable if not specified
           AS_IF([test "$localstatedir" = "\${prefix}/var"],
                 [localstatedir="/var"])
           AS_IF([test "$sysconfdir" = "\${prefix}/etc"],
                 [sysconfdir="/etc"])
       ])
 
 AC_MSG_NOTICE([Sanitizing exec_prefix: ${exec_prefix}])
 case $exec_prefix in
     prefix|NONE)
         exec_prefix=$prefix
         ;;
 esac
 
 AC_MSG_NOTICE([Sanitizing INITDIR: ${INITDIR}])
 case $INITDIR in
     prefix) INITDIR=$prefix;;
     "")
         AC_MSG_CHECKING([which init (rc) directory to use])
         for initdir in /etc/init.d /etc/rc.d/init.d /sbin/init.d \
             /usr/local/etc/rc.d /etc/rc.d
         do
             if
                 test -d $initdir
             then
                 INITDIR=$initdir
                 break
             fi
         done
         AC_MSG_RESULT([$INITDIR])
         ;;
 esac
 AC_SUBST(INITDIR)
 
 AC_MSG_NOTICE([Sanitizing libdir: ${libdir}])
 case $libdir in
     prefix|NONE)
         AC_MSG_CHECKING([which lib directory to use])
         for aDir in lib64 lib
         do
             trydir="${exec_prefix}/${aDir}"
             if
                 test -d ${trydir}
             then
                 libdir=${trydir}
                 break
             fi
         done
         AC_MSG_RESULT([$libdir]);
         ;;
 esac
 
 dnl Expand autoconf variables so that we don't end up with '${prefix}'
 dnl in #defines and python scripts
 dnl NOTE: Autoconf deliberately leaves them unexpanded to allow
 dnl    make exec_prefix=/foo install
 dnl No longer being able to do this seems like no great loss to me...
 
 eval prefix="`eval echo ${prefix}`"
 eval exec_prefix="`eval echo ${exec_prefix}`"
 eval bindir="`eval echo ${bindir}`"
 eval sbindir="`eval echo ${sbindir}`"
 eval libexecdir="`eval echo ${libexecdir}`"
 eval datadir="`eval echo ${datadir}`"
 eval sysconfdir="`eval echo ${sysconfdir}`"
 eval sharedstatedir="`eval echo ${sharedstatedir}`"
 eval localstatedir="`eval echo ${localstatedir}`"
 eval libdir="`eval echo ${libdir}`"
 eval includedir="`eval echo ${includedir}`"
 eval oldincludedir="`eval echo ${oldincludedir}`"
 eval infodir="`eval echo ${infodir}`"
 eval mandir="`eval echo ${mandir}`"
 
 dnl Home-grown variables
 
 if [ test "x${runstatedir}" = "x" ]; then
     if [ test "x${pcmk_runstatedir}" = "x" ]; then
         runstatedir="${localstatedir}/run"
     else
         runstatedir="${pcmk_runstatedir}"
     fi
 fi
 eval runstatedir="$(eval echo ${runstatedir})"
 AC_DEFINE_UNQUOTED([PCMK_RUN_DIR], ["$runstatedir"],
                    [Location for modifiable per-process data])
 AC_SUBST(runstatedir)
 
 eval INITDIR="${INITDIR}"
 eval docdir="`eval echo ${docdir}`"
 if test x"${docdir}" = x""; then
     docdir=${datadir}/doc/${PACKAGE}-${VERSION}
 fi
 AC_SUBST(docdir)
 if test x"${CONFIGDIR}" = x""; then
     CONFIGDIR="${sysconfdir}/sysconfig"
 fi
 AC_SUBST(CONFIGDIR)
 
 if test x"${CRM_LOG_DIR}" = x""; then
     CRM_LOG_DIR="${localstatedir}/log/pacemaker"
 fi
 AC_DEFINE_UNQUOTED(CRM_LOG_DIR,"$CRM_LOG_DIR", Location for Pacemaker log file)
 AC_SUBST(CRM_LOG_DIR)
 
 if test x"${CRM_BUNDLE_DIR}" = x""; then
     CRM_BUNDLE_DIR="${localstatedir}/log/pacemaker/bundles"
 fi
 AC_DEFINE_UNQUOTED(CRM_BUNDLE_DIR,"$CRM_BUNDLE_DIR", Location for Pacemaker bundle logs)
 AC_SUBST(CRM_BUNDLE_DIR)
 
 eval PCMK__FENCE_BINDIR="`eval echo ${PCMK__FENCE_BINDIR}`"
 AC_DEFINE_UNQUOTED(PCMK__FENCE_BINDIR,"$PCMK__FENCE_BINDIR",
                    [Location for executable fence agents])
 
 AS_IF([test x"${PCMK_GNUTLS_PRIORITIES}" != x""], [],
       [AC_MSG_ERROR([--with-gnutls-priorities value must not be empty])])
 AC_DEFINE_UNQUOTED([PCMK_GNUTLS_PRIORITIES], ["$PCMK_GNUTLS_PRIORITIES"],
                    [GnuTLS cipher priorities])
 
 if test x"${BUG_URL}" = x""; then
     BUG_URL="https://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker"
 fi
 AC_SUBST(BUG_URL)
 
 for j in prefix exec_prefix bindir sbindir libexecdir datadir sysconfdir \
     sharedstatedir localstatedir libdir includedir oldincludedir infodir \
     mandir INITDIR docdir CONFIGDIR
 do
     dirname=`eval echo '${'${j}'}'`
     if
         test ! -d "$dirname"
     then
         AC_MSG_WARN([$j directory ($dirname) does not exist (yet)])
     fi
 done
 
 us_auth=
 AC_CHECK_HEADER([sys/socket.h], [
     AC_CHECK_DECL([SO_PEERCRED], [
         # Linux
         AC_CHECK_TYPE([struct ucred], [
             us_auth=peercred_ucred;
             AC_DEFINE([US_AUTH_PEERCRED_UCRED], [1],
                       [Define if Unix socket auth method is
                        getsockopt(s, SO_PEERCRED, &ucred, ...)])
         ], [
             # OpenBSD
             AC_CHECK_TYPE([struct sockpeercred], [
                 us_auth=localpeercred_sockepeercred;
                 AC_DEFINE([US_AUTH_PEERCRED_SOCKPEERCRED], [1],
                           [Define if Unix socket auth method is
                            getsockopt(s, SO_PEERCRED, &sockpeercred, ...)])
             ], [], [[#include <sys/socket.h>]])
         ], [[#define _GNU_SOURCE
              #include <sys/socket.h>]])
     ], [], [[#include <sys/socket.h>]])
 ])
 
 AS_IF([test -z "${us_auth}"], [
     # FreeBSD
     AC_CHECK_DECL([getpeereid], [
         us_auth=getpeereid;
         AC_DEFINE([US_AUTH_GETPEEREID], [1],
                   [Define if Unix socket auth method is
                    getpeereid(s, &uid, &gid)])
     ], [
         # Solaris/OpenIndiana
         AC_CHECK_DECL([getpeerucred], [
             us_auth=getpeerucred;
             AC_DEFINE([US_AUTH_GETPEERUCRED], [1],
                       [Define if Unix socket auth method is
                        getpeercred(s, &ucred)])
         ], [
             AC_MSG_FAILURE([No way to authenticate a Unix socket peer])
         ], [[#include <ucred.h>]])
     ])
 ])
 
 dnl OS-based decision-making is poor autotools practice; feature-based
 dnl mechanisms are strongly preferred. Keep this section to a bare minimum;
 dnl regard as a "necessary evil".
 INIT_EXT=""
 PROCFS=0
 case "$host_os" in
     dnl Solaris and some *BSD versions support procfs but not files we need
     *bsd*)
         INIT_EXT=".sh"
         ;;
     *linux*)
         PROCFS=1
         ;;
     darwin*)
         LIBS="$LIBS -L${prefix}/lib"
         CFLAGS="$CFLAGS -I${prefix}/include"
         ;;
 esac
 
 AC_SUBST(INIT_EXT)
 AC_DEFINE_UNQUOTED([SUPPORT_PROCFS], [$PROCFS],
                    [Define to 1 if procfs is supported])
 
 case "$host_cpu" in
     ppc64|powerpc64)
         case $CFLAGS in
          *powerpc64*)
              ;;
          *)
              if test "$GCC" = yes; then
                  CFLAGS="$CFLAGS -m64"
              fi
              ;;
         esac
         ;;
 esac
 
 # C99 doesn't guarantee uint64_t type and related format specifiers, but
 # prerequisites, corosync + libqb, use that widely, so the target platforms
 # are already pre-constrained to those "64bit-clean" (doesn't imply native
 # bit width) and hence we deliberately refrain from artificial surrogates
 # (sans manipulation through cached values).
 AC_CACHE_VAL(
     [pcmk_cv_decl_inttypes],
     [
         AC_CHECK_DECLS(
             [PRIu64, PRIu32, PRIx32,
              SCNu64],
             [pcmk_cv_decl_inttypes="PRIu64 PRIu32 PRIx32 SCNu64"],
             [
                 # test shall only react on "no" cached result & error out respectively
                 if test "x$ac_cv_have_decl_PRIu64" = xno; then
                     AC_MSG_ERROR([lack of inttypes.h based specifier serving uint64_t (PRIu64)])
                 elif test "x$ac_cv_have_decl_PRIu32" = xno; then
                     AC_MSG_ERROR([lack of inttypes.h based specifier serving uint32_t (PRIu32)])
                 elif test "x$ac_cv_have_decl_PRIx32" = xno; then
                     AC_MSG_ERROR([lack of inttypes.h based hexa specifier serving uint32_t (PRIx32)])
                 elif test "x$ac_cv_have_decl_SCNu64" = xno; then
                     AC_MSG_ERROR([lack of inttypes.h based specifier gathering uint64_t (SCNu64)])
                 fi
             ],
             [[#include <inttypes.h>]]
         )
     ]
 )
 (
     set $pcmk_cv_decl_inttypes
     AC_DEFINE_UNQUOTED([U64T],  [$1], [Correct format specifier for U64T])
     AC_DEFINE_UNQUOTED([U32T],  [$2], [Correct format specifier for U32T])
     AC_DEFINE_UNQUOTED([X32T],  [$3], [Correct format specifier for X32T])
     AC_DEFINE_UNQUOTED([U64TS], [$4], [Correct format specifier for U64TS])
 )
 
 dnl ===============================================
 dnl Program Paths
 dnl ===============================================
 
 PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin"
 export PATH
 
 dnl Replacing AC_PROG_LIBTOOL with AC_CHECK_PROG because LIBTOOL
 dnl was NOT being expanded all the time thus causing things to fail.
 AC_CHECK_PROGS(LIBTOOL, glibtool libtool libtool15 libtool13)
 AS_IF([test "x${LIBTOOL}" != "x"], [],
       [AC_MSG_FAILURE([Could not find required build tool libtool (or equivalent)])])
 
 dnl Pacemaker's executable python scripts will invoke the python specified by
 dnl configure's PYTHON variable. If not specified, AM_PATH_PYTHON will check a
 dnl built-in list with (unversioned) "python" having precedence. To configure
 dnl Pacemaker to use a specific python interpreter version, define PYTHON
 dnl when calling configure, for example: ./configure PYTHON=/usr/bin/python3.6
 
 dnl Ensure PYTHON is an absolute path
 AS_IF([test x"${PYTHON}" != x""], [AC_PATH_PROG([PYTHON], [$PYTHON])])
 
 dnl Require a minimum Python version
 AM_PATH_PYTHON([3.4])
 
 AC_PATH_PROGS([ASCIIDOC_CONV], [asciidoc asciidoctor])
 AC_PATH_PROG([HELP2MAN], [help2man])
 AC_PATH_PROG([SPHINX], [sphinx-build])
 AC_PATH_PROG([INKSCAPE], [inkscape])
 AC_PATH_PROG([XSLTPROC], [xsltproc])
 AC_PATH_PROG([XMLCATALOG], [xmlcatalog])
 
 dnl Bash is needed for building man pages and running regression tests.
 dnl BASH is already an environment variable, so use something else.
 AC_PATH_PROG([BASH_PATH], [bash])
 AS_IF([test "x${BASH_PATH}" != "x"], [],
       [AC_MSG_FAILURE([Could not find required build tool bash])])
 
 AC_PATH_PROGS(VALGRIND_BIN, valgrind, /usr/bin/valgrind)
 AC_DEFINE_UNQUOTED(VALGRIND_BIN, "$VALGRIND_BIN", Valgrind command)
 
 AM_CONDITIONAL(BUILD_HELP, test x"${HELP2MAN}" != x"")
 if test x"${HELP2MAN}" != x""; then
     PCMK_FEATURES="$PCMK_FEATURES generated-manpages"
 fi
 
 MANPAGE_XSLT=""
 if test x"${XSLTPROC}" != x""; then
     AC_MSG_CHECKING([for DocBook-to-manpage transform])
     # first try to figure out correct template using xmlcatalog query,
     # resort to extensive (semi-deterministic) file search if that fails
     DOCBOOK_XSL_URI='http://docbook.sourceforge.net/release/xsl/current'
     DOCBOOK_XSL_PATH='manpages/docbook.xsl'
     MANPAGE_XSLT=$(${XMLCATALOG} "" ${DOCBOOK_XSL_URI}/${DOCBOOK_XSL_PATH} \
                    | sed -n 's|^file://||p;q')
     if test x"${MANPAGE_XSLT}" = x""; then
         DIRS=$(find "${datadir}" -name $(basename $(dirname ${DOCBOOK_XSL_PATH})) \
                -type d 2>/dev/null | LC_ALL=C sort)
         XSLT=$(basename ${DOCBOOK_XSL_PATH})
         for d in ${DIRS}
         do
             if test -f "${d}/${XSLT}"; then
                  MANPAGE_XSLT="${d}/${XSLT}"
                  break
             fi
         done
     fi
 fi
 AC_MSG_RESULT([$MANPAGE_XSLT])
 AC_SUBST(MANPAGE_XSLT)
 
 AM_CONDITIONAL(BUILD_XML_HELP, test x"${MANPAGE_XSLT}" != x"")
 if test x"${MANPAGE_XSLT}" != x""; then
     PCMK_FEATURES="$PCMK_FEATURES agent-manpages"
 fi
 
 AM_CONDITIONAL([IS_ASCIIDOC], [echo "${ASCIIDOC_CONV}" | grep -Eq 'asciidoc$'])
 AM_CONDITIONAL([BUILD_ASCIIDOC], [test "x${ASCIIDOC_CONV}" != x])
 if test "x${ASCIIDOC_CONV}" != x; then
     PCMK_FEATURES="$PCMK_FEATURES ascii-docs"
 fi
 
 AM_CONDITIONAL([BUILD_SPHINX_DOCS],
                [test x"${SPHINX}" != x"" && test x"${INKSCAPE}" != x""])
 AM_COND_IF([BUILD_SPHINX_DOCS], [PCMK_FEATURES="$PCMK_FEATURES books"])
 
 dnl Pacemaker's shell scripts (and thus man page builders) rely on GNU getopt
 AC_MSG_CHECKING([for GNU-compatible getopt])
 IFS_orig=$IFS
 IFS=:
 for PATH_DIR in $PATH
 do
     IFS=$IFS_orig
     GETOPT_PATH="${PATH_DIR}/getopt"
     if test -f "$GETOPT_PATH" && test -x "$GETOPT_PATH" ; then
         $GETOPT_PATH -T >/dev/null 2>/dev/null
         if test $? -eq 4; then
             break
         fi
     fi
     GETOPT_PATH=""
 done
 IFS=$IFS_orig
 AS_IF([test -n "$GETOPT_PATH"], [AC_MSG_RESULT([$GETOPT_PATH])],
       [
           AC_MSG_RESULT([no])
           AC_MSG_ERROR([Could not find required build tool GNU-compatible getopt])
       ])
 AC_SUBST([GETOPT_PATH])
 
 dnl ========================================================================
 dnl checks for library functions to replace them
 dnl
 dnl     NoSuchFunctionName:
 dnl             is a dummy function which no system supplies.  It is here to make
 dnl             the system compile semi-correctly on OpenBSD which doesn't know
 dnl             how to create an empty archive
 dnl
 dnl     scandir: Only on BSD.
 dnl             System-V systems may have it, but hidden and/or deprecated.
 dnl             A replacement function is supplied for it.
 dnl
 dnl     strerror: returns a string that corresponds to an errno.
 dnl             A replacement function is supplied for it.
 dnl
 dnl     strnlen: is a gnu function similar to strlen, but safer.
 dnl            We wrote a tolerably-fast replacement function for it.
 dnl
 dnl     strndup: is a gnu function similar to strdup, but safer.
 dnl            We wrote a tolerably-fast replacement function for it.
 
 AC_REPLACE_FUNCS(alphasort NoSuchFunctionName scandir strerror strchrnul strnlen strndup)
 
 dnl ===============================================
 dnl Libraries
 dnl ===============================================
 AC_CHECK_LIB(socket, socket)                    dnl -lsocket
 AC_CHECK_LIB(c, dlopen)                         dnl if dlopen is in libc...
 AC_CHECK_LIB(dl, dlopen)                        dnl -ldl (for Linux)
 AC_CHECK_LIB(rt, sched_getscheduler)            dnl -lrt (for Tru64)
 AC_CHECK_LIB(gnugetopt, getopt_long)            dnl -lgnugetopt ( if available )
 AC_CHECK_LIB(pam, pam_start)                    dnl -lpam (if available)
 
 PKG_CHECK_MODULES([UUID], [uuid],
                   [CPPFLAGS="${CPPFLAGS} ${UUID_CFLAGS}"
                    LIBS="${LIBS} ${UUID_LIBS}"])
 
 AC_CHECK_FUNCS([sched_setscheduler])
 if test "$ac_cv_func_sched_setscheduler" != yes; then
         PC_LIBS_RT=""
 else
         PC_LIBS_RT="-lrt"
 fi
 AC_SUBST(PC_LIBS_RT)
 
 # Require minimum glib version
 PKG_CHECK_MODULES([GLIB], [glib-2.0 >= 2.42.0],
                   [CPPFLAGS="${CPPFLAGS} ${GLIB_CFLAGS}"
                    LIBS="${LIBS} ${GLIB_LIBS}"])
 
 # Check whether high-resolution sleep function is available
 AC_CHECK_FUNCS([nanosleep usleep])
 
 #
 # Where is dlopen?
 #
 if test "$ac_cv_lib_c_dlopen" = yes; then
     LIBADD_DL=""
 elif test "$ac_cv_lib_dl_dlopen" = yes; then
     LIBADD_DL=-ldl
 else
     LIBADD_DL=${lt_cv_dlopen_libs}
 fi
 
 PKG_CHECK_MODULES(LIBXML2, [libxml-2.0],
                   [CPPFLAGS="${CPPFLAGS} ${LIBXML2_CFLAGS}"
                    LIBS="${LIBS} ${LIBXML2_LIBS}"])
 
 REQUIRE_LIB([xslt], [xsltApplyStylesheet])
 
 dnl ========================================================================
 dnl Headers
 dnl ========================================================================
 
 # Some distributions insert #warnings into deprecated headers. If we will
 # enable fatal warnings for the build, then enable them for the header checks
 # as well, otherwise the build could fail even though the header check
 # succeeds. (We should probably be doing this in more places.)
 cc_temp_flags "$CFLAGS $WERROR"
 
 # Optional headers (inclusion of these should be conditional in C code)
 AC_CHECK_HEADERS([getopt.h])
 AC_CHECK_HEADERS([linux/swab.h])
 AC_CHECK_HEADERS([stddef.h])
 AC_CHECK_HEADERS([sys/signalfd.h])
 AC_CHECK_HEADERS([uuid/uuid.h])
 AC_CHECK_HEADERS([security/pam_appl.h pam/pam_appl.h])
 
 # Required headers
 REQUIRE_HEADER([arpa/inet.h])
 REQUIRE_HEADER([ctype.h])
 REQUIRE_HEADER([dirent.h])
 REQUIRE_HEADER([errno.h])
 REQUIRE_HEADER([glib.h])
 REQUIRE_HEADER([grp.h])
 REQUIRE_HEADER([limits.h])
 REQUIRE_HEADER([netdb.h])
 REQUIRE_HEADER([netinet/in.h])
 REQUIRE_HEADER([netinet/ip.h], [
     #include <sys/types.h>
     #include <netinet/in.h>
 ])
 REQUIRE_HEADER([pwd.h])
 REQUIRE_HEADER([signal.h])
 REQUIRE_HEADER([stdio.h])
 REQUIRE_HEADER([stdlib.h])
 REQUIRE_HEADER([string.h])
 REQUIRE_HEADER([strings.h])
 REQUIRE_HEADER([sys/ioctl.h])
 REQUIRE_HEADER([sys/param.h])
 REQUIRE_HEADER([sys/reboot.h])
 REQUIRE_HEADER([sys/resource.h])
 REQUIRE_HEADER([sys/socket.h])
 REQUIRE_HEADER([sys/stat.h])
 REQUIRE_HEADER([sys/time.h])
 REQUIRE_HEADER([sys/types.h])
 REQUIRE_HEADER([sys/utsname.h])
 REQUIRE_HEADER([sys/wait.h])
 REQUIRE_HEADER([time.h])
 REQUIRE_HEADER([unistd.h])
 REQUIRE_HEADER([libxml/xpath.h])
 REQUIRE_HEADER([libxslt/xslt.h])
 
 cc_restore_flags
 
 AC_CHECK_FUNCS([uuid_unparse], [],
                [AC_MSG_FAILURE([Could not find required C function uuid_unparse()])])
 
 AC_CACHE_CHECK([whether __progname and __progname_full are available],
                [pf_cv_var_progname], AC_LINK_IFELSE([
                    AC_LANG_PROGRAM([[extern char *__progname, *__progname_full;]],
                                    [[__progname = "foo"; __progname_full = "foo bar";]],
                                    [pf_cv_var_progname="yes"],
                                    [pf_cv_var_progname="no"])
                ]))
 AS_IF([test "$pf_cv_var_progname" = "yes"], [AC_DEFINE(HAVE___PROGNAME,1,[ ])])
 
 dnl ========================================================================
 dnl Generic declarations
 dnl ========================================================================
 
 AC_CHECK_DECLS([CLOCK_MONOTONIC], [PCMK_FEATURES="$PCMK_FEATURES monotonic"], [], [[
     #include <time.h>
 ]])
 
 dnl ========================================================================
 dnl Structures
 dnl ========================================================================
 
 AC_CHECK_MEMBERS([struct tm.tm_gmtoff],,,[[#include <time.h>]])
 AC_CHECK_MEMBER([struct dirent.d_type],
     AC_DEFINE(HAVE_STRUCT_DIRENT_D_TYPE,1,[Define this if struct dirent has d_type]),,
     [#include <dirent.h>])
 
 dnl ========================================================================
 dnl Functions
 dnl ========================================================================
 
 REQUIRE_FUNC([getopt])
 REQUIRE_FUNC([setenv])
 REQUIRE_FUNC([unsetenv])
 
 AC_CACHE_CHECK(whether sscanf supports %m,
                pf_cv_var_sscanf,
                AC_RUN_IFELSE([AC_LANG_SOURCE([[
 #include <stdio.h>
 const char *s = "some-command-line-arg";
 int main(int argc, char **argv) {
 char *name = NULL;
 int n = sscanf(s, "%ms", &name);
 return n == 1 ? 0 : 1;
 }
 ]])],
                                  pf_cv_var_sscanf="yes", pf_cv_var_sscanf="no", pf_cv_var_sscanf="no"))
 
 AS_IF([test "$pf_cv_var_sscanf" = "yes"], [AC_DEFINE(SSCANF_HAS_M, 1, [ ])])
 
 dnl ========================================================================
 dnl   bzip2
 dnl ========================================================================
 REQUIRE_HEADER([bzlib.h])
 REQUIRE_LIB([bz2], [BZ2_bzBuffToBuffCompress])
 
 dnl ========================================================================
 dnl sighandler_t is missing from Illumos, Solaris11 systems
 dnl ========================================================================
 
 AC_MSG_CHECKING([for sighandler_t])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <signal.h>]], [[sighandler_t *f;]])],
                   [
                       AC_MSG_RESULT([yes])
                       AC_DEFINE([HAVE_SIGHANDLER_T], [1],
                                 [Define to 1 if sighandler_t is available])
                   ],
                   [AC_MSG_RESULT([no])])
 
 dnl ========================================================================
 dnl   ncurses
 dnl ========================================================================
 dnl
 dnl A few OSes (e.g. Linux) deliver a default "ncurses" alongside "curses".
 dnl Many non-Linux deliver "curses"; sites may add "ncurses".
 dnl
 dnl However, the source-code recommendation for both is to #include "curses.h"
 dnl (i.e. "ncurses" still wants the include to be simple, no-'n', "curses.h").
 dnl
 dnl ncurses takes precedence.
 dnl
 AC_CHECK_HEADERS([curses.h curses/curses.h ncurses.h ncurses/ncurses.h])
 
 dnl Although n-library is preferred, only look for it if the n-header was found.
 CURSESLIBS=''
 PC_NAME_CURSES=""
 PC_LIBS_CURSES=""
 AS_IF([test "$ac_cv_header_ncurses_h" = "yes"], [
     AC_CHECK_LIB(ncurses, printw,
                  [AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
     CURSESLIBS=`$PKG_CONFIG --libs ncurses` || CURSESLIBS='-lncurses'
     PC_NAME_CURSES="ncurses"
 ])
 
 AS_IF([test "$ac_cv_header_ncurses_ncurses_h" = "yes"], [
     AC_CHECK_LIB(ncurses, printw,
                  [AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
     CURSESLIBS=`$PKG_CONFIG --libs ncurses` || CURSESLIBS='-lncurses'
     PC_NAME_CURSES="ncurses"
 ])
 
 dnl Only look for non-n-library if there was no n-library.
 AS_IF([test X"$CURSESLIBS" = X"" && test "$ac_cv_header_curses_h" = "yes"], [
     AC_CHECK_LIB(curses, printw,
                  [CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
     PC_LIBS_CURSES="$CURSESLIBS"
 ])
 
 dnl Only look for non-n-library if there was no n-library.
 AS_IF([test X"$CURSESLIBS" = X"" && test "$ac_cv_header_curses_curses_h" = "yes"], [
     AC_CHECK_LIB(curses, printw,
                  [CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
     PC_LIBS_CURSES="$CURSESLIBS"
 ])
 
 if test "x$CURSESLIBS" != "x"; then
     PCMK_FEATURES="$PCMK_FEATURES ncurses"
 fi
 
 dnl Check for printw() prototype compatibility
 AS_IF([test X"$CURSESLIBS" != X"" && cc_supports_flag -Wcast-qual], [
     ac_save_LIBS=$LIBS
     LIBS="$CURSESLIBS"
     cc_temp_flags "-Wcast-qual $WERROR"
     # avoid broken test because of hardened build environment in Fedora 23+
     # - https://fedoraproject.org/wiki/Changes/Harden_All_Packages
     # - https://bugzilla.redhat.com/1297985
     AS_IF([cc_supports_flag -fPIC], [CFLAGS="$CFLAGS -fPIC"])
 
     AC_MSG_CHECKING([whether curses library is compatible])
     AC_LINK_IFELSE(
         [AC_LANG_PROGRAM([
 #if defined(HAVE_NCURSES_H)
 #  include <ncurses.h>
 #elif defined(HAVE_NCURSES_NCURSES_H)
 #  include <ncurses/ncurses.h>
 #elif defined(HAVE_CURSES_H)
 #  include <curses.h>
 #endif
                          ],
                          [printw((const char *)"Test");]
         )],
         [AC_MSG_RESULT([yes])],
         [
             AC_MSG_RESULT([no])
             AC_MSG_WARN(m4_normalize([Disabling curses because the printw()
                                       function of your (n)curses library is old.
                                       If you wish to enable curses, update to a
                                       newer version (ncurses 5.4 or later is
                                       recommended, available from
                                       https://invisible-island.net/ncurses/)
                                      ]))
             AC_DEFINE([HAVE_INCOMPATIBLE_PRINTW], [1],
                       [Define to 1 if curses library has incompatible printw()])
         ]
     )
 
     LIBS=$ac_save_LIBS
     cc_restore_flags
 ])
 
 AC_SUBST(CURSESLIBS)
 AC_SUBST(PC_NAME_CURSES)
 AC_SUBST(PC_LIBS_CURSES)
 
 dnl ========================================================================
 dnl    Profiling and GProf
 dnl ========================================================================
 
 CFLAGS_ORIG="$CFLAGS"
 AS_IF([test $with_coverage -ne $DISABLED],
       [
         with_profiling=$REQUIRED
         PCMK_FEATURES="$PCMK_FEATURES coverage"
         CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage"
         dnl During linking, make sure to specify -lgcov or -coverage
       ]
 )
 
 AS_IF([test $with_profiling -ne $DISABLED],
       [
           with_profiling=$REQUIRED
           PCMK_FEATURES="$PCMK_FEATURES profile"
 
           dnl Disable various compiler optimizations
           CFLAGS="$CFLAGS -fno-omit-frame-pointer -fno-inline -fno-builtin"
           dnl CFLAGS="$CFLAGS -fno-inline-functions"
           dnl CFLAGS="$CFLAGS -fno-default-inline"
           dnl CFLAGS="$CFLAGS -fno-inline-functions-called-once"
           dnl CFLAGS="$CFLAGS -fno-optimize-sibling-calls"
 
           dnl Turn off optimization so tools can get accurate line numbers
           CFLAGS=`echo $CFLAGS | sed \
                   -e 's/-O.\ //g' \
                   -e 's/-Wp,-D_FORTIFY_SOURCE=.\ //g' \
                   -e 's/-D_FORTIFY_SOURCE=.\ //g'`
           CFLAGS="$CFLAGS -O0 -g3 -gdwarf-2"
 
           AC_MSG_NOTICE([CFLAGS before adding profiling options: $CFLAGS_ORIG])
           AC_MSG_NOTICE([CFLAGS after: $CFLAGS])
       ]
 )
 AC_DEFINE_UNQUOTED([SUPPORT_PROFILING], [$with_profiling], [Support profiling])
 AM_CONDITIONAL([BUILD_PROFILING], [test "$with_profiling" = "$REQUIRED"])
 
 dnl ========================================================================
 dnl    Cluster infrastructure - LibQB
 dnl ========================================================================
 
 PKG_CHECK_MODULES(libqb, libqb >= 0.17)
 CPPFLAGS="$libqb_CFLAGS $CPPFLAGS"
 LIBS="$libqb_LIBS $LIBS"
 
 dnl libqb 2.0.2+ (2020-10)
 AC_CHECK_FUNCS(qb_ipcc_auth_get,
                AC_DEFINE(HAVE_IPCC_AUTH_GET, 1,
                          [Have qb_ipcc_auth_get function]))
 
 dnl libqb 2.0.0+ (2020-05)
 CHECK_ENUM_VALUE([qb/qblog.h],[qb_log_conf],[QB_LOG_CONF_MAX_LINE_LEN])
 CHECK_ENUM_VALUE([qb/qblog.h],[qb_log_conf],[QB_LOG_CONF_ELLIPSIS])
 
 dnl Support Linux-HA fence agents if available
 if test "$cross_compiling" != "yes"; then
     CPPFLAGS="$CPPFLAGS -I${prefix}/include/heartbeat"
 fi
 AC_CHECK_HEADERS([stonith/stonith.h],
                  [
                      AC_CHECK_LIB([pils], [PILLoadPlugin])
                      AC_CHECK_LIB([plumb], [G_main_add_IPC_Channel])
                      PCMK_FEATURES="$PCMK_FEATURES lha"
                  ])
 AM_CONDITIONAL([BUILD_LHA_SUPPORT], [test "$ac_cv_header_stonith_stonith_h" = "yes"])
 
 dnl ===============================================
 dnl Variables needed for substitution
 dnl ===============================================
 CRM_SCHEMA_DIRECTORY="${datadir}/pacemaker"
 AC_DEFINE_UNQUOTED(CRM_SCHEMA_DIRECTORY,"$CRM_SCHEMA_DIRECTORY", Location for the Pacemaker Relax-NG Schema)
 AC_SUBST(CRM_SCHEMA_DIRECTORY)
 
 CRM_CORE_DIR="${localstatedir}/lib/pacemaker/cores"
 AC_DEFINE_UNQUOTED(CRM_CORE_DIR,"$CRM_CORE_DIR", Location to store core files produced by Pacemaker daemons)
 AC_SUBST(CRM_CORE_DIR)
 
 if test x"${CRM_DAEMON_USER}" = x""; then
     CRM_DAEMON_USER="hacluster"
 fi
 AC_DEFINE_UNQUOTED(CRM_DAEMON_USER,"$CRM_DAEMON_USER", User to run Pacemaker daemons as)
 AC_SUBST(CRM_DAEMON_USER)
 
 if test x"${CRM_DAEMON_GROUP}" = x""; then
     CRM_DAEMON_GROUP="haclient"
 fi
 AC_DEFINE_UNQUOTED(CRM_DAEMON_GROUP,"$CRM_DAEMON_GROUP", Group to run Pacemaker daemons as)
 AC_SUBST(CRM_DAEMON_GROUP)
 
 CRM_PACEMAKER_DIR=${localstatedir}/lib/pacemaker
 AC_DEFINE_UNQUOTED(CRM_PACEMAKER_DIR,"$CRM_PACEMAKER_DIR", Location to store directory produced by Pacemaker daemons)
 AC_SUBST(CRM_PACEMAKER_DIR)
 
 CRM_BLACKBOX_DIR=${localstatedir}/lib/pacemaker/blackbox
 AC_DEFINE_UNQUOTED(CRM_BLACKBOX_DIR,"$CRM_BLACKBOX_DIR", Where to keep blackbox dumps)
 AC_SUBST(CRM_BLACKBOX_DIR)
 
 PE_STATE_DIR="${localstatedir}/lib/pacemaker/pengine"
 AC_DEFINE_UNQUOTED(PE_STATE_DIR,"$PE_STATE_DIR", Where to keep scheduler outputs)
 AC_SUBST(PE_STATE_DIR)
 
 CRM_CONFIG_DIR="${localstatedir}/lib/pacemaker/cib"
 AC_DEFINE_UNQUOTED(CRM_CONFIG_DIR,"$CRM_CONFIG_DIR", Where to keep configuration files)
 AC_SUBST(CRM_CONFIG_DIR)
 
 CRM_CONFIG_CTS="${localstatedir}/lib/pacemaker/cts"
 AC_DEFINE_UNQUOTED(CRM_CONFIG_CTS,"$CRM_CONFIG_CTS", Where to keep cts stateful data)
 AC_SUBST(CRM_CONFIG_CTS)
 
 CRM_DAEMON_DIR="${libexecdir}/pacemaker"
 AC_DEFINE_UNQUOTED(CRM_DAEMON_DIR,"$CRM_DAEMON_DIR", Location for Pacemaker daemons)
 AC_SUBST(CRM_DAEMON_DIR)
 
 CRM_STATE_DIR="${runstatedir}/crm"
 AC_DEFINE_UNQUOTED([CRM_STATE_DIR], ["$CRM_STATE_DIR"],
                    [Where to keep state files and sockets])
 AC_SUBST(CRM_STATE_DIR)
 
 CRM_RSCTMP_DIR="${runstatedir}/resource-agents"
 AC_DEFINE_UNQUOTED(CRM_RSCTMP_DIR,"$CRM_RSCTMP_DIR", Where resource agents should keep state files)
 AC_SUBST(CRM_RSCTMP_DIR)
 
 PACEMAKER_CONFIG_DIR="${sysconfdir}/pacemaker"
 AC_DEFINE_UNQUOTED(PACEMAKER_CONFIG_DIR,"$PACEMAKER_CONFIG_DIR", Where to keep configuration files like authkey)
 AC_SUBST(PACEMAKER_CONFIG_DIR)
 
 AC_DEFINE_UNQUOTED(SBIN_DIR,"$sbindir",[Location for system binaries])
 
 AC_PATH_PROGS(GIT, git false)
 
 AC_MSG_CHECKING([build version])
 BUILD_VERSION=$Format:%h$
 if test $BUILD_VERSION != ":%h$"; then
     AC_MSG_RESULT([$BUILD_VERSION (archive hash)])
 elif test -x $GIT && test -d .git; then
     BUILD_VERSION=`$GIT log --pretty="format:%h" -n 1`
     AC_MSG_RESULT([$BUILD_VERSION (git hash)])
 else
     # The current directory name make a reasonable default
     # Most generated archives will include the hash or tag
     BASE=`basename $PWD`
     BUILD_VERSION=`echo $BASE | sed s:.*[[Pp]]acemaker-::`
     AC_MSG_RESULT([$BUILD_VERSION (directory name)])
 fi
 
 AC_DEFINE_UNQUOTED(BUILD_VERSION, "$BUILD_VERSION", Build version)
 AC_SUBST(BUILD_VERSION)
 
 HAVE_dbus=1
 PKG_CHECK_MODULES([DBUS], [dbus-1],
                   [CPPFLAGS="${CPPFLAGS} ${DBUS_CFLAGS}"],
                   [HAVE_dbus=0])
 AC_DEFINE_UNQUOTED(SUPPORT_DBUS, $HAVE_dbus, Support dbus)
 AM_CONDITIONAL(BUILD_DBUS, test $HAVE_dbus = 1)
 AC_CHECK_TYPES([DBusBasicValue],,,[[#include <dbus/dbus.h>]])
 if test $HAVE_dbus = 0; then
         PC_NAME_DBUS=""
 else
         PC_NAME_DBUS="dbus-1"
 fi
 AC_SUBST(PC_NAME_DBUS)
 
 AS_CASE([$enable_systemd],
         [$REQUIRED], [
             AS_IF([test $HAVE_dbus = 0],
                   [AC_MSG_FAILURE([Cannot support systemd resources without DBus])])
             AS_IF([test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
                   [AC_MSG_FAILURE([Cannot support systemd resources without monotonic clock])])
             AS_IF([check_systemdsystemunitdir], [],
                   [AC_MSG_FAILURE([Cannot support systemd resources without systemdsystemunitdir])])
         ],
         [$OPTIONAL], [
             AS_IF([test $HAVE_dbus = 0 \
                    || test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
                   [enable_systemd=$DISABLED],
                   [
                       AC_MSG_CHECKING([for systemd version (using dbus-send)])
                       ret=$({ dbus-send --system --print-reply \
                                   --dest=org.freedesktop.systemd1 \
                                   /org/freedesktop/systemd1 \
                                   org.freedesktop.DBus.Properties.Get \
                                   string:org.freedesktop.systemd1.Manager \
                                   string:Version 2>/dev/null \
                               || echo "version unavailable"; } | tail -n1)
                       # sanitize output a bit (interested just in value, not type),
                       # ret is intentionally unenquoted so as to normalize whitespace
                       ret=$(echo ${ret} | cut -d' ' -f2-)
                       AC_MSG_RESULT([${ret}])
                       AS_IF([test "$ret" != "unavailable" \
                              || systemctl --version 2>/dev/null | grep -q systemd],
                             [
                                 AS_IF([check_systemdsystemunitdir],
                                       [enable_systemd=$REQUIRED],
                                       [enable_systemd=$DISABLED])
                             ],
                             [enable_systemd=$DISABLED]
                       )
                   ])
         ],
 )
 AC_MSG_CHECKING([whether to enable support for managing resources via systemd])
 AS_IF([test $enable_systemd -eq $DISABLED], [AC_MSG_RESULT([no])],
       [
           AC_MSG_RESULT([yes])
           PCMK_FEATURES="$PCMK_FEATURES systemd"
       ]
 )
 AC_SUBST([systemdsystemunitdir])
 AC_DEFINE_UNQUOTED([SUPPORT_SYSTEMD], [$enable_systemd],
                    [Support systemd resources])
 AM_CONDITIONAL([BUILD_SYSTEMD], [test $enable_systemd = $REQUIRED])
 AC_SUBST(SUPPORT_SYSTEMD)
 
 AS_CASE([$enable_upstart],
         [$REQUIRED], [
             AS_IF([test $HAVE_dbus = 0],
                   [AC_MSG_FAILURE([Cannot support Upstart resources without DBus])])
         ],
         [$OPTIONAL], [
             AS_IF([test $HAVE_dbus = 0], [enable_upstart=$DISABLED],
                   [
                       AC_MSG_CHECKING([for Upstart version (using dbus-send)])
                       ret=$({ dbus-send --system --print-reply \
                                   --dest=com.ubuntu.Upstart \
                                   /com/ubuntu/Upstart org.freedesktop.DBus.Properties.Get \
                                   string:com.ubuntu.Upstart0_6 string:version 2>/dev/null \
                               || echo "version unavailable"; } | tail -n1)
                       # sanitize output a bit (interested just in value, not type),
                       # ret is intentionally unenquoted so as to normalize whitespace
                       ret=$(echo ${ret} | cut -d' ' -f2-)
                       AC_MSG_RESULT([${ret}])
                       AS_IF([test "$ret" != "unavailable" \
                              || initctl --version 2>/dev/null | grep -q upstart],
                             [enable_upstart=$REQUIRED],
                             [enable_upstart=$DISABLED]
                       )
                   ])
         ],
 )
 AC_MSG_CHECKING([whether to enable support for managing resources via Upstart])
 AS_IF([test $enable_upstart -eq $DISABLED], [AC_MSG_RESULT([no])],
       [
           AC_MSG_RESULT([yes])
           PCMK_FEATURES="$PCMK_FEATURES upstart"
       ]
 )
 AC_DEFINE_UNQUOTED([SUPPORT_UPSTART], [$enable_upstart],
                    [Support Upstart resources])
 AM_CONDITIONAL([BUILD_UPSTART], [test $enable_upstart -eq $REQUIRED])
 AC_SUBST(SUPPORT_UPSTART)
 
 AS_CASE([$with_nagios],
         [$REQUIRED], [
             AS_IF([test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
                   [AC_MSG_FAILURE([Cannot support nagios resources without monotonic clock])])
         ],
         [$OPTIONAL], [
             AS_IF([test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
                   [with_nagios=$DISABLED], [with_nagios=$REQUIRED])
         ]
 )
 AS_IF([test $with_nagios -eq $REQUIRED], [PCMK_FEATURES="$PCMK_FEATURES nagios"])
 AC_DEFINE_UNQUOTED([SUPPORT_NAGIOS], [$with_nagios], [Support nagios plugins])
 AM_CONDITIONAL([BUILD_NAGIOS], [test $with_nagios -eq $REQUIRED])
 
 if test x"$NAGIOS_PLUGIN_DIR" = x""; then
     NAGIOS_PLUGIN_DIR="${libexecdir}/nagios/plugins"
 fi
 
 AC_DEFINE_UNQUOTED(NAGIOS_PLUGIN_DIR, "$NAGIOS_PLUGIN_DIR", Directory for nagios plugins)
 AC_SUBST(NAGIOS_PLUGIN_DIR)
 
 if test x"$NAGIOS_METADATA_DIR" = x""; then
     NAGIOS_METADATA_DIR="${datadir}/nagios/plugins-metadata"
 fi
 
 AC_DEFINE_UNQUOTED(NAGIOS_METADATA_DIR, "$NAGIOS_METADATA_DIR", Directory for nagios plugins metadata)
 AC_SUBST(NAGIOS_METADATA_DIR)
 
 STACKS=""
 CLUSTERLIBS=""
 PC_NAME_CLUSTER=""
 
 dnl ========================================================================
 dnl    Cluster stack - Corosync
 dnl ========================================================================
 
 COROSYNC_LIBS=""
 
 AS_CASE([$with_corosync],
         [$REQUIRED], [
             # These will be fatal if unavailable
             PKG_CHECK_MODULES([cpg], [libcpg])
             PKG_CHECK_MODULES([cfg], [libcfg])
             PKG_CHECK_MODULES([cmap], [libcmap])
             PKG_CHECK_MODULES([quorum], [libquorum])
             PKG_CHECK_MODULES([libcorosync_common], [libcorosync_common])
         ]
         [$OPTIONAL], [
             PKG_CHECK_MODULES([cpg], [libcpg], [], [with_corosync=$DISABLED])
             PKG_CHECK_MODULES([cfg], [libcfg], [], [with_corosync=$DISABLED])
             PKG_CHECK_MODULES([cmap], [libcmap], [], [with_corosync=$DISABLED])
             PKG_CHECK_MODULES([quorum], [libquorum], [], [with_corosync=$DISABLED])
             PKG_CHECK_MODULES([libcorosync_common], [libcorosync_common], [], [with_corosync=$DISABLED])
             AS_IF([test $with_corosync -ne $DISABLED], [with_corosync=$REQUIRED])
         ]
 )
 AS_IF([test $with_corosync -ne $DISABLED],
       [
           AC_MSG_CHECKING([for Corosync 2 or later])
           AC_MSG_RESULT([yes])
           CFLAGS="$CFLAGS $libqb_CFLAGS $cpg_CFLAGS $cfg_CFLAGS $cmap_CFLAGS $quorum_CFLAGS $libcorosync_common_CFLAGS"
           COROSYNC_LIBS="$COROSYNC_LIBS $cpg_LIBS $cfg_LIBS $cmap_LIBS $quorum_LIBS $libcorosync_common_LIBS"
           CLUSTERLIBS="$CLUSTERLIBS $COROSYNC_LIBS"
           PC_NAME_CLUSTER="$PC_CLUSTER_NAME libcfg libcmap libcorosync_common libcpg libquorum"
           STACKS="$STACKS corosync-ge-2"
 
           dnl Shutdown tracking added (back) to corosync Jan 2021
           saved_LIBS="$LIBS"
           LIBS="$LIBS $COROSYNC_LIBS"
           AC_CHECK_FUNCS(corosync_cfg_trackstart,
           AC_DEFINE(HAVE_COROSYNC_CFG_TRACKSTART, 1,
                          [Have corosync_cfg_trackstart function]))
           LIBS="$saved_LIBS"
       ]
 )
 AC_DEFINE_UNQUOTED([SUPPORT_COROSYNC], [$with_corosync],
                    [Support the Corosync messaging and membership layer])
 AM_CONDITIONAL([BUILD_CS_SUPPORT], [test $with_corosync -eq $REQUIRED])
 AC_SUBST([SUPPORT_COROSYNC])
 
 dnl
 dnl    Cluster stack - Sanity
 dnl
 
 AS_IF([test "x$STACKS" != "x"], [AC_MSG_NOTICE([Supported stacks:${STACKS}])],
       [AC_MSG_FAILURE([At least one cluster stack must be supported])])
 
 PCMK_FEATURES="${PCMK_FEATURES}${STACKS}"
 
 AC_SUBST(CLUSTERLIBS)
 AC_SUBST(PC_NAME_CLUSTER)
 
 dnl ========================================================================
 dnl    CIB secrets
 dnl ========================================================================
 
 AS_IF([test $with_cibsecrets -ne $DISABLED],
       [
           with_cibsecrets=$REQUIRED
           PCMK_FEATURES="$PCMK_FEATURES cibsecrets"
           LRM_CIBSECRETS_DIR="${localstatedir}/lib/pacemaker/lrm/secrets"
           AC_DEFINE_UNQUOTED([LRM_CIBSECRETS_DIR], ["$LRM_CIBSECRETS_DIR"],
                              [Location for CIB secrets])
           AC_SUBST([LRM_CIBSECRETS_DIR])
       ]
 )
 AC_DEFINE_UNQUOTED([SUPPORT_CIBSECRETS], [$with_cibsecrets], [Support CIB secrets])
 AM_CONDITIONAL([BUILD_CIBSECRETS], [test $with_cibsecrets -eq $REQUIRED])
 
 dnl ========================================================================
 dnl    GnuTLS
 dnl ========================================================================
 
 dnl Require GnuTLS >=2.12.0 (2011-03) for Pacemaker Remote support
 PC_NAME_GNUTLS=""
 AS_CASE([$with_gnutls],
         [$REQUIRED], [
             REQUIRE_LIB([gnutls], [gnutls_sec_param_to_pk_bits])
             REQUIRE_HEADER([gnutls/gnutls.h])
         ],
         [$OPTIONAL], [
             AC_CHECK_LIB([gnutls], [gnutls_sec_param_to_pk_bits],
                          [], [with_gnutls=$DISABLED])
             AC_CHECK_HEADERS([gnutls/gnutls.h], [], [with_gnutls=$DISABLED])
         ]
 )
 AS_IF([test $with_gnutls -ne $DISABLED],
       [
           PC_NAME_GNUTLS="gnutls"
           PCMK_FEATURES="$PCMK_FEATURES remote"
       ]
 )
 AC_SUBST([PC_NAME_GNUTLS])
 AM_CONDITIONAL([BUILD_REMOTE], [test $with_gnutls -ne $DISABLED])
 
 dnl ========================================================================
 dnl    System Health
 dnl ========================================================================
 
 dnl Check if servicelog development package is installed
 SERVICELOG=servicelog-1
 SERVICELOG_EXISTS="no"
 AC_MSG_CHECKING([for $SERVICELOG packages])
 if
     $PKG_CONFIG --exists $SERVICELOG
 then
     PKG_CHECK_MODULES([SERVICELOG], [servicelog-1])
     SERVICELOG_EXISTS="yes"
     PCMK_FEATURES="$PCMK_FEATURES servicelog"
 fi
 AC_MSG_RESULT([$SERVICELOG_EXISTS])
 AM_CONDITIONAL(BUILD_SERVICELOG, test "$SERVICELOG_EXISTS" = "yes")
 
 dnl Check if OpenIMPI packages and servicelog are installed
 OPENIPMI="OpenIPMI OpenIPMIposix"
 OPENIPMI_SERVICELOG_EXISTS="no"
 AC_MSG_CHECKING([for $SERVICELOG $OPENIPMI packages])
 if
     $PKG_CONFIG --exists $OPENIPMI $SERVICELOG
 then
     PKG_CHECK_MODULES([OPENIPMI_SERVICELOG],[OpenIPMI OpenIPMIposix])
     REQUIRE_HEADER([malloc.h])
     OPENIPMI_SERVICELOG_EXISTS="yes"
     PCMK_FEATURES="$PCMK_FEATURES ipmiservicelogd"
 fi
 AC_MSG_RESULT([$OPENIPMI_SERVICELOG_EXISTS])
 AM_CONDITIONAL(BUILD_OPENIPMI_SERVICELOG, test "$OPENIPMI_SERVICELOG_EXISTS" = "yes")
 
 # --- ASAN/UBSAN/TSAN (see man gcc) ---
 # when using SANitizers, we need to pass the -fsanitize..
 # to both CFLAGS and LDFLAGS. The CFLAGS/LDFLAGS must be
 # specified as first in the list or there will be runtime
 # issues (for example user has to LD_PRELOAD asan for it to work
 # properly).
 
 AS_IF([test -n "${SANITIZERS}"], [
   SANITIZERS=$(echo $SANITIZERS | sed -e 's/,/ /g')
   for SANITIZER in $SANITIZERS
   do
     AS_CASE([$SANITIZER],
             [asan|ASAN], [
                 SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=address"
                 SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=address -lasan"
                 PCMK_FEATURES="$PCMK_FEATURES asan"
                 REQUIRE_LIB([asan],[main])
             ],
             [ubsan|UBSAN], [
                 SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=undefined"
                 SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=undefined -lubsan"
                 PCMK_FEATURES="$PCMK_FEATURES ubsan"
                 REQUIRE_LIB([ubsan],[main])
             ],
             [tsan|TSAN], [
                 SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=thread"
                 SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=thread -ltsan"
                 PCMK_FEATURES="$PCMK_FEATURES tsan"
                 REQUIRE_LIB([tsan],[main])
             ])
   done
 ])
 
 
 dnl ========================================================================
 dnl Compiler flags
 dnl ========================================================================
 
 dnl Make sure that CFLAGS is not exported. If the user did
 dnl not have CFLAGS in their environment then this should have
 dnl no effect. However if CFLAGS was exported from the user's
 dnl environment, then the new CFLAGS will also be exported
 dnl to sub processes.
 if export | fgrep " CFLAGS=" > /dev/null; then
     SAVED_CFLAGS="$CFLAGS"
     unset CFLAGS
     CFLAGS="$SAVED_CFLAGS"
     unset SAVED_CFLAGS
 fi
 
 AC_ARG_VAR([CFLAGS_HARDENED_LIB], [extra C compiler flags for hardened libraries])
 AC_ARG_VAR([LDFLAGS_HARDENED_LIB], [extra linker flags for hardened libraries])
 
 AC_ARG_VAR([CFLAGS_HARDENED_EXE], [extra C compiler flags for hardened executables])
 AC_ARG_VAR([LDFLAGS_HARDENED_EXE], [extra linker flags for hardened executables])
 
 CC_EXTRAS=""
 
 AS_IF([test "$GCC" != yes], [CFLAGS="$CFLAGS -g"], [
     CFLAGS="$CFLAGS -ggdb"
 
 dnl When we don't have diagnostic push / pull, we can't explicitly disable
 dnl checking for nonliteral formats in the places where they occur on purpose
 dnl thus we disable nonliteral format checking globally as we are aborting
 dnl on warnings. 
 dnl what makes the things really ugly is that nonliteral format checking is 
 dnl obviously available as an extra switch in very modern gcc but for older
 dnl gcc this is part of -Wformat=2 
 dnl so if we have push/pull we can enable -Wformat=2 -Wformat-nonliteral
 dnl if we don't have push/pull but -Wformat-nonliteral we can enable -Wformat=2
 dnl otherwise none of both
 
     gcc_diagnostic_push_pull=no
     cc_temp_flags "$CFLAGS $WERROR"
     AC_MSG_CHECKING([for gcc diagnostic push / pull])
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #pragma GCC diagnostic push
 #pragma GCC diagnostic pop
                       ]])],
                       [
                           AC_MSG_RESULT([yes])
                           gcc_diagnostic_push_pull=yes
                       ], AC_MSG_RESULT([no]))
     cc_restore_flags
 
     AS_IF([cc_supports_flag "-Wformat-nonliteral"],
           [gcc_format_nonliteral=yes],
           [gcc_format_nonliteral=no])
         
     # We had to eliminate -Wnested-externs because of libtool changes
     # Make sure to order options so that the former stand for prerequisites
     # of the latter (e.g., -Wformat-nonliteral requires -Wformat).
     EXTRA_FLAGS="-fgnu89-inline"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wall"
     EXTRA_FLAGS="$EXTRA_FLAGS -Waggregate-return"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wbad-function-cast"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wcast-align"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wdeclaration-after-statement"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wendif-labels"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wfloat-equal"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wformat-security"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wmissing-prototypes"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wmissing-declarations"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wnested-externs"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wno-long-long"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wno-strict-aliasing"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wpointer-arith"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wstrict-prototypes"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wwrite-strings"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wunused-but-set-variable"
     EXTRA_FLAGS="$EXTRA_FLAGS -Wunsigned-char"
 
     AS_IF([test "x$gcc_diagnostic_push_pull" = "xyes"],
           [
               AC_DEFINE([GCC_FORMAT_NONLITERAL_CHECKING_ENABLED], [],
                         [gcc can complain about nonliterals in format])
               EXTRA_FLAGS="$EXTRA_FLAGS -Wformat=2 -Wformat-nonliteral"
           ],
           [test "x$gcc_format_nonliteral" = "xyes"],
           [EXTRA_FLAGS="$EXTRA_FLAGS -Wformat=2"])
 
 # Additional warnings it might be nice to enable one day
 #                -Wshadow
 #                -Wunreachable-code
     for j in $EXTRA_FLAGS
     do
         AS_IF([cc_supports_flag $CC_EXTRAS $j], [CC_EXTRAS="$CC_EXTRAS $j"])
     done
 
     AC_MSG_NOTICE([Using additional gcc flags: ${CC_EXTRAS}])
 ])
 
 dnl
 dnl Hardening flags
 dnl
 dnl The prime control of whether to apply (targeted) hardening build flags and
 dnl which ones is --{enable,disable}-hardening option passed to ./configure:
 dnl
 dnl --enable-hardening=try (default):
 dnl     depending on whether any of CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE,
 dnl     CFLAGS_HARDENED_LIB or LDFLAGS_HARDENED_LIB environment variables
 dnl     (see below) is set and non-null, all these custom flags (even if not
 dnl     set) are used as are, otherwise the best effort is made to offer
 dnl     reasonably strong hardening in several categories (RELRO, PIE,
 dnl     "bind now", stack protector) according to what the selected toolchain
 dnl     can offer
 dnl
 dnl --enable-hardening:
 dnl     same effect as --enable-hardening=try when the environment variables
 dnl     in question are suppressed
 dnl
 dnl --disable-hardening:
 dnl     do not apply any targeted hardening measures at all
 dnl
 dnl The user-injected environment variables that regulate the hardening in
 dnl default case are as follows:
 dnl
 dnl * CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE
 dnl    compiler and linker flags (respectively) for daemon programs
 dnl    (pacemakerd, pacemaker-attrd, pacemaker-controld, pacemaker-execd,
 dnl    pacemaker-based, pacemaker-fenced, pacemaker-remoted,
 dnl    pacemaker-schedulerd)
 dnl
 dnl * CFLAGS_HARDENED_LIB, LDFLAGS_HARDENED_LIB
 dnl    compiler and linker flags (respectively) for libraries linked
 dnl    with the daemon programs
 dnl
 dnl Note that these are purposedly targeted variables (addressing particular
 dnl targets all over the scattered Makefiles) and have no effect outside of
 dnl the predestined scope (e.g., CLI utilities).  For a global reach,
 dnl use CFLAGS, LDFLAGS, etc. as usual.
 dnl
 dnl For guidance on the suitable flags consult, for instance:
 dnl https://fedoraproject.org/wiki/Changes/Harden_All_Packages#Detailed_Harden_Flags_Description
 dnl https://owasp.org/index.php/C-Based_Toolchain_Hardening#GCC.2FBinutils
 dnl
 
 AS_IF([test $enable_hardening -eq $OPTIONAL],
       [
           AS_IF([test "$(env | grep -Ec '^(C|LD)FLAGS_HARDENED_(EXE|LIB)=.')" = 0],
                 [enable_hardening=$REQUIRED],
                 [AC_MSG_NOTICE([Hardening: using custom flags from environment])]
           )
       ],
       [
           unset CFLAGS_HARDENED_EXE
           unset CFLAGS_HARDENED_LIB
           unset LDFLAGS_HARDENED_EXE
           unset LDFLAGS_HARDENED_LIB
       ]
 )
 AS_CASE([$enable_hardening],
         [$DISABLED], [AC_MSG_NOTICE([Hardening: explicitly disabled])],
         [$REQUIRED], [
             CFLAGS_HARDENED_EXE=
             CFLAGS_HARDENED_LIB=
             LDFLAGS_HARDENED_EXE=
             LDFLAGS_HARDENED_LIB=
             relro=0
             pie=0
             bindnow=0
             stackprot="none"
             # daemons incl. libs: partial RELRO
             flag="-Wl,-z,relro"
             CC_CHECK_LDFLAGS(["${flag}"],
                              [
                                  LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
                                  LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"
                                  relro=1
                              ])
             # daemons: PIE for both CFLAGS and LDFLAGS
             AS_IF([cc_supports_flag -fPIE],
                   [
                       flag="-pie"
                       CC_CHECK_LDFLAGS(["${flag}"],
                                        [
                                            CFLAGS_HARDENED_EXE="${CFLAGS_HARDENED_EXE} -fPIE"
                                            LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
                                            pie=1
                                        ])
                   ]
             )
             # daemons incl. libs: full RELRO if sensible + as-needed linking
             #                     so as to possibly mitigate startup performance
             #                     hit caused by excessive linking with unneeded
             #                     libraries
             AS_IF([test "${relro}" = 1 && test "${pie}" = 1],
                   [
                       flag="-Wl,-z,now"
                       CC_CHECK_LDFLAGS(["${flag}"],
                                        [
                                            LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
                                            LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"
                                            bindnow=1
                                        ])
                   ]
             )
             AS_IF([test "${bindnow}" = 1],
                   [
                       flag="-Wl,--as-needed"
                       CC_CHECK_LDFLAGS(["${flag}"],
                                        [
                                            LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
                                            LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"
                                        ])
                   ])
             # universal: prefer strong > all > default stack protector if possible
             flag=
             AS_IF([cc_supports_flag -fstack-protector-strong],
                   [
                       flag="-fstack-protector-strong"
                       stackprot="strong"
                   ],
                   [cc_supports_flag -fstack-protector-all],
                   [
                       flag="-fstack-protector-all"
                       stackprot="all"
                   ],
                   [cc_supports_flag -fstack-protector],
                   [
                       flag="-fstack-protector"
                       stackprot="default"
                   ]
             )
             AS_IF([test -n "${flag}"], [CC_EXTRAS="${CC_EXTRAS} ${flag}"])
             # universal: enable stack clash protection if possible
             AS_IF([cc_supports_flag -fstack-clash-protection],
                   [
                       CC_EXTRAS="${CC_EXTRAS} -fstack-clash-protection"
                       AS_IF([test "${stackprot}" = "none"],
                             [stackprot="clash-only"],
                             [stackprot="${stackprot}+clash"]
                       )
                   ]
             )
             # Log a summary
             AS_IF([test "${relro}" = 1 || test "${pie}" = 1 || test "${stackprot}" != "none"],
                   [AC_MSG_NOTICE(m4_normalize([Hardening:
                         relro=${relro}
                         pie=${pie}
                         bindnow=${bindnow}
                         stackprot=${stackprot}]))
                   ],
                   [AC_MSG_WARN([Hardening: no suitable features in the toolchain detected])]
             )
         ],
 )
 
 CFLAGS="$SANITIZERS_CFLAGS $CFLAGS $CC_EXTRAS"
 LDFLAGS="$SANITIZERS_LDFLAGS $LDFLAGS"
 CFLAGS_HARDENED_EXE="$SANITIZERS_CFLAGS $CFLAGS_HARDENED_EXE"
 LDFLAGS_HARDENED_EXE="$SANITIZERS_LDFLAGS $LDFLAGS_HARDENED_EXE"
 
 NON_FATAL_CFLAGS="$CFLAGS"
 AC_SUBST(NON_FATAL_CFLAGS)
 
 dnl
 dnl We reset CFLAGS to include our warnings *after* all function
 dnl checking goes on, so that our warning flags don't keep the
 dnl AC_*FUNCS() calls above from working.  In particular, -Werror will
 dnl *always* cause us troubles if we set it before here.
 dnl
 dnl
 AS_IF([test $enable_fatal_warnings -ne $DISABLED], [
     AC_MSG_NOTICE([Enabling fatal compiler warnings])
     CFLAGS="$CFLAGS $WERROR"
 ])
 AC_SUBST(CFLAGS)
 
 dnl This is useful for use in Makefiles that need to remove one specific flag
 CFLAGS_COPY="$CFLAGS"
 AC_SUBST(CFLAGS_COPY)
 
 AC_SUBST(LIBADD_DL)        dnl extra flags for dynamic linking libraries
 
 AC_SUBST(LOCALE)
 
 dnl Options for cleaning up the compiler output
 AS_IF([test $enable_quiet -ne $DISABLED],
       [
           AC_MSG_NOTICE([Suppressing make details])
           QUIET_LIBTOOL_OPTS="--silent"
           QUIET_MAKE_OPTS="-s"  # POSIX compliant
       ],
       [
           QUIET_LIBTOOL_OPTS=""
           QUIET_MAKE_OPTS=""
       ]
 )
 
 dnl Put the above variables to use
 LIBTOOL="${LIBTOOL} --tag=CC \$(QUIET_LIBTOOL_OPTS)"
 MAKEFLAGS="${MAKEFLAGS} ${QUIET_MAKE_OPTS}"
 
 # Make features list available (sorted alphabetically, without leading space)
 PCMK_FEATURES=`echo "$PCMK_FEATURES" | sed -e 's/^ //' -e 's/ /\n/g' | sort | xargs`
 AC_DEFINE_UNQUOTED(CRM_FEATURES, "$PCMK_FEATURES", Set of enabled features)
 AC_SUBST(PCMK_FEATURES)
 
 AC_SUBST(CC)
 AC_SUBST(MAKEFLAGS)
 AC_SUBST(LIBTOOL)
 AC_SUBST(QUIET_LIBTOOL_OPTS)
 
 dnl Files we output that need to be executable
 CONFIG_FILES_EXEC([cts/cts-cli],
                   [cts/cts-coverage],
                   [cts/cts-exec],
                   [cts/cts-fencing],
                   [cts/cts-regression],
                   [cts/cts-scheduler],
                   [cts/lxc_autogen.sh],
                   [cts/benchmark/clubench],
                   [cts/lab/CTSlab.py],
                   [cts/lab/OCFIPraTest.py],
                   [cts/lab/cluster_test],
                   [cts/lab/cts],
                   [cts/lab/cts-log-watcher],
                   [cts/support/LSBDummy],
                   [cts/support/cts-support],
                   [cts/support/fence_dummy],
                   [cts/support/pacemaker-cts-dummyd],
                   [daemons/fenced/fence_legacy],
                   [daemons/fenced/fence_watchdog],
                   [doc/abi-check],
                   [extra/resources/ClusterMon],
                   [extra/resources/HealthSMART],
                   [extra/resources/SysInfo],
                   [extra/resources/ifspeed],
                   [extra/resources/o2cb],
                   [maint/bumplibs],
                   [tools/crm_failcount],
                   [tools/crm_master],
                   [tools/crm_report],
                   [tools/crm_standby],
                   [tools/cibsecret],
                   [tools/pcmk_simtimes])
 
 dnl Other files we output
 AC_CONFIG_FILES(Makefile                                            \
                 cts/Makefile                                        \
                 cts/benchmark/Makefile                              \
                 cts/lab/Makefile                                    \
                 cts/lab/CTSvars.py                                  \
                 cts/scheduler/Makefile                              \
                 cts/scheduler/dot/Makefile                          \
                 cts/scheduler/exp/Makefile                          \
                 cts/scheduler/scores/Makefile                       \
                 cts/scheduler/stderr/Makefile                       \
                 cts/scheduler/summary/Makefile                      \
                 cts/scheduler/xml/Makefile                          \
                 cts/support/Makefile                                \
                 cts/support/pacemaker-cts-dummyd@.service           \
                 daemons/Makefile                                    \
                 daemons/attrd/Makefile                              \
                 daemons/based/Makefile                              \
                 daemons/controld/Makefile                           \
                 daemons/execd/Makefile                              \
                 daemons/execd/pacemaker_remote                      \
                 daemons/execd/pacemaker_remote.service              \
                 daemons/fenced/Makefile                             \
                 daemons/pacemakerd/Makefile                         \
                 daemons/pacemakerd/pacemaker.service                \
                 daemons/pacemakerd/pacemaker.upstart                \
                 daemons/pacemakerd/pacemaker.combined.upstart       \
                 daemons/schedulerd/Makefile                         \
                 devel/Makefile                                      \
                 doc/Doxyfile                                        \
                 doc/Makefile                                        \
                 doc/sphinx/Makefile                                 \
                 etc/Makefile                                        \
                 etc/init.d/pacemaker                                \
                 etc/logrotate.d/pacemaker                           \
                 extra/Makefile                                      \
                 extra/alerts/Makefile                               \
                 extra/resources/Makefile                            \
                 include/Makefile                                    \
                 include/crm/Makefile                                \
                 include/crm/cib/Makefile                            \
                 include/crm/common/Makefile                         \
                 include/crm/cluster/Makefile                        \
                 include/crm/fencing/Makefile                        \
                 include/crm/pengine/Makefile                        \
                 include/pcmki/Makefile                              \
                 replace/Makefile                                    \
                 lib/Makefile                                        \
                 lib/libpacemaker.pc                                 \
                 lib/pacemaker.pc                                    \
                 lib/pacemaker-cib.pc                                \
                 lib/pacemaker-lrmd.pc                               \
                 lib/pacemaker-service.pc                            \
                 lib/pacemaker-pe_rules.pc                           \
                 lib/pacemaker-pe_status.pc                          \
                 lib/pacemaker-fencing.pc                            \
                 lib/pacemaker-cluster.pc                            \
                 lib/common/Makefile                                 \
                 lib/common/tests/Makefile                           \
                 lib/common/tests/agents/Makefile                    \
                 lib/common/tests/cmdline/Makefile                   \
                 lib/common/tests/flags/Makefile                     \
+                lib/common/tests/io/Makefile                        \
                 lib/common/tests/iso8601/Makefile                   \
                 lib/common/tests/operations/Makefile                \
                 lib/common/tests/results/Makefile                   \
                 lib/common/tests/strings/Makefile                   \
                 lib/common/tests/utils/Makefile                     \
                 lib/common/tests/xpath/Makefile                     \
                 lib/cluster/Makefile                                \
                 lib/cib/Makefile                                    \
                 lib/gnu/Makefile                                    \
                 lib/pacemaker/Makefile                              \
                 lib/pengine/Makefile                                \
                 lib/pengine/tests/Makefile                          \
                 lib/pengine/tests/rules/Makefile                    \
                 lib/fencing/Makefile                                \
                 lib/lrmd/Makefile                                   \
                 lib/services/Makefile                               \
                 maint/Makefile                                      \
                 tests/Makefile                                      \
                 tools/Makefile                                      \
                 tools/report.collector                              \
                 tools/report.common                                 \
                 tools/crm_mon.service                               \
                 tools/crm_mon.upstart                               \
                 xml/Makefile                                        \
                 xml/pacemaker-schemas.pc                            \
 )
 
 dnl Now process the entire list of files added by previous
 dnl  calls to AC_CONFIG_FILES()
 AC_OUTPUT()
 
 dnl *****************
 dnl Configure summary
 dnl *****************
 
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([$PACKAGE configuration:])
 AC_MSG_NOTICE([  Version                  = ${VERSION} (Build: $BUILD_VERSION)])
 AC_MSG_NOTICE([  Features                 = ${PCMK_FEATURES}])
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([  Prefix                   = ${prefix}])
 AC_MSG_NOTICE([  Executables              = ${sbindir}])
 AC_MSG_NOTICE([  Man pages                = ${mandir}])
 AC_MSG_NOTICE([  Libraries                = ${libdir}])
 AC_MSG_NOTICE([  Header files             = ${includedir}])
 AC_MSG_NOTICE([  Arch-independent files   = ${datadir}])
 AC_MSG_NOTICE([  State information        = ${localstatedir}])
 AC_MSG_NOTICE([  System configuration     = ${sysconfdir}])
 AC_MSG_NOTICE([  OCF agents               = ${OCF_ROOT_DIR}])
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([  HA group name            = ${CRM_DAEMON_GROUP}])
 AC_MSG_NOTICE([  HA user name             = ${CRM_DAEMON_USER}])
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([  CFLAGS                   = ${CFLAGS}])
 AC_MSG_NOTICE([  CFLAGS_HARDENED_EXE      = ${CFLAGS_HARDENED_EXE}])
 AC_MSG_NOTICE([  CFLAGS_HARDENED_LIB      = ${CFLAGS_HARDENED_LIB}])
 AC_MSG_NOTICE([  LDFLAGS_HARDENED_EXE     = ${LDFLAGS_HARDENED_EXE}])
 AC_MSG_NOTICE([  LDFLAGS_HARDENED_LIB     = ${LDFLAGS_HARDENED_LIB}])
 AC_MSG_NOTICE([  Libraries                = ${LIBS}])
 AC_MSG_NOTICE([  Stack Libraries          = ${CLUSTERLIBS}])
 AC_MSG_NOTICE([  Unix socket auth method  = ${us_auth}])
diff --git a/daemons/controld/controld_remote_ra.c b/daemons/controld/controld_remote_ra.c
index aacbf8e66e..dc2c9441ab 100644
--- a/daemons/controld/controld_remote_ra.c
+++ b/daemons/controld/controld_remote_ra.c
@@ -1,1310 +1,1311 @@
 /*
  * Copyright 2013-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml_internal.h>
 #include <crm/lrmd.h>
 #include <crm/services.h>
 
 #include <pacemaker-controld.h>
 
 #define REMOTE_LRMD_RA "remote"
 
 /* The max start timeout before cmd retry */
 #define MAX_START_TIMEOUT_MS 10000
 
 typedef struct remote_ra_cmd_s {
     /*! the local node the cmd is issued from */
     char *owner;
     /*! the remote node the cmd is executed on */
     char *rsc_id;
     /*! the action to execute */
     char *action;
     /*! some string the client wants us to give it back */
     char *userdata;
     char *exit_reason;          // descriptive text on error
     /*! start delay in ms */
     int start_delay;
     /*! timer id used for start delay. */
     int delay_id;
     /*! timeout in ms for cmd */
     int timeout;
     int remaining_timeout;
     /*! recurring interval in ms */
     guint interval_ms;
     /*! interval timer id */
     int interval_id;
     int reported_success;
     int monitor_timeout_id;
     int takeover_timeout_id;
     /*! action parameters */
     lrmd_key_value_t *params;
     /*! executed rc */
     int rc;
     int op_status;
     int call_id;
     time_t start_time;
     gboolean cancel;
 } remote_ra_cmd_t;
 
 enum remote_migration_status {
     expect_takeover = 1,
     takeover_complete,
 };
 
 typedef struct remote_ra_data_s {
     crm_trigger_t *work;
     remote_ra_cmd_t *cur_cmd;
     GList *cmds;
     GList *recurring_cmds;
 
     enum remote_migration_status migrate_status;
 
     gboolean active;
 
     /* Maintenance mode is difficult to determine from the controller's context,
      * so we have it signalled back with the transition from the scheduler.
      */
     gboolean is_maintenance;
 
     /* Similar for whether we are controlling a guest node or remote node.
      * Fortunately there is a meta-attribute in the transition already and
      * as the situation doesn't change over time we can use the
      * resource start for noting down the information for later use when
      * the attributes aren't at hand.
      */
     gboolean controlling_guest;
 } remote_ra_data_t;
 
 static int handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms);
 static void handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd);
 static GList *fail_all_monitor_cmds(GList * list);
 
 static void
 free_cmd(gpointer user_data)
 {
     remote_ra_cmd_t *cmd = user_data;
 
     if (!cmd) {
         return;
     }
     if (cmd->delay_id) {
         g_source_remove(cmd->delay_id);
     }
     if (cmd->interval_id) {
         g_source_remove(cmd->interval_id);
     }
     if (cmd->monitor_timeout_id) {
         g_source_remove(cmd->monitor_timeout_id);
     }
     if (cmd->takeover_timeout_id) {
         g_source_remove(cmd->takeover_timeout_id);
     }
     free(cmd->owner);
     free(cmd->rsc_id);
     free(cmd->action);
     free(cmd->userdata);
     free(cmd->exit_reason);
     lrmd_key_value_freeall(cmd->params);
     free(cmd);
 }
 
 static int
 generate_callid(void)
 {
     static int remote_ra_callid = 0;
 
     remote_ra_callid++;
     if (remote_ra_callid <= 0) {
         remote_ra_callid = 1;
     }
 
     return remote_ra_callid;
 }
 
 static gboolean
 recurring_helper(gpointer data)
 {
     remote_ra_cmd_t *cmd = data;
     lrm_state_t *connection_rsc = NULL;
 
     cmd->interval_id = 0;
     connection_rsc = lrm_state_find(cmd->rsc_id);
     if (connection_rsc && connection_rsc->remote_ra_data) {
         remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
 
         ra_data->recurring_cmds = g_list_remove(ra_data->recurring_cmds, cmd);
 
         ra_data->cmds = g_list_append(ra_data->cmds, cmd);
         mainloop_set_trigger(ra_data->work);
     }
     return FALSE;
 }
 
 static gboolean
 start_delay_helper(gpointer data)
 {
     remote_ra_cmd_t *cmd = data;
     lrm_state_t *connection_rsc = NULL;
 
     cmd->delay_id = 0;
     connection_rsc = lrm_state_find(cmd->rsc_id);
     if (connection_rsc && connection_rsc->remote_ra_data) {
         remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
 
         mainloop_set_trigger(ra_data->work);
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Handle cluster communication related to pacemaker_remote node joining
  *
  * \param[in] node_name  Name of newly integrated pacemaker_remote node
  */
 static void
 remote_node_up(const char *node_name)
 {
     int call_opt, call_id = 0;
     xmlNode *update, *state;
     crm_node_t *node;
     enum controld_section_e section = controld_section_all;
 
     CRM_CHECK(node_name != NULL, return);
     crm_info("Announcing Pacemaker Remote node %s", node_name);
 
     /* Clear node's entire state (resource history and transient attributes)
      * other than shutdown locks. The transient attributes should and normally
      * will be cleared when the node leaves, but since remote node state has a
      * number of corner cases, clear them here as well, to be sure.
      */
     call_opt = crmd_cib_smart_opt();
     if (controld_shutdown_lock_enabled) {
         section = controld_section_all_unlocked;
     }
     /* Purge node from attrd's memory */
     update_attrd_remote_node_removed(node_name, NULL);
 
     controld_delete_node_state(node_name, section, call_opt);
 
     /* Clear node's probed attribute */
     update_attrd(node_name, CRM_OP_PROBED, NULL, NULL, TRUE);
 
     /* Ensure node is in the remote peer cache with member status */
     node = crm_remote_peer_get(node_name);
     CRM_CHECK(node != NULL, return);
     pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);
 
     /* pacemaker_remote nodes don't participate in the membership layer,
      * so cluster nodes don't automatically get notified when they come and go.
      * We send a cluster message to the DC, and update the CIB node state entry,
      * so the DC will get it sooner (via message) or later (via CIB refresh),
      * and any other interested parties can query the CIB.
      */
     send_remote_state_message(node_name, TRUE);
 
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     state = create_node_state_update(node, node_update_cluster, update,
                                      __func__);
 
     /* Clear the XML_NODE_IS_FENCED flag in the node state. If the node ever
      * needs to be fenced, this flag will allow various actions to determine
      * whether the fencing has happened yet.
      */
     crm_xml_add(state, XML_NODE_IS_FENCED, "0");
 
     /* TODO: If the remote connection drops, and this (async) CIB update either
      * failed or has not yet completed, later actions could mistakenly think the
      * node has already been fenced (if the XML_NODE_IS_FENCED attribute was
      * previously set, because it won't have been cleared). This could prevent
      * actual fencing or allow recurring monitor failures to be cleared too
      * soon. Ideally, we wouldn't rely on the CIB for the fenced status.
      */
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id < 0) {
         crm_perror(LOG_WARNING, "%s CIB node state setup", node_name);
     }
     free_xml(update);
 }
 
 enum down_opts {
     DOWN_KEEP_LRM,
     DOWN_ERASE_LRM
 };
 
 /*!
  * \internal
  * \brief Handle cluster communication related to pacemaker_remote node leaving
  *
  * \param[in] node_name  Name of lost node
  * \param[in] opts       Whether to keep or erase LRM history
  */
 static void
 remote_node_down(const char *node_name, const enum down_opts opts)
 {
     xmlNode *update;
     int call_id = 0;
     int call_opt = crmd_cib_smart_opt();
     crm_node_t *node;
 
     /* Purge node from attrd's memory */
     update_attrd_remote_node_removed(node_name, NULL);
 
     /* Normally, only node attributes should be erased, and the resource history
      * should be kept until the node comes back up. However, after a successful
      * fence, we want to clear the history as well, so we don't think resources
      * are still running on the node.
      */
     if (opts == DOWN_ERASE_LRM) {
         controld_delete_node_state(node_name, controld_section_all, call_opt);
     } else {
         controld_delete_node_state(node_name, controld_section_attrs, call_opt);
     }
 
     /* Ensure node is in the remote peer cache with lost state */
     node = crm_remote_peer_get(node_name);
     CRM_CHECK(node != NULL, return);
     pcmk__update_peer_state(__func__, node, CRM_NODE_LOST, 0);
 
     /* Notify DC */
     send_remote_state_message(node_name, FALSE);
 
     /* Update CIB node state */
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     create_node_state_update(node, node_update_cluster, update, __func__);
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id < 0) {
         crm_perror(LOG_ERR, "%s CIB node state update", node_name);
     }
     free_xml(update);
 }
 
 /*!
  * \internal
  * \brief Handle effects of a remote RA command on node state
  *
  * \param[in] cmd  Completed remote RA command
  */
 static void
 check_remote_node_state(remote_ra_cmd_t *cmd)
 {
     /* Only successful actions can change node state */
     if (cmd->rc != PCMK_OCF_OK) {
         return;
     }
 
     if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
         remote_node_up(cmd->rsc_id);
 
     } else if (pcmk__str_eq(cmd->action, "migrate_from", pcmk__str_casei)) {
         /* After a successful migration, we don't need to do remote_node_up()
          * because the DC already knows the node is up, and we don't want to
          * clear LRM history etc. We do need to add the remote node to this
          * host's remote peer cache, because (unless it happens to be DC)
          * it hasn't been tracking the remote node, and other code relies on
          * the cache to distinguish remote nodes from unseen cluster nodes.
          */
         crm_node_t *node = crm_remote_peer_get(cmd->rsc_id);
 
         CRM_CHECK(node != NULL, return);
         pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);
 
     } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
         lrm_state_t *lrm_state = lrm_state_find(cmd->rsc_id);
         remote_ra_data_t *ra_data = lrm_state? lrm_state->remote_ra_data : NULL;
 
         if (ra_data) {
             if (ra_data->migrate_status != takeover_complete) {
                 /* Stop means down if we didn't successfully migrate elsewhere */
                 remote_node_down(cmd->rsc_id, DOWN_KEEP_LRM);
             } else if (AM_I_DC == FALSE) {
                 /* Only the connection host and DC track node state,
                  * so if the connection migrated elsewhere and we aren't DC,
                  * un-cache the node, so we don't have stale info
                  */
                 crm_remote_peer_cache_remove(cmd->rsc_id);
             }
         }
     }
 
     /* We don't do anything for successful monitors, which is correct for
      * routine recurring monitors, and for monitors on nodes where the
      * connection isn't supposed to be (the cluster will stop the connection in
      * that case). However, if the initial probe finds the connection already
      * active on the node where we want it, we probably should do
      * remote_node_up(). Unfortunately, we can't distinguish that case here.
      * Given that connections have to be initiated by the cluster, the chance of
      * that should be close to zero.
      */
 }
 
 static void
 report_remote_ra_result(remote_ra_cmd_t * cmd)
 {
     lrmd_event_data_t op = { 0, };
 
     check_remote_node_state(cmd);
 
     op.type = lrmd_event_exec_complete;
     op.rsc_id = cmd->rsc_id;
     op.op_type = cmd->action;
     op.user_data = cmd->userdata;
     op.exit_reason = cmd->exit_reason;
     op.timeout = cmd->timeout;
     op.interval_ms = cmd->interval_ms;
     op.rc = cmd->rc;
     op.op_status = cmd->op_status;
     op.t_run = (unsigned int) cmd->start_time;
     op.t_rcchange = (unsigned int) cmd->start_time;
     if (cmd->reported_success && cmd->rc != PCMK_OCF_OK) {
         op.t_rcchange = (unsigned int) time(NULL);
         /* This edge case will likely never ever occur, but if it does the
          * result is that a failure will not be processed correctly. This is only
          * remotely possible because we are able to detect a connection resource's tcp
          * connection has failed at any moment after start has completed. The actual
          * recurring operation is just a connectivity ping.
          *
          * basically, we are not guaranteed that the first successful monitor op and
          * a subsequent failed monitor op will not occur in the same timestamp. We have to
          * make it look like the operations occurred at separate times though. */
         if (op.t_rcchange == op.t_run) {
             op.t_rcchange++;
         }
     }
 
     if (cmd->params) {
         lrmd_key_value_t *tmp;
 
         op.params = pcmk__strkey_table(free, free);
         for (tmp = cmd->params; tmp; tmp = tmp->next) {
             g_hash_table_insert(op.params, strdup(tmp->key), strdup(tmp->value));
         }
 
     }
     op.call_id = cmd->call_id;
     op.remote_nodename = cmd->owner;
 
     lrm_op_callback(&op);
 
     if (op.params) {
         g_hash_table_destroy(op.params);
     }
 }
 
 static void
 update_remaining_timeout(remote_ra_cmd_t * cmd)
 {
     cmd->remaining_timeout = ((cmd->timeout / 1000) - (time(NULL) - cmd->start_time)) * 1000;
 }
 
 static gboolean
 retry_start_cmd_cb(gpointer data)
 {
     lrm_state_t *lrm_state = data;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     remote_ra_cmd_t *cmd = NULL;
     int rc = pcmk_rc_error;
 
     if (!ra_data || !ra_data->cur_cmd) {
         return FALSE;
     }
     cmd = ra_data->cur_cmd;
     if (!pcmk__strcase_any_of(cmd->action, "start", "migrate_from", NULL)) {
         return FALSE;
     }
     update_remaining_timeout(cmd);
 
     if (cmd->remaining_timeout > 0) {
         rc = handle_remote_ra_start(lrm_state, cmd, cmd->remaining_timeout);
     }
 
     if (rc != pcmk_rc_ok) {
         report_remote_ra_result(cmd);
 
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
         ra_data->cur_cmd = NULL;
         free_cmd(cmd);
     } else {
         /* wait for connection event */
     }
 
     return FALSE;
 }
 
 
 static gboolean
 connection_takeover_timeout_cb(gpointer data)
 {
     lrm_state_t *lrm_state = NULL;
     remote_ra_cmd_t *cmd = data;
 
     crm_info("takeover event timed out for node %s", cmd->rsc_id);
     cmd->takeover_timeout_id = 0;
 
     lrm_state = lrm_state_find(cmd->rsc_id);
 
     handle_remote_ra_stop(lrm_state, cmd);
     free_cmd(cmd);
 
     return FALSE;
 }
 
 static gboolean
 monitor_timeout_cb(gpointer data)
 {
     lrm_state_t *lrm_state = NULL;
     remote_ra_cmd_t *cmd = data;
 
     lrm_state = lrm_state_find(cmd->rsc_id);
 
     crm_info("Timed out waiting for remote poke response from %s%s",
              cmd->rsc_id, (lrm_state? "" : " (no LRM state)"));
     cmd->monitor_timeout_id = 0;
     cmd->op_status = PCMK_EXEC_TIMEOUT;
     cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
 
     if (lrm_state && lrm_state->remote_ra_data) {
         remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
         if (ra_data->cur_cmd == cmd) {
             ra_data->cur_cmd = NULL;
         }
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
     }
 
     report_remote_ra_result(cmd);
     free_cmd(cmd);
 
     if(lrm_state) {
         lrm_state_disconnect(lrm_state);
     }
     return FALSE;
 }
 
 static void
 synthesize_lrmd_success(lrm_state_t *lrm_state, const char *rsc_id, const char *op_type)
 {
     lrmd_event_data_t op = { 0, };
 
     if (lrm_state == NULL) {
         /* if lrm_state not given assume local */
         lrm_state = lrm_state_find(fsa_our_uname);
     }
     CRM_ASSERT(lrm_state != NULL);
 
     op.type = lrmd_event_exec_complete;
     op.rsc_id = rsc_id;
     op.op_type = op_type;
     op.rc = PCMK_OCF_OK;
     op.op_status = PCMK_EXEC_DONE;
     op.t_run = (unsigned int) time(NULL);
     op.t_rcchange = op.t_run;
     op.call_id = generate_callid();
     process_lrm_event(lrm_state, &op, NULL, NULL);
 }
 
 void
 remote_lrm_op_callback(lrmd_event_data_t * op)
 {
     gboolean cmd_handled = FALSE;
     lrm_state_t *lrm_state = NULL;
     remote_ra_data_t *ra_data = NULL;
     remote_ra_cmd_t *cmd = NULL;
 
     crm_debug("Processing '%s%s%s' event on remote connection to %s: %s "
               "(%d) status=%s (%d)",
               (op->op_type? op->op_type : ""), (op->op_type? " " : ""),
               lrmd_event_type2str(op->type), op->remote_nodename,
               services_ocf_exitcode_str(op->rc), op->rc,
               pcmk_exec_status_str(op->op_status), op->op_status);
 
     lrm_state = lrm_state_find(op->remote_nodename);
     if (!lrm_state || !lrm_state->remote_ra_data) {
         crm_debug("No state information found for remote connection event");
         return;
     }
     ra_data = lrm_state->remote_ra_data;
 
     if (op->type == lrmd_event_new_client) {
         // Another client has connected to the remote daemon
 
         if (ra_data->migrate_status == expect_takeover) {
             // Great, we knew this was coming
             ra_data->migrate_status = takeover_complete;
 
         } else {
             crm_err("Disconnecting from Pacemaker Remote node %s due to "
                     "unexpected client takeover", op->remote_nodename);
             /* In this case, lrmd_tls_connection_destroy() will be called under the control of mainloop. */
             /* Do not free lrm_state->conn yet. */
             /* It'll be freed in the following stop action. */
             lrm_state_disconnect_only(lrm_state);
         }
         return;
     }
 
     /* filter all EXEC events up */
     if (op->type == lrmd_event_exec_complete) {
         if (ra_data->migrate_status == takeover_complete) {
             crm_debug("ignoring event, this connection is taken over by another node");
         } else {
             lrm_op_callback(op);
         }
         return;
     }
 
     if ((op->type == lrmd_event_disconnect) && (ra_data->cur_cmd == NULL)) {
 
         if (ra_data->active == FALSE) {
             crm_debug("Disconnection from Pacemaker Remote node %s complete",
                       lrm_state->node_name);
 
         } else if (!remote_ra_is_in_maintenance(lrm_state)) {
             crm_err("Lost connection to Pacemaker Remote node %s",
                     lrm_state->node_name);
             ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
             ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);
 
         } else {
             crm_notice("Unmanaged Pacemaker Remote node %s disconnected",
                        lrm_state->node_name);
             /* Do roughly what a 'stop' on the remote-resource would do */
             handle_remote_ra_stop(lrm_state, NULL);
             remote_node_down(lrm_state->node_name, DOWN_KEEP_LRM);
             /* now fake the reply of a successful 'stop' */
             synthesize_lrmd_success(NULL, lrm_state->node_name, "stop");
         }
         return;
     }
 
     if (!ra_data->cur_cmd) {
         crm_debug("no event to match");
         return;
     }
 
     cmd = ra_data->cur_cmd;
 
     /* Start actions and migrate from actions complete after connection
      * comes back to us. */
     if (op->type == lrmd_event_connect && pcmk__strcase_any_of(cmd->action, "start",
                                                                "migrate_from", NULL)) {
         if (op->connection_rc < 0) {
             update_remaining_timeout(cmd);
 
             if (op->connection_rc == -ENOKEY) {
                 // Hard error, don't retry
                 cmd->op_status = PCMK_EXEC_ERROR;
                 cmd->rc = PCMK_OCF_INVALID_PARAM;
                 cmd->exit_reason = strdup("Authentication key not readable");
 
             } else if (cmd->remaining_timeout > 3000) {
                 crm_trace("rescheduling start, remaining timeout %d", cmd->remaining_timeout);
                 g_timeout_add(1000, retry_start_cmd_cb, lrm_state);
                 return;
 
             } else {
                 crm_trace("can't reschedule start, remaining timeout too small %d",
                           cmd->remaining_timeout);
                 cmd->op_status = PCMK_EXEC_TIMEOUT;
                 cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
             }
 
         } else {
             lrm_state_reset_tables(lrm_state, TRUE);
             cmd->rc = PCMK_OCF_OK;
             cmd->op_status = PCMK_EXEC_DONE;
             ra_data->active = TRUE;
         }
 
         crm_debug("Remote connection event matched %s action", cmd->action);
         report_remote_ra_result(cmd);
         cmd_handled = TRUE;
 
     } else if (op->type == lrmd_event_poke && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
 
         if (cmd->monitor_timeout_id) {
             g_source_remove(cmd->monitor_timeout_id);
             cmd->monitor_timeout_id = 0;
         }
 
         /* Only report success the first time, after that only worry about failures.
          * For this function, if we get the poke pack, it is always a success. Pokes
          * only fail if the send fails, or the response times out. */
         if (!cmd->reported_success) {
             cmd->rc = PCMK_OCF_OK;
             cmd->op_status = PCMK_EXEC_DONE;
             report_remote_ra_result(cmd);
             cmd->reported_success = 1;
         }
 
         crm_debug("Remote poke event matched %s action", cmd->action);
 
         /* success, keep rescheduling if interval is present. */
         if (cmd->interval_ms && (cmd->cancel == FALSE)) {
             ra_data->recurring_cmds = g_list_append(ra_data->recurring_cmds, cmd);
             cmd->interval_id = g_timeout_add(cmd->interval_ms,
                                              recurring_helper, cmd);
             cmd = NULL;         /* prevent free */
         }
         cmd_handled = TRUE;
 
     } else if (op->type == lrmd_event_disconnect && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
         if (ra_data->active == TRUE && (cmd->cancel == FALSE)) {
             cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
             cmd->op_status = PCMK_EXEC_ERROR;
             report_remote_ra_result(cmd);
             crm_err("Remote connection to %s unexpectedly dropped during monitor",
                     lrm_state->node_name);
         }
         cmd_handled = TRUE;
 
     } else if (op->type == lrmd_event_new_client && pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
 
         handle_remote_ra_stop(lrm_state, cmd);
         cmd_handled = TRUE;
 
     } else {
         crm_debug("Event did not match %s action", ra_data->cur_cmd->action);
     }
 
     if (cmd_handled) {
         ra_data->cur_cmd = NULL;
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
         free_cmd(cmd);
     }
 }
 
 static void
 handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd)
 {
     remote_ra_data_t *ra_data = NULL;
 
     CRM_ASSERT(lrm_state);
     ra_data = lrm_state->remote_ra_data;
 
     if (ra_data->migrate_status != takeover_complete) {
         /* delete pending ops when ever the remote connection is intentionally stopped */
         g_hash_table_remove_all(lrm_state->pending_ops);
     } else {
         /* we no longer hold the history if this connection has been migrated,
          * however, we keep metadata cache for future use */
         lrm_state_reset_tables(lrm_state, FALSE);
     }
 
     ra_data->active = FALSE;
     lrm_state_disconnect(lrm_state);
 
     if (ra_data->cmds) {
         g_list_free_full(ra_data->cmds, free_cmd);
     }
     if (ra_data->recurring_cmds) {
         g_list_free_full(ra_data->recurring_cmds, free_cmd);
     }
     ra_data->cmds = NULL;
     ra_data->recurring_cmds = NULL;
     ra_data->cur_cmd = NULL;
 
     if (cmd) {
         cmd->rc = PCMK_OCF_OK;
         cmd->op_status = PCMK_EXEC_DONE;
         report_remote_ra_result(cmd);
     }
 }
 
 // \return Standard Pacemaker return code
 static int
 handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms)
 {
     const char *server = NULL;
     lrmd_key_value_t *tmp = NULL;
     int port = 0;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     int timeout_used = timeout_ms > MAX_START_TIMEOUT_MS ? MAX_START_TIMEOUT_MS : timeout_ms;
     int rc = pcmk_rc_ok;
 
     for (tmp = cmd->params; tmp; tmp = tmp->next) {
         if (pcmk__strcase_any_of(tmp->key, XML_RSC_ATTR_REMOTE_RA_ADDR,
                                  XML_RSC_ATTR_REMOTE_RA_SERVER, NULL)) {
             server = tmp->value;
         } else if (pcmk__str_eq(tmp->key, XML_RSC_ATTR_REMOTE_RA_PORT, pcmk__str_casei)) {
             port = atoi(tmp->value);
         } else if (pcmk__str_eq(tmp->key, CRM_META "_" XML_RSC_ATTR_CONTAINER, pcmk__str_casei)) {
             ra_data->controlling_guest = TRUE;
         }
     }
 
     rc = controld_connect_remote_executor(lrm_state, server, port,
                                           timeout_used);
     if (rc != pcmk_rc_ok) {
         cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
         cmd->op_status = PCMK_EXEC_ERROR;
+        cmd->exit_reason = strdup(pcmk_rc_str(rc));
     }
     return rc;
 }
 
 static gboolean
 handle_remote_ra_exec(gpointer user_data)
 {
     int rc = 0;
     lrm_state_t *lrm_state = user_data;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     remote_ra_cmd_t *cmd;
     GList *first = NULL;
 
     if (ra_data->cur_cmd) {
         /* still waiting on previous cmd */
         return TRUE;
     }
 
     while (ra_data->cmds) {
         first = ra_data->cmds;
         cmd = first->data;
         if (cmd->delay_id) {
             /* still waiting for start delay timer to trip */
             return TRUE;
         }
 
         ra_data->cmds = g_list_remove_link(ra_data->cmds, first);
         g_list_free_1(first);
 
         if (!strcmp(cmd->action, "start") || !strcmp(cmd->action, "migrate_from")) {
             ra_data->migrate_status = 0;
             if (handle_remote_ra_start(lrm_state, cmd,
                                        cmd->timeout) == pcmk_rc_ok) {
                 /* take care of this later when we get async connection result */
                 crm_debug("Initiated async remote connection, %s action will complete after connect event",
                           cmd->action);
                 ra_data->cur_cmd = cmd;
                 return TRUE;
             }
             report_remote_ra_result(cmd);
 
         } else if (!strcmp(cmd->action, "monitor")) {
 
             if (lrm_state_is_connected(lrm_state) == TRUE) {
                 rc = lrm_state_poke_connection(lrm_state);
                 if (rc < 0) {
                     cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
                     cmd->op_status = PCMK_EXEC_ERROR;
                 }
             } else {
                 rc = -1;
                 cmd->op_status = PCMK_EXEC_DONE;
                 cmd->rc = PCMK_OCF_NOT_RUNNING;
             }
 
             if (rc == 0) {
                 crm_debug("Poked Pacemaker Remote at node %s, waiting for async response",
                           cmd->rsc_id);
                 ra_data->cur_cmd = cmd;
                 cmd->monitor_timeout_id = g_timeout_add(cmd->timeout, monitor_timeout_cb, cmd);
                 return TRUE;
             }
             report_remote_ra_result(cmd);
 
         } else if (!strcmp(cmd->action, "stop")) {
 
             if (ra_data->migrate_status == expect_takeover) {
                 /* briefly wait on stop for the takeover event to occur. If the
                  * takeover event does not occur during the wait period, that's fine.
                  * It just means that the remote-node's lrm_status section is going to get
                  * cleared which will require all the resources running in the remote-node
                  * to be explicitly re-detected via probe actions.  If the takeover does occur
                  * successfully, then we can leave the status section intact. */
                 cmd->takeover_timeout_id = g_timeout_add((cmd->timeout/2), connection_takeover_timeout_cb, cmd);
                 ra_data->cur_cmd = cmd;
                 return TRUE;
             }
 
             handle_remote_ra_stop(lrm_state, cmd);
 
         } else if (!strcmp(cmd->action, "migrate_to")) {
             ra_data->migrate_status = expect_takeover;
             cmd->rc = PCMK_OCF_OK;
             cmd->op_status = PCMK_EXEC_DONE;
             report_remote_ra_result(cmd);
         } else if (pcmk__str_any_of(cmd->action, CRMD_ACTION_RELOAD,
                                     CRMD_ACTION_RELOAD_AGENT, NULL))  {
             /* Currently the only reloadable parameter is reconnect_interval,
              * which is only used by the scheduler via the CIB, so reloads are a
              * no-op.
              *
              * @COMPAT DC <2.1.0: We only need to check for "reload" in case
              * we're in a rolling upgrade with a DC scheduling "reload" instead
              * of "reload-agent". An OCF 1.1 "reload" would be a no-op anyway,
              * so this would work for that purpose as well.
              */
             cmd->rc = PCMK_OCF_OK;
             cmd->op_status = PCMK_EXEC_DONE;
             report_remote_ra_result(cmd);
         }
 
         free_cmd(cmd);
     }
 
     return TRUE;
 }
 
 static void
 remote_ra_data_init(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = NULL;
 
     if (lrm_state->remote_ra_data) {
         return;
     }
 
     ra_data = calloc(1, sizeof(remote_ra_data_t));
     ra_data->work = mainloop_add_trigger(G_PRIORITY_HIGH, handle_remote_ra_exec, lrm_state);
     lrm_state->remote_ra_data = ra_data;
 }
 
 void
 remote_ra_cleanup(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
     if (!ra_data) {
         return;
     }
 
     if (ra_data->cmds) {
         g_list_free_full(ra_data->cmds, free_cmd);
     }
 
     if (ra_data->recurring_cmds) {
         g_list_free_full(ra_data->recurring_cmds, free_cmd);
     }
     mainloop_destroy_trigger(ra_data->work);
     free(ra_data);
     lrm_state->remote_ra_data = NULL;
 }
 
 gboolean
 is_remote_lrmd_ra(const char *agent, const char *provider, const char *id)
 {
     if (agent && provider && !strcmp(agent, REMOTE_LRMD_RA) && !strcmp(provider, "pacemaker")) {
         return TRUE;
     }
     if (id && lrm_state_find(id) && !pcmk__str_eq(id, fsa_our_uname, pcmk__str_casei)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 lrmd_rsc_info_t *
 remote_ra_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id)
 {
     lrmd_rsc_info_t *info = NULL;
 
     if ((lrm_state_find(rsc_id))) {
         info = calloc(1, sizeof(lrmd_rsc_info_t));
 
         info->id = strdup(rsc_id);
         info->type = strdup(REMOTE_LRMD_RA);
         info->standard = strdup(PCMK_RESOURCE_CLASS_OCF);
         info->provider = strdup("pacemaker");
     }
 
     return info;
 }
 
 static gboolean
 is_remote_ra_supported_action(const char *action)
 {
     return pcmk__str_any_of(action,
                             CRMD_ACTION_START,
                             CRMD_ACTION_STOP,
                             CRMD_ACTION_STATUS,
                             CRMD_ACTION_MIGRATE,
                             CRMD_ACTION_MIGRATED,
                             CRMD_ACTION_RELOAD_AGENT,
                             CRMD_ACTION_RELOAD,
                             NULL);
 }
 
 static GList *
 fail_all_monitor_cmds(GList * list)
 {
     GList *rm_list = NULL;
     remote_ra_cmd_t *cmd = NULL;
     GList *gIter = NULL;
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms > 0) && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
             rm_list = g_list_append(rm_list, cmd);
         }
     }
 
     for (gIter = rm_list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
 
         cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
         cmd->op_status = PCMK_EXEC_ERROR;
         crm_trace("Pre-emptively failing %s %s (interval=%u, %s)",
                   cmd->action, cmd->rsc_id, cmd->interval_ms, cmd->userdata);
         report_remote_ra_result(cmd);
 
         list = g_list_remove(list, cmd);
         free_cmd(cmd);
     }
 
     /* frees only the list data, not the cmds */
     g_list_free(rm_list);
     return list;
 }
 
 static GList *
 remove_cmd(GList * list, const char *action, guint interval_ms)
 {
     remote_ra_cmd_t *cmd = NULL;
     GList *gIter = NULL;
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms == interval_ms)
             && pcmk__str_eq(cmd->action, action, pcmk__str_casei)) {
             break;
         }
         cmd = NULL;
     }
     if (cmd) {
         list = g_list_remove(list, cmd);
         free_cmd(cmd);
     }
     return list;
 }
 
 int
 remote_ra_cancel(lrm_state_t *lrm_state, const char *rsc_id,
                  const char *action, guint interval_ms)
 {
     lrm_state_t *connection_rsc = NULL;
     remote_ra_data_t *ra_data = NULL;
 
     connection_rsc = lrm_state_find(rsc_id);
     if (!connection_rsc || !connection_rsc->remote_ra_data) {
         return -EINVAL;
     }
 
     ra_data = connection_rsc->remote_ra_data;
     ra_data->cmds = remove_cmd(ra_data->cmds, action, interval_ms);
     ra_data->recurring_cmds = remove_cmd(ra_data->recurring_cmds, action,
                                          interval_ms);
     if (ra_data->cur_cmd &&
         (ra_data->cur_cmd->interval_ms == interval_ms) &&
         (pcmk__str_eq(ra_data->cur_cmd->action, action, pcmk__str_casei))) {
 
         ra_data->cur_cmd->cancel = TRUE;
     }
 
     return 0;
 }
 
 static remote_ra_cmd_t *
 handle_dup_monitor(remote_ra_data_t *ra_data, guint interval_ms,
                    const char *userdata)
 {
     GList *gIter = NULL;
     remote_ra_cmd_t *cmd = NULL;
 
     /* there are 3 places a potential duplicate monitor operation
      * could exist.
      * 1. recurring_cmds list. where the op is waiting for its next interval
      * 2. cmds list, where the op is queued to get executed immediately
      * 3. cur_cmd, which means the monitor op is in flight right now.
      */
     if (interval_ms == 0) {
         return NULL;
     }
 
     if (ra_data->cur_cmd &&
         ra_data->cur_cmd->cancel == FALSE &&
         (ra_data->cur_cmd->interval_ms == interval_ms) &&
         pcmk__str_eq(ra_data->cur_cmd->action, "monitor", pcmk__str_casei)) {
 
         cmd = ra_data->cur_cmd;
         goto handle_dup;
     }
 
     for (gIter = ra_data->recurring_cmds; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms == interval_ms)
             && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
             goto handle_dup;
         }
     }
 
     for (gIter = ra_data->cmds; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms == interval_ms)
             && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
             goto handle_dup;
         }
     }
 
     return NULL;
 
 handle_dup:
 
     crm_trace("merging duplicate monitor cmd " PCMK__OP_FMT,
               cmd->rsc_id, "monitor", interval_ms);
 
     /* update the userdata */
     if (userdata) {
        free(cmd->userdata);
        cmd->userdata = strdup(userdata);
     }
 
     /* if we've already reported success, generate a new call id */
     if (cmd->reported_success) {
         cmd->start_time = time(NULL);
         cmd->call_id = generate_callid();
         cmd->reported_success = 0;
     }
 
     /* if we have an interval_id set, that means we are in the process of
      * waiting for this cmd's next interval. instead of waiting, cancel
      * the timer and execute the action immediately */
     if (cmd->interval_id) {
         g_source_remove(cmd->interval_id);
         cmd->interval_id = 0;
         recurring_helper(cmd);
     }
 
     return cmd;
 }
 
 int
 remote_ra_exec(lrm_state_t *lrm_state, const char *rsc_id, const char *action,
                const char *userdata, guint interval_ms,
                int timeout,     /* ms */
                int start_delay, /* ms */
                lrmd_key_value_t * params)
 {
     int rc = 0;
     lrm_state_t *connection_rsc = NULL;
     remote_ra_cmd_t *cmd = NULL;
     remote_ra_data_t *ra_data = NULL;
 
     if (is_remote_ra_supported_action(action) == FALSE) {
         rc = -EINVAL;
         goto exec_done;
     }
 
     connection_rsc = lrm_state_find(rsc_id);
     if (!connection_rsc) {
         rc = -EINVAL;
         goto exec_done;
     }
 
     remote_ra_data_init(connection_rsc);
     ra_data = connection_rsc->remote_ra_data;
 
     cmd = handle_dup_monitor(ra_data, interval_ms, userdata);
     if (cmd) {
         rc = cmd->call_id;
         goto exec_done;
     }
 
     cmd = calloc(1, sizeof(remote_ra_cmd_t));
     cmd->owner = strdup(lrm_state->node_name);
     cmd->rsc_id = strdup(rsc_id);
     cmd->action = strdup(action);
     cmd->userdata = strdup(userdata);
     cmd->interval_ms = interval_ms;
     cmd->timeout = timeout;
     cmd->start_delay = start_delay;
     cmd->params = params;
     cmd->start_time = time(NULL);
 
     cmd->call_id = generate_callid();
 
     if (cmd->start_delay) {
         cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
     }
 
     ra_data->cmds = g_list_append(ra_data->cmds, cmd);
     mainloop_set_trigger(ra_data->work);
 
     return cmd->call_id;
   exec_done:
 
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Immediately fail all monitors of a remote node, if proxied here
  *
  * \param[in] node_name  Name of pacemaker_remote node
  */
 void
 remote_ra_fail(const char *node_name)
 {
     lrm_state_t *lrm_state = lrm_state_find(node_name);
 
     if (lrm_state && lrm_state_is_connected(lrm_state)) {
         remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
         crm_info("Failing monitors on Pacemaker Remote node %s", node_name);
         ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
         ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);
     }
 }
 
 /* A guest node fencing implied by host fencing looks like:
  *
  *  <pseudo_event id="103" operation="stonith" operation_key="stonith-lxc1-off"
  *                on_node="lxc1" on_node_uuid="lxc1">
  *     <attributes CRM_meta_on_node="lxc1" CRM_meta_on_node_uuid="lxc1"
  *                 CRM_meta_stonith_action="off" crm_feature_set="3.0.12"/>
  *     <downed>
  *       <node id="lxc1"/>
  *     </downed>
  *  </pseudo_event>
  */
 #define XPATH_PSEUDO_FENCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \
     "[@" XML_LRM_ATTR_TASK "='stonith']/" XML_GRAPH_TAG_DOWNED \
     "/" XML_CIB_TAG_NODE
 
 /*!
  * \internal
  * \brief Check a pseudo-action for Pacemaker Remote node side effects
  *
  * \param[in] xml  XML of pseudo-action to check
  */
 void
 remote_ra_process_pseudo(xmlNode *xml)
 {
     xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_FENCE);
 
     if (numXpathResults(search) == 1) {
         xmlNode *result = getXpathResult(search, 0);
 
         /* Normally, we handle the necessary side effects of a guest node stop
          * action when reporting the remote agent's result. However, if the stop
          * is implied due to fencing, it will be a fencing pseudo-event, and
          * there won't be a result to report. Handle that case here.
          *
          * This will result in a duplicate call to remote_node_down() if the
          * guest stop was real instead of implied, but that shouldn't hurt.
          *
          * There is still one corner case that isn't handled: if a guest node
          * isn't running any resources when its host is fenced, it will appear
          * to be cleanly stopped, so there will be no pseudo-fence, and our
          * peer cache state will be incorrect unless and until the guest is
          * recovered.
          */
         if (result) {
             const char *remote = ID(result);
 
             if (remote) {
                 remote_node_down(remote, DOWN_ERASE_LRM);
             }
         }
     }
     freeXpathObject(search);
 }
 
 static void
 remote_ra_maintenance(lrm_state_t * lrm_state, gboolean maintenance)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     xmlNode *update, *state;
     int call_opt, call_id = 0;
     crm_node_t *node;
 
     call_opt = crmd_cib_smart_opt();
     node = crm_remote_peer_get(lrm_state->node_name);
     CRM_CHECK(node != NULL, return);
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     state = create_node_state_update(node, node_update_none, update,
                                      __func__);
     crm_xml_add(state, XML_NODE_IS_MAINTENANCE, maintenance?"1":"0");
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id < 0) {
         crm_perror(LOG_WARNING, "%s CIB node state update failed", lrm_state->node_name);
     } else {
         /* TODO: still not 100% sure that async update will succeed ... */
         ra_data->is_maintenance = maintenance;
     }
     free_xml(update);
 }
 
 #define XPATH_PSEUDO_MAINTENANCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \
     "[@" XML_LRM_ATTR_TASK "='" CRM_OP_MAINTENANCE_NODES "']/" \
     XML_GRAPH_TAG_MAINTENANCE
 
 /*!
  * \internal
  * \brief Check a pseudo-action holding updates for maintenance state
  *
  * \param[in] xml  XML of pseudo-action to check
  */
 
 void
 remote_ra_process_maintenance_nodes(xmlNode *xml)
 {
     xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_MAINTENANCE);
 
     if (numXpathResults(search) == 1) {
         xmlNode *node;
         int cnt = 0, cnt_remote = 0;
 
         for (node =
                 first_named_child(getXpathResult(search, 0), XML_CIB_TAG_NODE);
             node != NULL; node = pcmk__xml_next(node)) {
             lrm_state_t *lrm_state = lrm_state_find(ID(node));
 
             cnt++;
             if (lrm_state && lrm_state->remote_ra_data &&
                 ((remote_ra_data_t *) lrm_state->remote_ra_data)->active) {
                 int is_maint;
 
                 cnt_remote++;
                 pcmk__scan_min_int(crm_element_value(node, XML_NODE_IS_MAINTENANCE),
                                    &is_maint, 0);
                 remote_ra_maintenance(lrm_state, is_maint);
             }
         }
         crm_trace("Action holds %d nodes (%d remotes found) "
                     "adjusting maintenance-mode", cnt, cnt_remote);
     }
     freeXpathObject(search);
 }
 
 gboolean
 remote_ra_is_in_maintenance(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
     return ra_data->is_maintenance;
 }
 
 gboolean
 remote_ra_controlling_guest(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
     return ra_data->controlling_guest;
 }
diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h
index d50d9547ec..19cff43aa4 100644
--- a/include/crm/common/internal.h
+++ b/include/crm/common/internal.h
@@ -1,356 +1,357 @@
 /*
  * Copyright 2015-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef CRM_COMMON_INTERNAL__H
 #define CRM_COMMON_INTERNAL__H
 
 #include <unistd.h>             // getpid()
 #include <stdbool.h>            // bool
 #include <stdint.h>             // uint8_t, uint64_t
 #include <string.h>             // strcmp()
 #include <fcntl.h>              // open()
 #include <sys/types.h>          // uid_t, gid_t, pid_t
 
 #include <glib.h>               // guint, GList, GHashTable
 #include <libxml/tree.h>        // xmlNode
 
 #include <crm/common/util.h>    // crm_strdup_printf()
 #include <crm/common/logging.h>  // do_crm_log_unlikely(), etc.
 #include <crm/common/mainloop.h> // mainloop_io_t, struct ipc_client_callbacks
 #include <crm/common/results_internal.h>
 #include <crm/common/strings_internal.h>
 
 /* This says whether the current application is a Pacemaker daemon or not,
  * and is used to change default logging settings such as whether to log to
  * stderr, etc., as well as a few other details such as whether blackbox signal
  * handling is enabled.
  *
  * It is set when logging is initialized, and does not need to be set directly.
  */
 extern bool pcmk__is_daemon;
 
 // Number of elements in a statically defined array
 #define PCMK__NELEM(a) ((int) (sizeof(a)/sizeof(a[0])) )
 
 // Internal ACL-related utilities (from acl.c)
 
 char *pcmk__uid2username(uid_t uid);
 const char *pcmk__update_acl_user(xmlNode *request, const char *field,
                                   const char *peer_user);
 
 static inline bool
 pcmk__is_privileged(const char *user)
 {
     return user && (!strcmp(user, CRM_DAEMON_USER) || !strcmp(user, "root"));
 }
 
 #if SUPPORT_CIBSECRETS
 // Internal CIB utilities (from cib_secrets.c) */
 
 int pcmk__substitute_secrets(const char *rsc_id, GHashTable *params);
 #endif
 
 
 /* internal digest-related utilities (from digest.c) */
 
 bool pcmk__verify_digest(xmlNode *input, const char *expected);
 
 
 /* internal I/O utilities (from io.c) */
 
 int pcmk__real_path(const char *path, char **resolved_path);
 
 char *pcmk__series_filename(const char *directory, const char *series,
                             int sequence, bool bzip);
 int pcmk__read_series_sequence(const char *directory, const char *series,
                                unsigned int *seq);
 void pcmk__write_series_sequence(const char *directory, const char *series,
                                  unsigned int sequence, int max);
 int pcmk__chown_series_sequence(const char *directory, const char *series,
                                 uid_t uid, gid_t gid);
 
 int pcmk__build_path(const char *path_c, mode_t mode);
+char *pcmk__full_path(const char *filename, const char *dirname);
 bool pcmk__daemon_can_write(const char *dir, const char *file);
 void pcmk__sync_directory(const char *name);
 
 int pcmk__file_contents(const char *filename, char **contents);
 int pcmk__write_sync(int fd, const char *contents);
 int pcmk__set_nonblocking(int fd);
 const char *pcmk__get_tmpdir(void);
 
 void pcmk__close_fds_in_child(bool);
 
 /*!
  * \internal
  * \brief Open /dev/null to consume next available file descriptor
  *
  * Open /dev/null, disregarding the result. This is intended when daemonizing to
  * be able to null stdin, stdout, and stderr.
  *
  * \param[in] flags  O_RDONLY (stdin) or O_WRONLY (stdout and stderr)
  */
 static inline void
 pcmk__open_devnull(int flags)
 {
     // Static analysis clutter
     // cppcheck-suppress leakReturnValNotUsed
     (void) open("/dev/null", flags);
 }
 
 
 /* internal main loop utilities (from mainloop.c) */
 
 int pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata,
                            struct ipc_client_callbacks *callbacks,
                            mainloop_io_t **source);
 guint pcmk__mainloop_timer_get_period(mainloop_timer_t *timer);
 
 
 /* internal messaging utilities (from messages.c) */
 
 const char *pcmk__message_name(const char *name);
 
 
 /* internal name/value utilities (from nvpair.c) */
 
 int pcmk__scan_nvpair(const char *input, char **name, char **value);
 char *pcmk__format_nvpair(const char *name, const char *value,
                           const char *units);
 char *pcmk__format_named_time(const char *name, time_t epoch_time);
 
 
 /* internal procfs utilities (from procfs.c) */
 
 pid_t pcmk__procfs_pid_of(const char *name);
 unsigned int pcmk__procfs_num_cores(void);
 
 
 /* internal XML schema functions (from xml.c) */
 
 void crm_schema_init(void);
 void crm_schema_cleanup(void);
 
 
 /* internal functions related to process IDs (from pid.c) */
 
 /*!
  * \internal
  * \brief Check whether process exists (by PID and optionally executable path)
  *
  * \param[in] pid     PID of process to check
  * \param[in] daemon  If not NULL, path component to match with procfs entry
  *
  * \return Standard Pacemaker return code
  * \note Particular return codes of interest include pcmk_rc_ok for alive,
  *       ESRCH for process is not alive (verified by kill and/or executable path
  *       match), EACCES for caller unable or not allowed to check. A result of
  *       "alive" is less reliable when \p daemon is not provided or procfs is
  *       not available, since there is no guarantee that the PID has not been
  *       recycled for another process.
  * \note This function cannot be used to verify \e authenticity of the process.
  */
 int pcmk__pid_active(pid_t pid, const char *daemon);
 
 int pcmk__read_pidfile(const char *filename, pid_t *pid);
 int pcmk__pidfile_matches(const char *filename, pid_t expected_pid,
                           const char *expected_name, pid_t *pid);
 int pcmk__lock_pidfile(const char *filename, const char *name);
 
 
 /* internal functions related to resource operations (from operations.c) */
 
 // printf-style format to create operation ID from resource, action, interval
 #define PCMK__OP_FMT "%s_%s_%u"
 
 char *pcmk__op_key(const char *rsc_id, const char *op_type, guint interval_ms);
 char *pcmk__notify_key(const char *rsc_id, const char *notify_type,
                        const char *op_type);
 char *pcmk__transition_key(int transition_id, int action_id, int target_rc,
                            const char *node);
 void pcmk__filter_op_for_digest(xmlNode *param_set);
 
 
 // bitwise arithmetic utilities
 
 /*!
  * \internal
  * \brief Set specified flags in a flag group
  *
  * \param[in] function    Function name of caller
  * \param[in] line        Line number of caller
  * \param[in] log_level   Log a message at this level
  * \param[in] flag_type   Label describing this flag group (for logging)
  * \param[in] target      Name of object whose flags these are (for logging)
  * \param[in] flag_group  Flag group being manipulated
  * \param[in] flags       Which flags in the group should be set
  * \param[in] flags_str   Readable equivalent of \p flags (for logging)
  *
  * \return Possibly modified flag group
  */
 static inline uint64_t
 pcmk__set_flags_as(const char *function, int line, uint8_t log_level,
                    const char *flag_type, const char *target,
                    uint64_t flag_group, uint64_t flags, const char *flags_str)
 {
     uint64_t result = flag_group | flags;
 
     if (result != flag_group) {
         do_crm_log_unlikely(log_level,
                             "%s flags 0x%.8llx (%s) for %s set by %s:%d",
                             ((flag_type == NULL)? "Group of" : flag_type),
                             (unsigned long long) flags,
                             ((flags_str == NULL)? "flags" : flags_str),
                             ((target == NULL)? "target" : target),
                             function, line);
     }
     return result;
 }
 
 /*!
  * \internal
  * \brief Clear specified flags in a flag group
  *
  * \param[in] function    Function name of caller
  * \param[in] line        Line number of caller
  * \param[in] log_level   Log a message at this level
  * \param[in] flag_type   Label describing this flag group (for logging)
  * \param[in] target      Name of object whose flags these are (for logging)
  * \param[in] flag_group  Flag group being manipulated
  * \param[in] flags       Which flags in the group should be cleared
  * \param[in] flags_str   Readable equivalent of \p flags (for logging)
  *
  * \return Possibly modified flag group
  */
 static inline uint64_t
 pcmk__clear_flags_as(const char *function, int line, uint8_t log_level,
                      const char *flag_type, const char *target,
                      uint64_t flag_group, uint64_t flags, const char *flags_str)
 {
     uint64_t result = flag_group & ~flags;
 
     if (result != flag_group) {
         do_crm_log_unlikely(log_level,
                             "%s flags 0x%.8llx (%s) for %s cleared by %s:%d",
                             ((flag_type == NULL)? "Group of" : flag_type),
                             (unsigned long long) flags,
                             ((flags_str == NULL)? "flags" : flags_str),
                             ((target == NULL)? "target" : target),
                             function, line);
     }
     return result;
 }
 
 // miscellaneous utilities (from utils.c)
 
 void pcmk__daemonize(const char *name, const char *pidfile);
 void pcmk__panic(const char *origin);
 pid_t pcmk__locate_sbd(void);
 void pcmk__sleep_ms(unsigned int ms);
 
 extern int pcmk__score_red;
 extern int pcmk__score_green;
 extern int pcmk__score_yellow;
 
 /*!
  * \internal
  * \brief Resize a dynamically allocated memory block
  *
  * \param[in] ptr   Memory block to resize (or NULL to allocate new memory)
  * \param[in] size  New size of memory block in bytes (must be > 0)
  *
  * \return Pointer to resized memory block
  *
  * \note This asserts on error, so the result is guaranteed to be non-NULL
  *       (which is the main advantage of this over directly using realloc()).
  */
 static inline void *
 pcmk__realloc(void *ptr, size_t size)
 {
     void *new_ptr;
 
     // realloc(p, 0) can replace free(p) but this wrapper can't
     CRM_ASSERT(size > 0);
 
     new_ptr = realloc(ptr, size);
     if (new_ptr == NULL) {
         free(ptr);
         abort();
     }
     return new_ptr;
 }
 
 
 static inline char *
 pcmk__getpid_s(void)
 {
     return crm_strdup_printf("%lu", (unsigned long) getpid());
 }
 
 // More efficient than g_list_length(list) == 1
 static inline bool
 pcmk__list_of_1(GList *list)
 {
     return list && (list->next == NULL);
 }
 
 // More efficient than g_list_length(list) > 1
 static inline bool
 pcmk__list_of_multiple(GList *list)
 {
     return list && (list->next != NULL);
 }
 
 /* convenience functions for failure-related node attributes */
 
 #define PCMK__FAIL_COUNT_PREFIX   "fail-count"
 #define PCMK__LAST_FAILURE_PREFIX "last-failure"
 
 /*!
  * \internal
  * \brief Generate a failure-related node attribute name for a resource
  *
  * \param[in] prefix       Start of attribute name
  * \param[in] rsc_id       Resource name
  * \param[in] op           Operation name
  * \param[in] interval_ms  Operation interval
  *
  * \return Newly allocated string with attribute name
  *
  * \note Failure attributes are named like PREFIX-RSC#OP_INTERVAL (for example,
  *       "fail-count-myrsc#monitor_30000"). The '#' is used because it is not
  *       a valid character in a resource ID, to reliably distinguish where the
  *       operation name begins. The '_' is used simply to be more comparable to
  *       action labels like "myrsc_monitor_30000".
  */
 static inline char *
 pcmk__fail_attr_name(const char *prefix, const char *rsc_id, const char *op,
                    guint interval_ms)
 {
     CRM_CHECK(prefix && rsc_id && op, return NULL);
     return crm_strdup_printf("%s-%s#%s_%u", prefix, rsc_id, op, interval_ms);
 }
 
 static inline char *
 pcmk__failcount_name(const char *rsc_id, const char *op, guint interval_ms)
 {
     return pcmk__fail_attr_name(PCMK__FAIL_COUNT_PREFIX, rsc_id, op,
                                 interval_ms);
 }
 
 static inline char *
 pcmk__lastfailure_name(const char *rsc_id, const char *op, guint interval_ms)
 {
     return pcmk__fail_attr_name(PCMK__LAST_FAILURE_PREFIX, rsc_id, op,
                                 interval_ms);
 }
 
 // internal resource agent functions (from agents.c)
 int pcmk__effective_rc(int rc);
 
 #endif /* CRM_COMMON_INTERNAL__H */
diff --git a/lib/common/io.c b/lib/common/io.c
index c1a8505dbb..ffdcae2069 100644
--- a/lib/common/io.c
+++ b/lib/common/io.c
@@ -1,635 +1,662 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/resource.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <string.h>
 #include <stdlib.h>
 #include <fcntl.h>
 #include <dirent.h>
 #include <errno.h>
 #include <limits.h>
 #include <pwd.h>
 #include <grp.h>
 
 #include <crm/crm.h>
 #include <crm/common/util.h>
 
 /*!
  * \internal
  * \brief Create a directory, including any parent directories needed
  *
  * \param[in] path_c Pathname of the directory to create
  * \param[in] mode Permissions to be used (with current umask) when creating
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__build_path(const char *path_c, mode_t mode)
 {
     int offset = 1, len = 0;
     int rc = pcmk_rc_ok;
     char *path = strdup(path_c);
 
     // cppcheck seems not to understand the abort logic in CRM_CHECK
     // cppcheck-suppress memleak
     CRM_CHECK(path != NULL, return -ENOMEM);
     for (len = strlen(path); offset < len; offset++) {
         if (path[offset] == '/') {
             path[offset] = 0;
             if ((mkdir(path, mode) < 0) && (errno != EEXIST)) {
                 rc = errno;
                 goto done;
             }
             path[offset] = '/';
         }
     }
     if ((mkdir(path, mode) < 0) && (errno != EEXIST)) {
         rc = errno;
     }
 done:
     free(path);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Return canonicalized form of a path name
  *
  * \param[in]  path           Pathname to canonicalize
  * \param[out] resolved_path  Where to store canonicalized pathname
  *
  * \return Standard Pacemaker return code
  * \note The caller is responsible for freeing \p resolved_path on success.
  * \note This function exists because not all C library versions of
  *       realpath(path, resolved_path) support a NULL resolved_path.
  */
 int
 pcmk__real_path(const char *path, char **resolved_path)
 {
     CRM_CHECK((path != NULL) && (resolved_path != NULL), return EINVAL);
 
 #if _POSIX_VERSION >= 200809L
     /* Recent C libraries can dynamically allocate memory as needed */
     *resolved_path = realpath(path, NULL);
     return (*resolved_path == NULL)? errno : pcmk_rc_ok;
 
 #elif defined(PATH_MAX)
     /* Older implementations require pre-allocated memory */
     /* (this is less desirable because PATH_MAX may be huge or not defined) */
     *resolved_path = malloc(PATH_MAX);
     if ((*resolved_path == NULL) || (realpath(path, *resolved_path) == NULL)) {
         return errno;
     }
     return pcmk_rc_ok;
 #else
     *resolved_path = NULL;
     return ENOTSUP;
 #endif
 }
 
 /*!
  * \internal
  * \brief Create a file name using a sequence number
  *
  * \param[in] directory  Directory that contains the file series
  * \param[in] series     Start of file name
  * \param[in] sequence   Sequence number
  * \param[in] bzip       Whether to use ".bz2" instead of ".raw" as extension
  *
  * \return Newly allocated file path (asserts on error, so always non-NULL)
  * \note The caller is responsible for freeing the return value.
  */
 char *
 pcmk__series_filename(const char *directory, const char *series, int sequence,
                       bool bzip)
 {
     CRM_ASSERT((directory != NULL) && (series != NULL));
     return crm_strdup_printf("%s/%s-%d.%s", directory, series, sequence,
                              (bzip? "bz2" : "raw"));
 }
 
 /*!
  * \internal
  * \brief Read sequence number stored in a file series' .last file
  *
  * \param[in]  directory  Directory that contains the file series
  * \param[in]  series     Start of file name
  * \param[out] seq        Where to store the sequence number
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__read_series_sequence(const char *directory, const char *series,
                            unsigned int *seq)
 {
     int rc;
     FILE *fp = NULL;
     char *series_file = NULL;
 
     if ((directory == NULL) || (series == NULL) || (seq == NULL)) {
         return EINVAL;
     }
 
     series_file = crm_strdup_printf("%s/%s.last", directory, series);
     fp = fopen(series_file, "r");
     if (fp == NULL) {
         rc = errno;
         crm_debug("Could not open series file %s: %s",
                   series_file, strerror(rc));
         free(series_file);
         return rc;
     }
     errno = 0;
     if (fscanf(fp, "%u", seq) != 1) {
         rc = (errno == 0)? pcmk_rc_unknown_format : errno;
         crm_debug("Could not read sequence number from series file %s: %s",
                   series_file, pcmk_rc_str(rc));
         fclose(fp);
         return rc;
     }
     fclose(fp);
     crm_trace("Found last sequence number %u in series file %s",
               *seq, series_file);
     free(series_file);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Write sequence number to a file series' .last file
  *
  * \param[in] directory  Directory that contains the file series
  * \param[in] series     Start of file name
  * \param[in] sequence   Sequence number to write
  * \param[in] max        Maximum sequence value, after which it is reset to 0
  *
  * \note This function logs some errors but does not return any to the caller
  */
 void
 pcmk__write_series_sequence(const char *directory, const char *series,
                             unsigned int sequence, int max)
 {
     int rc = 0;
     FILE *file_strm = NULL;
     char *series_file = NULL;
 
     CRM_CHECK(directory != NULL, return);
     CRM_CHECK(series != NULL, return);
 
     if (max == 0) {
         return;
     }
     if (max > 0 && sequence >= max) {
         sequence = 0;
     }
 
     series_file = crm_strdup_printf("%s/%s.last", directory, series);
     file_strm = fopen(series_file, "w");
     if (file_strm != NULL) {
         rc = fprintf(file_strm, "%u", sequence);
         if (rc < 0) {
             crm_perror(LOG_ERR, "Cannot write to series file %s", series_file);
         }
 
     } else {
         crm_err("Cannot open series file %s for writing", series_file);
     }
 
     if (file_strm != NULL) {
         fflush(file_strm);
         fclose(file_strm);
     }
 
     crm_trace("Wrote %d to %s", sequence, series_file);
     free(series_file);
 }
 
 /*!
  * \internal
  * \brief Change the owner and group of a file series' .last file
  *
  * \param[in] dir  Directory that contains series
  * \param[in] uid  User ID of desired file owner
  * \param[in] gid  Group ID of desired file group
  *
  * \return Standard Pacemaker return code
  * \note The caller must have the appropriate privileges.
  */
 int
 pcmk__chown_series_sequence(const char *directory, const char *series,
                             uid_t uid, gid_t gid)
 {
     char *series_file = NULL;
     int rc = pcmk_rc_ok;
 
     if ((directory == NULL) || (series == NULL)) {
         return EINVAL;
     }
     series_file = crm_strdup_printf("%s/%s.last", directory, series);
     if (chown(series_file, uid, gid) < 0) {
         rc = errno;
     }
     free(series_file);
     return rc;
 }
 
 static bool
 pcmk__daemon_user_can_write(const char *target_name, struct stat *target_stat)
 {
     struct passwd *sys_user = NULL;
 
     errno = 0;
     sys_user = getpwnam(CRM_DAEMON_USER);
     if (sys_user == NULL) {
         crm_notice("Could not find user %s: %s",
                    CRM_DAEMON_USER, pcmk_strerror(errno));
         return FALSE;
     }
     if (target_stat->st_uid != sys_user->pw_uid) {
         crm_notice("%s is not owned by user %s " CRM_XS " uid %d != %d",
                    target_name, CRM_DAEMON_USER, sys_user->pw_uid,
                    target_stat->st_uid);
         return FALSE;
     }
     if ((target_stat->st_mode & (S_IRUSR | S_IWUSR)) == 0) {
         crm_notice("%s is not readable and writable by user %s "
                    CRM_XS " st_mode=0%lo",
                    target_name, CRM_DAEMON_USER,
                    (unsigned long) target_stat->st_mode);
         return FALSE;
     }
     return TRUE;
 }
 
 static bool
 pcmk__daemon_group_can_write(const char *target_name, struct stat *target_stat)
 {
     struct group *sys_grp = NULL;
 
     errno = 0;
     sys_grp = getgrnam(CRM_DAEMON_GROUP);
     if (sys_grp == NULL) {
         crm_notice("Could not find group %s: %s",
                    CRM_DAEMON_GROUP, pcmk_strerror(errno));
         return FALSE;
     }
 
     if (target_stat->st_gid != sys_grp->gr_gid) {
         crm_notice("%s is not owned by group %s " CRM_XS " uid %d != %d",
                    target_name, CRM_DAEMON_GROUP,
                    sys_grp->gr_gid, target_stat->st_gid);
         return FALSE;
     }
 
     if ((target_stat->st_mode & (S_IRGRP | S_IWGRP)) == 0) {
         crm_notice("%s is not readable and writable by group %s "
                    CRM_XS " st_mode=0%lo",
                    target_name, CRM_DAEMON_GROUP,
                    (unsigned long) target_stat->st_mode);
         return FALSE;
     }
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Check whether a directory or file is writable by the cluster daemon
  *
  * Return true if either the cluster daemon user or cluster daemon group has
  * write permission on a specified file or directory.
  *
  * \param[in] dir      Directory to check (this argument must be specified, and
  *                     the directory must exist)
  * \param[in] file     File to check (only the directory will be checked if this
  *                     argument is not specified or the file does not exist)
  *
  * \return true if target is writable by cluster daemon, false otherwise
  */
 bool
 pcmk__daemon_can_write(const char *dir, const char *file)
 {
     int s_res = 0;
     struct stat buf;
     char *full_file = NULL;
     const char *target = NULL;
 
     // Caller must supply directory
     CRM_ASSERT(dir != NULL);
 
     // If file is given, check whether it exists as a regular file
     if (file != NULL) {
         full_file = crm_strdup_printf("%s/%s", dir, file);
         target = full_file;
 
         s_res = stat(full_file, &buf);
         if (s_res < 0) {
             crm_notice("%s not found: %s", target, pcmk_strerror(errno));
             free(full_file);
             full_file = NULL;
             target = NULL;
 
         } else if (S_ISREG(buf.st_mode) == FALSE) {
             crm_err("%s must be a regular file " CRM_XS " st_mode=0%lo",
                     target, (unsigned long) buf.st_mode);
             free(full_file);
             return false;
         }
     }
 
     // If file is not given, ensure dir exists as directory
     if (target == NULL) {
         target = dir;
         s_res = stat(dir, &buf);
         if (s_res < 0) {
             crm_err("%s not found: %s", dir, pcmk_strerror(errno));
             return false;
 
         } else if (S_ISDIR(buf.st_mode) == FALSE) {
             crm_err("%s must be a directory " CRM_XS " st_mode=0%lo",
                     dir, (unsigned long) buf.st_mode);
             return false;
         }
     }
 
     if (!pcmk__daemon_user_can_write(target, &buf)
         && !pcmk__daemon_group_can_write(target, &buf)) {
 
         crm_err("%s must be owned and writable by either user %s or group %s "
                 CRM_XS " st_mode=0%lo",
                 target, CRM_DAEMON_USER, CRM_DAEMON_GROUP,
                 (unsigned long) buf.st_mode);
         free(full_file);
         return false;
     }
 
     free(full_file);
     return true;
 }
 
 /*!
  * \internal
  * \brief Flush and sync a directory to disk
  *
  * \param[in] name Directory to flush and sync
  * \note This function logs errors but does not return them to the caller
  */
 void
 pcmk__sync_directory(const char *name)
 {
     int fd;
     DIR *directory;
 
     directory = opendir(name);
     if (directory == NULL) {
         crm_perror(LOG_ERR, "Could not open %s for syncing", name);
         return;
     }
 
     fd = dirfd(directory);
     if (fd < 0) {
         crm_perror(LOG_ERR, "Could not obtain file descriptor for %s", name);
         return;
     }
 
     if (fsync(fd) < 0) {
         crm_perror(LOG_ERR, "Could not sync %s", name);
     }
     if (closedir(directory) < 0) {
         crm_perror(LOG_ERR, "Could not close %s after fsync", name);
     }
 }
 
 /*!
  * \internal
  * \brief Read the contents of a file
  *
  * \param[in]  filename  Name of file to read
  * \param[out] contents  Where to store file contents
  *
  * \return Standard Pacemaker return code
  * \note On success, the caller is responsible for freeing contents.
  */
 int
 pcmk__file_contents(const char *filename, char **contents)
 {
     FILE *fp;
     int length, read_len;
     int rc = pcmk_rc_ok;
 
     if ((filename == NULL) || (contents == NULL)) {
         return EINVAL;
     }
 
     fp = fopen(filename, "r");
     if ((fp == NULL) || (fseek(fp, 0L, SEEK_END) < 0)) {
         rc = errno;
         goto bail;
     }
 
     length = ftell(fp);
     if (length < 0) {
         rc = errno;
         goto bail;
     }
 
     if (length == 0) {
         *contents = NULL;
     } else {
         *contents = calloc(length + 1, sizeof(char));
         if (*contents == NULL) {
             rc = errno;
             goto bail;
         }
         rewind(fp);
         read_len = fread(*contents, 1, length, fp); /* Coverity: False positive */
         if (read_len != length) {
             free(*contents);
             *contents = NULL;
             rc = EIO;
         }
     }
 
 bail:
     if (fp != NULL) {
         fclose(fp);
     }
     return rc;
 }
 
 /*!
  * \internal
  * \brief Write text to a file, flush and sync it to disk, then close the file
  *
  * \param[in] fd        File descriptor opened for writing
  * \param[in] contents  String to write to file
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__write_sync(int fd, const char *contents)
 {
     int rc = 0;
     FILE *fp = fdopen(fd, "w");
 
     if (fp == NULL) {
         return errno;
     }
     if ((contents != NULL) && (fprintf(fp, "%s", contents) < 0)) {
         rc = EIO;
     }
     if (fflush(fp) != 0) {
         rc = errno;
     }
     if (fsync(fileno(fp)) < 0) {
         rc = errno;
     }
     fclose(fp);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Set a file descriptor to non-blocking
  *
  * \param[in] fd  File descriptor to use
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__set_nonblocking(int fd)
 {
     int flag = fcntl(fd, F_GETFL);
 
     if (flag < 0) {
         return errno;
     }
     if (fcntl(fd, F_SETFL, flag | O_NONBLOCK) < 0) {
         return errno;
     }
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Get directory name for temporary files
  *
  * Return the value of the TMPDIR environment variable if it is set to a
  * full path, otherwise return "/tmp".
  *
  * \return Name of directory to be used for temporary files
  */
 const char *
 pcmk__get_tmpdir()
 {
     const char *dir = getenv("TMPDIR");
 
     return (dir && (*dir == '/'))? dir : "/tmp";
 }
 
 /*!
  * \internal
  * \brief Close open file descriptors
  *
  * Close all file descriptors (except optionally stdin, stdout, and stderr),
  * which is a best practice for a new child process forked for the purpose of
  * executing an external program.
  *
  * \param[in] bool  If true, close stdin, stdout, and stderr as well
  */
 void
 pcmk__close_fds_in_child(bool all)
 {
     DIR *dir;
     struct rlimit rlim;
     rlim_t max_fd;
     int min_fd = (all? 0 : (STDERR_FILENO + 1));
 
     /* Find the current process's (soft) limit for open files. getrlimit()
      * should always work, but have a fallback just in case.
      */
     if (getrlimit(RLIMIT_NOFILE, &rlim) == 0) {
         max_fd = rlim.rlim_cur - 1;
     } else {
         long conf_max = sysconf(_SC_OPEN_MAX);
 
         max_fd = (conf_max > 0)? conf_max : 1024;
     }
 
     /* /proc/self/fd (on Linux) or /dev/fd (on most OSes) contains symlinks to
      * all open files for the current process, named as the file descriptor.
      * Use this if available, because it's more efficient than a shotgun
      * approach to closing descriptors.
      */
 #if SUPPORT_PROCFS
     dir = opendir("/proc/self/fd");
     if (dir == NULL) {
         dir = opendir("/dev/fd");
     }
 #else
     dir = opendir("/dev/fd");
 #endif
     if (dir != NULL) {
         struct dirent *entry;
         int dir_fd = dirfd(dir);
 
         while ((entry = readdir(dir)) != NULL) {
             int lpc = atoi(entry->d_name);
 
             /* How could one of these entries be higher than max_fd, you ask?
              * It isn't possible in normal operation, but when run under
              * valgrind, valgrind can open high-numbered file descriptors for
              * its own use that are higher than the process's soft limit.
              * These will show up in the fd directory but aren't closable.
              */
             if ((lpc >= min_fd) && (lpc <= max_fd) && (lpc != dir_fd)) {
                 close(lpc);
             }
         }
         closedir(dir);
         return;
     }
 
     /* If no fd directory is available, iterate over all possible descriptors.
      * This is less efficient due to the overhead of many system calls.
      */
     for (int lpc = max_fd; lpc >= min_fd; lpc--) {
         close(lpc);
     }
 }
 
+/*!
+ * \brief Duplicate a file path, inserting a prefix if not absolute
+ *
+ * \param[in] filename  File path to duplicate
+ * \param[in] dirname   If filename is not absolute, prefix to add
+ *
+ * \return Newly allocated memory with full path (guaranteed non-NULL)
+ */
+char *
+pcmk__full_path(const char *filename, const char *dirname)
+{
+    char *path = NULL;
+
+    CRM_ASSERT(filename != NULL);
+
+    if (filename[0] == '/') {
+        path = strdup(filename);
+        CRM_ASSERT(path != NULL);
+
+    } else {
+        CRM_ASSERT(dirname != NULL);
+        path = crm_strdup_printf("%s/%s", dirname, filename);
+    }
+
+    return path;
+}
+
 // Deprecated functions kept only for backward API compatibility
 // LCOV_EXCL_START
 
 #include <crm/common/util_compat.h>
 
 void
 crm_build_path(const char *path_c, mode_t mode)
 {
     int rc = pcmk__build_path(path_c, mode);
 
     if (rc != pcmk_rc_ok) {
         crm_err("Could not create directory '%s': %s",
                 path_c, pcmk_rc_str(rc));
     }
 }
 
 // LCOV_EXCL_STOP
 // End deprecated API
diff --git a/lib/common/tests/Makefile.am b/lib/common/tests/Makefile.am
index 4bacad4bf8..cd97327e55 100644
--- a/lib/common/tests/Makefile.am
+++ b/lib/common/tests/Makefile.am
@@ -1 +1,20 @@
-SUBDIRS = agents cmdline flags iso8601 operations results strings utils xpath
+#
+# Copyright 2020-2021 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+SUBDIRS = \
+	agents		\
+	cmdline		\
+	flags		\
+	io		\
+	iso8601		\
+	operations	\
+	results		\
+	strings		\
+	utils		\
+	xpath
diff --git a/lib/common/tests/io/Makefile.am b/lib/common/tests/io/Makefile.am
new file mode 100644
index 0000000000..6760a1d9c9
--- /dev/null
+++ b/lib/common/tests/io/Makefile.am
@@ -0,0 +1,30 @@
+#
+# Copyright 2020-2021 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include
+LDADD = $(top_builddir)/lib/common/libcrmcommon.la
+
+include $(top_srcdir)/mk/glib-tap.mk
+
+# Add each test program here.  Each test should be written as a little standalone
+# program using the glib unit testing functions.  See the documentation for more
+# information.
+#
+# https://developer.gnome.org/glib/unstable/glib-Testing.html
+#
+# Add "_test" to the end of all test program names to simplify .gitignore.
+test_programs =	\
+	pcmk__full_path_test
+
+# If any extra data needs to be added to the source distribution, add it to the
+# following list.
+dist_test_data =
+
+# If any extra data needs to be used by tests but should not be added to the
+# source distribution, add it to the following list.
+test_data =
diff --git a/lib/common/tests/io/pcmk__full_path_test.c b/lib/common/tests/io/pcmk__full_path_test.c
new file mode 100644
index 0000000000..03b3d13838
--- /dev/null
+++ b/lib/common/tests/io/pcmk__full_path_test.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020-2021 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/types.h>
+#include <crm_internal.h>
+
+static void
+full_path(void)
+{
+    char *path = NULL;
+
+    path = pcmk__full_path("file", "/dir");
+    g_assert_cmpint(strcmp(path, "/dir/file"), ==, 0);
+    free(path);
+
+    path = pcmk__full_path("/full/path", "/dir");
+    g_assert_cmpint(strcmp(path, "/full/path"), ==, 0);
+    free(path);
+
+    path = pcmk__full_path("../relative/path", "/dir");
+    g_assert_cmpint(strcmp(path, "/dir/../relative/path"), ==, 0);
+    free(path);
+}
+
+int
+main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/common/io/full_path/full_path", full_path);
+
+    return g_test_run();
+}
diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
index c0cc50c83a..64779671be 100644
--- a/lib/lrmd/lrmd_client.c
+++ b/lib/lrmd/lrmd_client.c
@@ -1,2344 +1,2347 @@
 /*
  * Copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdint.h>         // uint32_t, uint64_t
 #include <stdarg.h>
 #include <string.h>
 #include <ctype.h>
 #include <errno.h>
 
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <glib.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/lrmd.h>
 #include <crm/lrmd_internal.h>
 #include <crm/services.h>
+#include <crm/services_internal.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/common/remote_internal.h>
 #include <crm/msg_xml.h>
 
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  undef KEYFILE
 #  include <gnutls/gnutls.h>
 #endif
 
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <netinet/ip.h>
 #include <arpa/inet.h>
 #include <netdb.h>
 
 #define MAX_TLS_RECV_WAIT 10000
 
 CRM_TRACE_INIT_DATA(lrmd);
 
 static int lrmd_api_disconnect(lrmd_t * lrmd);
 static int lrmd_api_is_connected(lrmd_t * lrmd);
 
 /* IPC proxy functions */
 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
 gnutls_psk_client_credentials_t psk_cred_s;
 static void lrmd_tls_disconnect(lrmd_t * lrmd);
 static int global_remote_msg_id = 0;
 static void lrmd_tls_connection_destroy(gpointer userdata);
 #endif
 
 typedef struct lrmd_private_s {
     uint64_t type;
     char *token;
     mainloop_io_t *source;
 
     /* IPC parameters */
     crm_ipc_t *ipc;
 
     pcmk__remote_t *remote;
 
     /* Extra TLS parameters */
     char *remote_nodename;
 #ifdef HAVE_GNUTLS_GNUTLS_H
     char *server;
     int port;
     gnutls_psk_client_credentials_t psk_cred_c;
 
     /* while the async connection is occurring, this is the id
      * of the connection timeout timer. */
     int async_timer;
     int sock;
     /* since tls requires a round trip across the network for a
      * request/reply, there are times where we just want to be able
      * to send a request from the client and not wait around (or even care
      * about) what the reply is. */
     int expected_late_replies;
     GList *pending_notify;
     crm_trigger_t *process_notify;
 #endif
 
     lrmd_event_callback callback;
 
     /* Internal IPC proxy msg passing for remote guests */
     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
     void *proxy_callback_userdata;
     char *peer_version;
 } lrmd_private_t;
 
 static lrmd_list_t *
 lrmd_list_add(lrmd_list_t * head, const char *value)
 {
     lrmd_list_t *p, *end;
 
     p = calloc(1, sizeof(lrmd_list_t));
     p->val = strdup(value);
 
     end = head;
     while (end && end->next) {
         end = end->next;
     }
 
     if (end) {
         end->next = p;
     } else {
         head = p;
     }
 
     return head;
 }
 
 void
 lrmd_list_freeall(lrmd_list_t * head)
 {
     lrmd_list_t *p;
 
     while (head) {
         char *val = (char *)head->val;
 
         p = head->next;
         free(val);
         free(head);
         head = p;
     }
 }
 
 lrmd_key_value_t *
 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
 {
     lrmd_key_value_t *p, *end;
 
     p = calloc(1, sizeof(lrmd_key_value_t));
     p->key = strdup(key);
     p->value = strdup(value);
 
     end = head;
     while (end && end->next) {
         end = end->next;
     }
 
     if (end) {
         end->next = p;
     } else {
         head = p;
     }
 
     return head;
 }
 
 void
 lrmd_key_value_freeall(lrmd_key_value_t * head)
 {
     lrmd_key_value_t *p;
 
     while (head) {
         p = head->next;
         free(head->key);
         free(head->value);
         free(head);
         head = p;
     }
 }
 
 /*!
  * Create a new lrmd_event_data_t object
  *
  * \param[in] rsc_id       ID of resource involved in event
  * \param[in] task         Action name
  * \param[in] interval_ms  Action interval
  *
  * \return Newly allocated and initialized lrmd_event_data_t
  * \note This functions asserts on memory errors, so the return value is
  *       guaranteed to be non-NULL. The caller is responsible for freeing the
  *       result with lrmd_free_event().
  */
 lrmd_event_data_t *
 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
 {
     lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));
 
     CRM_ASSERT(event != NULL);
     if (rsc_id != NULL) {
         event->rsc_id = strdup(rsc_id);
         CRM_ASSERT(event->rsc_id != NULL);
     }
     if (task != NULL) {
         event->op_type = strdup(task);
         CRM_ASSERT(event->op_type != NULL);
     }
     event->interval_ms = interval_ms;
     return event;
 }
 
 lrmd_event_data_t *
 lrmd_copy_event(lrmd_event_data_t * event)
 {
     lrmd_event_data_t *copy = NULL;
 
     copy = calloc(1, sizeof(lrmd_event_data_t));
 
     /* This will get all the int values.
      * we just have to be careful not to leave any
      * dangling pointers to strings. */
     memcpy(copy, event, sizeof(lrmd_event_data_t));
 
     copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
     copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
     copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
     copy->output = event->output ? strdup(event->output) : NULL;
     copy->exit_reason = event->exit_reason ? strdup(event->exit_reason) : NULL;
     copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;
     copy->params = pcmk__str_table_dup(event->params);
 
     return copy;
 }
 
 /*!
  * \brief Free an executor event
  *
  * \param[in]  Executor event object to free
  */
 void
 lrmd_free_event(lrmd_event_data_t *event)
 {
     if (event == NULL) {
         return;
     }
     // @TODO Why are these const char *?
     free((void *) event->rsc_id);
     free((void *) event->op_type);
     free((void *) event->user_data);
     free((void *) event->output);
     free((void *) event->exit_reason);
     free((void *) event->remote_nodename);
     if (event->params != NULL) {
         g_hash_table_destroy(event->params);
     }
     free(event);
 }
 
 static void
 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
 {
     const char *type;
     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
     lrmd_private_t *native = lrmd->lrmd_private;
     lrmd_event_data_t event = { 0, };
 
     if (proxy_session != NULL) {
         /* this is proxy business */
         lrmd_internal_proxy_dispatch(lrmd, msg);
         return;
     } else if (!native->callback) {
         /* no callback set */
         crm_trace("notify event received but client has not set callback");
         return;
     }
 
     event.remote_nodename = native->remote_nodename;
     type = crm_element_value(msg, F_LRMD_OPERATION);
     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 
     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
         event.type = lrmd_event_register;
     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
         event.type = lrmd_event_unregister;
     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
         time_t epoch = 0;
 
         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
         crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 
         crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
         event.t_run = (unsigned int) epoch;
 
         crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
         event.t_rcchange = (unsigned int) epoch;
 
         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 
         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
         event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
         event.exit_reason = crm_element_value(msg, F_LRMD_RSC_EXIT_REASON);
         event.type = lrmd_event_exec_complete;
 
         event.params = xml2list(msg);
     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
         event.type = lrmd_event_new_client;
     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
         event.type = lrmd_event_poke;
     } else {
         return;
     }
 
     crm_trace("op %s notify event received", type);
     native->callback(&event);
 
     if (event.params) {
         g_hash_table_destroy(event.params);
     }
 }
 
 // \return Always 0, to indicate that IPC mainloop source should be kept
 static int
 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->callback != NULL) {
         xmlNode *msg = string2xml(buffer);
 
         lrmd_dispatch_internal(lrmd, msg);
         free_xml(msg);
     }
     return 0;
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_free_xml(gpointer userdata)
 {
     free_xml((xmlNode *) userdata);
 }
 
 static bool
 remote_executor_connected(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     return (native->remote->tls_session != NULL);
 }
 
 /*!
  * \internal
  * \brief TLS dispatch function (for both trigger and file descriptor sources)
  *
  * \param[in] userdata  API connection
  *
  * \return Always return a nonnegative value, which as a file descriptor
  *         dispatch function means keep the mainloop source, and as a
  *         trigger dispatch function, 0 means remove the trigger from the
  *         mainloop while 1 means keep it (and job completed)
  */
 static int
 lrmd_tls_dispatch(gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *xml = NULL;
     int rc = pcmk_rc_ok;
 
     if (!remote_executor_connected(lrmd)) {
         crm_trace("TLS dispatch triggered after disconnect");
         return 0;
     }
 
     crm_trace("TLS dispatch triggered");
 
     /* First check if there are any pending notifies to process that came
      * while we were waiting for replies earlier. */
     if (native->pending_notify) {
         GList *iter = NULL;
 
         crm_trace("Processing pending notifies");
         for (iter = native->pending_notify; iter; iter = iter->next) {
             lrmd_dispatch_internal(lrmd, iter->data);
         }
         g_list_free_full(native->pending_notify, lrmd_free_xml);
         native->pending_notify = NULL;
     }
 
     /* Next read the current buffer and see if there are any messages to handle. */
     switch (pcmk__remote_ready(native->remote, 0)) {
         case pcmk_rc_ok:
             rc = pcmk__read_remote_message(native->remote, -1);
             xml = pcmk__remote_message_xml(native->remote);
             break;
         case ETIME:
             // Nothing to read, check if a full message is already in buffer
             xml = pcmk__remote_message_xml(native->remote);
             break;
         default:
             rc = ENOTCONN;
             break;
     }
     while (xml) {
         const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
         if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
             lrmd_dispatch_internal(lrmd, xml);
         } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
             if (native->expected_late_replies > 0) {
                 native->expected_late_replies--;
             } else {
                 int reply_id = 0;
                 crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
                 /* if this happens, we want to know about it */
                 crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
             }
         }
         free_xml(xml);
         xml = pcmk__remote_message_xml(native->remote);
     }
 
     if (rc == ENOTCONN) {
         crm_info("Lost %s executor connection while reading data",
                  (native->remote_nodename? native->remote_nodename : "local"));
         lrmd_tls_disconnect(lrmd);
         return 0;
     }
     return 1;
 }
 #endif
 
 /* Not used with mainloop */
 int
 lrmd_poll(lrmd_t * lrmd, int timeout)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case pcmk__client_ipc:
             return crm_ipc_ready(native->ipc);
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case pcmk__client_tls:
             if (native->pending_notify) {
                 return 1;
             } else {
                 int rc = pcmk__remote_ready(native->remote, 0);
 
                 switch (rc) {
                     case pcmk_rc_ok:
                         return 1;
                     case ETIME:
                         return 0;
                     default:
                         return pcmk_rc2legacy(rc);
                 }
             }
 #endif
         default:
             crm_err("Unsupported executor connection type (bug?): %d",
                     native->type);
             return -EPROTONOSUPPORT;
     }
 }
 
 /* Not used with mainloop */
 bool
 lrmd_dispatch(lrmd_t * lrmd)
 {
     lrmd_private_t *private = NULL;
 
     CRM_ASSERT(lrmd != NULL);
 
     private = lrmd->lrmd_private;
     switch (private->type) {
         case pcmk__client_ipc:
             while (crm_ipc_ready(private->ipc)) {
                 if (crm_ipc_read(private->ipc) > 0) {
                     const char *msg = crm_ipc_buffer(private->ipc);
 
                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
                 }
             }
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case pcmk__client_tls:
             lrmd_tls_dispatch(lrmd);
             break;
 #endif
         default:
             crm_err("Unsupported executor connection type (bug?): %d",
                     private->type);
     }
 
     if (lrmd_api_is_connected(lrmd) == FALSE) {
         crm_err("Connection closed");
         return FALSE;
     }
 
     return TRUE;
 }
 
 static xmlNode *
 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
                enum lrmd_call_options options)
 {
     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 
     CRM_CHECK(op_msg != NULL, return NULL);
     CRM_CHECK(token != NULL, return NULL);
 
     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
     crm_xml_add(op_msg, F_TYPE, T_LRMD);
     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 
     if (data != NULL) {
         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
     }
 
     crm_trace("Created executor %s command with call options %.8lx (%d)",
               op, (long)options, options);
     return op_msg;
 }
 
 static void
 lrmd_ipc_connection_destroy(gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     crm_info("IPC connection destroyed");
 
     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
     native->ipc = NULL;
     native->source = NULL;
 
     if (native->callback) {
         lrmd_event_data_t event = { 0, };
         event.type = lrmd_event_disconnect;
         event.remote_nodename = native->remote_nodename;
         native->callback(&event);
     }
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_tls_connection_destroy(gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     crm_info("TLS connection destroyed");
 
     if (native->remote->tls_session) {
         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
     }
     if (native->psk_cred_c) {
         gnutls_psk_free_client_credentials(native->psk_cred_c);
     }
     if (native->sock) {
         close(native->sock);
     }
     if (native->process_notify) {
         mainloop_destroy_trigger(native->process_notify);
         native->process_notify = NULL;
     }
     if (native->pending_notify) {
         g_list_free_full(native->pending_notify, lrmd_free_xml);
         native->pending_notify = NULL;
     }
 
     free(native->remote->buffer);
     native->remote->buffer = NULL;
     native->source = 0;
     native->sock = 0;
     native->psk_cred_c = NULL;
     native->remote->tls_session = NULL;
     native->sock = 0;
 
     if (native->callback) {
         lrmd_event_data_t event = { 0, };
         event.remote_nodename = native->remote_nodename;
         event.type = lrmd_event_disconnect;
         native->callback(&event);
     }
     return;
 }
 
 // \return Standard Pacemaker return code
 int
 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
                       const char *msg_type)
 {
     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
     return pcmk__remote_send_xml(session, msg);
 }
 
 // \return Standard Pacemaker return code
 static int
 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
                   xmlNode **reply)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
     time_t start = time(NULL);
     const char *msg_type = NULL;
     int reply_id = 0;
     int remaining_timeout = 0;
     int rc = pcmk_rc_ok;
 
     /* A timeout of 0 here makes no sense.  We have to wait a period of time
      * for the response to come back.  If -1 or 0, default to 10 seconds. */
     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
         total_timeout = MAX_TLS_RECV_WAIT;
     }
 
     for (*reply = NULL; *reply == NULL; ) {
 
         *reply = pcmk__remote_message_xml(native->remote);
         if (*reply == NULL) {
             /* read some more off the tls buffer if we still have time left. */
             if (remaining_timeout) {
                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
             } else {
                 remaining_timeout = total_timeout;
             }
             if (remaining_timeout <= 0) {
                 return ETIME;
             }
 
             rc = pcmk__read_remote_message(native->remote, remaining_timeout);
             if (rc != pcmk_rc_ok) {
                 return rc;
             }
 
             *reply = pcmk__remote_message_xml(native->remote);
             if (*reply == NULL) {
                 return ENOMSG;
             }
         }
 
         crm_element_value_int(*reply, F_LRMD_REMOTE_MSG_ID, &reply_id);
         msg_type = crm_element_value(*reply, F_LRMD_REMOTE_MSG_TYPE);
 
         if (!msg_type) {
             crm_err("Empty msg type received while waiting for reply");
             free_xml(*reply);
             *reply = NULL;
         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
             /* got a notify while waiting for reply, trigger the notify to be processed later */
             crm_info("queueing notify");
             native->pending_notify = g_list_append(native->pending_notify, *reply);
             if (native->process_notify) {
                 crm_info("notify trigger set.");
                 mainloop_set_trigger(native->process_notify);
             }
             *reply = NULL;
         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
             /* msg isn't a reply, make some noise */
             crm_err("Expected a reply, got %s", msg_type);
             free_xml(*reply);
             *reply = NULL;
         } else if (reply_id != expected_reply_id) {
             if (native->expected_late_replies > 0) {
                 native->expected_late_replies--;
             } else {
                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
             }
             free_xml(*reply);
             *reply = NULL;
         }
     }
 
     if (native->remote->buffer && native->process_notify) {
         mainloop_set_trigger(native->process_notify);
     }
 
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
 {
     int rc = pcmk_rc_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     global_remote_msg_id++;
     if (global_remote_msg_id <= 0) {
         global_remote_msg_id = 1;
     }
 
     rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
                                "request");
     if (rc != pcmk_rc_ok) {
         crm_err("Disconnecting because TLS message could not be sent to "
                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
         lrmd_tls_disconnect(lrmd);
     }
     return rc;
 }
 
 static int
 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
 {
     int rc = 0;
     xmlNode *xml = NULL;
 
     if (!remote_executor_connected(lrmd)) {
         return -ENOTCONN;
     }
 
     rc = send_remote_message(lrmd, msg);
     if (rc != pcmk_rc_ok) {
         return pcmk_rc2legacy(rc);
     }
 
     rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
     if (rc != pcmk_rc_ok) {
         crm_err("Disconnecting remote after request %d reply not received: %s "
                 CRM_XS " rc=%d timeout=%dms",
                 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
         lrmd_tls_disconnect(lrmd);
     }
 
     if (reply) {
         *reply = xml;
     } else {
         free_xml(xml);
     }
 
     return pcmk_rc2legacy(rc);
 }
 #endif
 
 static int
 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case pcmk__client_ipc:
             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case pcmk__client_tls:
             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
             break;
 #endif
         default:
             crm_err("Unsupported executor connection type (bug?): %d",
                     native->type);
             rc = -EPROTONOSUPPORT;
     }
 
     return rc;
 }
 
 static int
 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case pcmk__client_ipc:
             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case pcmk__client_tls:
             rc = send_remote_message(lrmd, msg);
             if (rc == pcmk_rc_ok) {
                 /* we don't want to wait around for the reply, but
                  * since the request/reply protocol needs to behave the same
                  * as libqb, a reply will eventually come later anyway. */
                 native->expected_late_replies++;
             }
             rc = pcmk_rc2legacy(rc);
             break;
 #endif
         default:
             crm_err("Unsupported executor connection type (bug?): %d",
                     native->type);
             rc = -EPROTONOSUPPORT;
     }
 
     return rc;
 }
 
 static int
 lrmd_api_is_connected(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case pcmk__client_ipc:
             return crm_ipc_connected(native->ipc);
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case pcmk__client_tls:
             return remote_executor_connected(lrmd);
 #endif
         default:
             crm_err("Unsupported executor connection type (bug?): %d",
                     native->type);
             return 0;
     }
 }
 
 /*!
  * \internal
  * \brief Send a prepared API command to the executor
  *
  * \param[in]  lrmd          Existing connection to the executor
  * \param[in]  op            Name of API command to send
  * \param[in]  data          Command data XML to add to the sent command
  * \param[out] output_data   If expecting a reply, it will be stored here
  * \param[in]  timeout       Timeout in milliseconds (if 0, defaults to
  *                           a sensible value per the type of connection,
  *                           standard vs. pacemaker remote);
  *                           also propagated to the command XML
  * \param[in]  call_options  Call options to pass to server when sending
  * \param[in]  expect_reply  If TRUE, wait for a reply from the server;
  *                           must be TRUE for IPC (as opposed to TLS) clients
  *
  * \return pcmk_ok on success, -errno on error
  */
 static int
 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
                   xmlNode **output_data, int timeout,
                   enum lrmd_call_options options, gboolean expect_reply)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *op_msg = NULL;
     xmlNode *op_reply = NULL;
 
     if (!lrmd_api_is_connected(lrmd)) {
         return -ENOTCONN;
     }
 
     if (op == NULL) {
         crm_err("No operation specified");
         return -EINVAL;
     }
 
     CRM_CHECK(native->token != NULL,;
         );
     crm_trace("Sending %s op to executor", op);
 
     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 
     if (op_msg == NULL) {
         return -EINVAL;
     }
 
     if (expect_reply) {
         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
     } else {
         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
         goto done;
     }
 
     if (rc < 0) {
         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
         rc = -ECOMM;
         goto done;
 
     } else if(op_reply == NULL) {
         rc = -ENOMSG;
         goto done;
     }
 
     rc = pcmk_ok;
     crm_trace("%s op reply received", op);
     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
         rc = -ENOMSG;
         goto done;
     }
 
     crm_log_xml_trace(op_reply, "Reply");
 
     if (output_data) {
         *output_data = op_reply;
         op_reply = NULL;        /* Prevent subsequent free */
     }
 
   done:
     if (lrmd_api_is_connected(lrmd) == FALSE) {
         crm_err("Executor disconnected");
     }
 
     free_xml(op_msg);
     free_xml(op_reply);
     return rc;
 }
 
 static int
 lrmd_api_poke_connection(lrmd_t * lrmd)
 {
     int rc;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
                            (native->type == pcmk__client_ipc));
     free_xml(data);
 
     return rc < 0 ? rc : pcmk_ok;
 }
 
 // \return Standard Pacemaker return code
 int
 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
 {
     int rc = pcmk_rc_ok;
     const char *value;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 
     value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
     if ((value) &&
         (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
        crm_xml_add(data, F_LRMD_WATCHDOG, value);
     }
 
     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
                            (native->type == pcmk__client_ipc));
     free_xml(data);
     return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
 }
 
 static int
 lrmd_handshake(lrmd_t * lrmd, const char *name)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *reply = NULL;
     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 
     crm_xml_add(hello, F_TYPE, T_LRMD);
     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 
     /* advertise that we are a proxy provider */
     if (native->proxy_callback) {
         crm_xml_add(hello, F_LRMD_IS_IPC_PROVIDER, "true");
     }
 
     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 
     if (rc < 0) {
         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
         rc = -ECOMM;
     } else if (reply == NULL) {
         crm_err("Did not receive registration reply");
         rc = -EPROTO;
     } else {
         const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 
         crm_element_value_int(reply, F_LRMD_RC, &rc);
 
         if (rc == -EPROTO) {
             crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
                 LRMD_PROTOCOL_VERSION, version);
             crm_log_xml_err(reply, "Protocol Error");
 
         } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
             crm_err("Invalid registration message: %s", msg_type);
             crm_log_xml_err(reply, "Bad reply");
             rc = -EPROTO;
         } else if (tmp_ticket == NULL) {
             crm_err("No registration token provided");
             crm_log_xml_err(reply, "Bad reply");
             rc = -EPROTO;
         } else {
             crm_trace("Obtained registration token: %s", tmp_ticket);
             native->token = strdup(tmp_ticket);
             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
             rc = pcmk_ok;
         }
     }
 
     free_xml(reply);
     free_xml(hello);
 
     if (rc != pcmk_ok) {
         lrmd_api_disconnect(lrmd);
     }
     return rc;
 }
 
 static int
 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     struct ipc_client_callbacks lrmd_callbacks = {
         .dispatch = lrmd_ipc_dispatch,
         .destroy = lrmd_ipc_connection_destroy
     };
 
     crm_info("Connecting to executor");
 
     if (fd) {
         /* No mainloop */
         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
         if (native->ipc && crm_ipc_connect(native->ipc)) {
             *fd = crm_ipc_get_fd(native->ipc);
         } else if (native->ipc) {
             crm_perror(LOG_ERR, "Connection to executor failed");
             rc = -ENOTCONN;
         }
     } else {
         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
         native->ipc = mainloop_get_ipc_client(native->source);
     }
 
     if (native->ipc == NULL) {
         crm_debug("Could not connect to the executor API");
         rc = -ENOTCONN;
     }
 
     return rc;
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
 {
     dest->data = gnutls_malloc(source->size);
     CRM_ASSERT(dest->data);
     memcpy(dest->data, source->data, source->size);
     dest->size = source->size;
 }
 
 static void
 clear_gnutls_datum(gnutls_datum_t *datum)
 {
     gnutls_free(datum->data);
     datum->data = NULL;
     datum->size = 0;
 }
 
 #define KEY_READ_LEN 256    // Chunk size for reading key from file
 
 // \return Standard Pacemaker return code
 static int
 read_gnutls_key(const char *location, gnutls_datum_t *key)
 {
     FILE *stream = NULL;
     size_t buf_len = KEY_READ_LEN;
 
     if ((location == NULL) || (key == NULL)) {
         return EINVAL;
     }
 
     stream = fopen(location, "r");
     if (stream == NULL) {
         return errno;
     }
 
     key->data = gnutls_malloc(buf_len);
     key->size = 0;
     while (!feof(stream)) {
         int next = fgetc(stream);
 
         if (next == EOF) {
             if (!feof(stream)) {
                 crm_warn("Pacemaker Remote key read was partially successful "
                          "(copy in memory may be corrupted)");
             }
             break;
         }
         if (key->size == buf_len) {
             buf_len = key->size + KEY_READ_LEN;
             key->data = gnutls_realloc(key->data, buf_len);
             CRM_ASSERT(key->data);
         }
         key->data[key->size++] = (unsigned char) next;
     }
     fclose(stream);
 
     if (key->size == 0) {
         clear_gnutls_datum(key);
         return ENOKEY;
     }
     return pcmk_rc_ok;
 }
 
 // Cache the most recently used Pacemaker Remote authentication key
 
 struct key_cache_s {
     time_t updated;         // When cached key was read (valid for 1 minute)
     const char *location;   // Where cached key was read from
     gnutls_datum_t key;     // Cached key
 };
 
 static bool
 key_is_cached(struct key_cache_s *key_cache)
 {
     return key_cache->updated != 0;
 }
 
 static bool
 key_cache_expired(struct key_cache_s *key_cache)
 {
     return (time(NULL) - key_cache->updated) >= 60;
 }
 
 static void
 clear_key_cache(struct key_cache_s *key_cache)
 {
     clear_gnutls_datum(&(key_cache->key));
     if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
         key_cache->updated = 0;
         key_cache->location = NULL;
         crm_debug("Cleared Pacemaker Remote key cache");
     }
 }
 
 static void
 get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
 {
     copy_gnutls_datum(key, &(key_cache->key));
     crm_debug("Using cached Pacemaker Remote key from %s",
               crm_str(key_cache->location));
 }
 
 static void
 cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
           const char *location)
 {
     key_cache->updated = time(NULL);
     key_cache->location = location;
     copy_gnutls_datum(&(key_cache->key), key);
     crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
               crm_str(location));
 }
 
 /*!
  * \internal
  * \brief Get Pacemaker Remote authentication key from file or cache
  *
  * \param[in]  location         Path to key file to try (this memory must
  *                              persist across all calls of this function)
  * \param[out] key              Key from location or cache
  *
  * \return Standard Pacemaker return code
  */
 static int
 get_remote_key(const char *location, gnutls_datum_t *key)
 {
     static struct key_cache_s key_cache = { 0, };
     int rc = pcmk_rc_ok;
 
     if ((location == NULL) || (key == NULL)) {
         return EINVAL;
     }
 
     if (key_is_cached(&key_cache)) {
         if (key_cache_expired(&key_cache)) {
             clear_key_cache(&key_cache);
         } else {
             get_cached_key(&key_cache, key);
             return pcmk_rc_ok;
         }
     }
 
     rc = read_gnutls_key(location, key);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
     cache_key(&key_cache, key, location);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Initialize the Pacemaker Remote authentication key
  *
  * Try loading the Pacemaker Remote authentication key from cache if available,
  * otherwise from these locations, in order of preference: the value of the
  * PCMK_authkey_location environment variable, if set; the Pacemaker default key
  * file location; or (for historical reasons) /etc/corosync/authkey.
  *
  * \param[out] key  Where to store key
  *
  * \return Standard Pacemaker return code
  */
 int
 lrmd__init_remote_key(gnutls_datum_t *key)
 {
     static const char *env_location = NULL;
     static bool need_env = true;
 
     int env_rc = pcmk_rc_ok;
     int default_rc = pcmk_rc_ok;
     int alt_rc = pcmk_rc_ok;
 
     bool env_is_default = false;
     bool env_is_fallback = false;
 
     if (need_env) {
         env_location = getenv("PCMK_authkey_location");
         need_env = false;
     }
 
     // Try location in environment variable, if set
     if (env_location != NULL) {
         env_rc = get_remote_key(env_location, key);
         if (env_rc == pcmk_rc_ok) {
             return pcmk_rc_ok;
         }
 
         env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
         env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
 
         /* @TODO It would be more secure to fail, rather than fall back to the
          * default, if an explicitly set key location is not readable, and it
          * would be better to never use the Corosync location as a fallback.
          * However, that would break any deployments currently working with the
          * fallbacks.
          */
     }
 
     // Try default location, if environment wasn't explicitly set to it
     if (env_is_default) {
         default_rc = env_rc;
     } else {
         default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
     }
 
     // Try fallback location, if environment wasn't set to it and default failed
     if (env_is_fallback) {
         alt_rc = env_rc;
     } else if (default_rc != pcmk_rc_ok) {
         alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
     }
 
     // We have all results, so log and return
 
     if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
         && (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
 
         crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
                  env_location,
                  env_is_default? "" : "or default location ",
                  env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
                  !env_is_default && !env_is_fallback? " " : "",
                  env_is_fallback? "" : "or fallback location ",
                  env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
                  pcmk_rc_str(env_rc));
         return ENOKEY;
     }
 
     if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
         crm_warn("Could not read Pacemaker Remote key from %s "
                  "(using %s location %s instead): %s",
                  env_location,
                  (default_rc == pcmk_rc_ok)? "default" : "fallback",
                  (default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
                  pcmk_rc_str(env_rc));
         return pcmk_rc_ok;
     }
 
     if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
         // Environment unset, defaults failed
         crm_warn("Could not read Pacemaker Remote key from default location %s"
                  " (or fallback location %s): %s",
                  DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
                  pcmk_rc_str(default_rc));
         return ENOKEY;
     }
 
     return pcmk_rc_ok; // Environment variable unset, a default worked
 }
 
 static void
 lrmd_gnutls_global_init(void)
 {
     static int gnutls_init = 0;
 
     if (!gnutls_init) {
         crm_gnutls_global_init();
     }
     gnutls_init = 1;
 }
 #endif
 
 static void
 report_async_connection_result(lrmd_t * lrmd, int rc)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->callback) {
         lrmd_event_data_t event = { 0, };
         event.type = lrmd_event_connect;
         event.remote_nodename = native->remote_nodename;
         event.connection_rc = rc;
         native->callback(&event);
     }
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static inline int
 lrmd__tls_client_handshake(pcmk__remote_t *remote)
 {
     return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
 }
 
 /*!
  * \internal
  * \brief Add trigger and file descriptor mainloop sources for TLS
  *
  * \param[in] lrmd          API connection with established TLS session
  * \param[in] do_handshake  Whether to perform executor handshake
  *
  * \return Standard Pacemaker return code
  */
 static int
 add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
     int rc = pcmk_rc_ok;
 
     char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
                                    native->server, native->port);
 
     struct mainloop_fd_callbacks tls_fd_callbacks = {
         .dispatch = lrmd_tls_dispatch,
         .destroy = lrmd_tls_connection_destroy,
     };
 
     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
                                                   lrmd_tls_dispatch, lrmd);
     native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
                                      &tls_fd_callbacks);
 
     /* Async connections lose the client name provided by the API caller, so we
      * have to use our generated name here to perform the executor handshake.
      *
      * @TODO Keep track of the caller-provided name. Perhaps we should be using
      * that name in this function instead of generating one anyway.
      */
     if (do_handshake) {
         rc = lrmd_handshake(lrmd, name);
         rc = pcmk_legacy2rc(rc);
     }
     free(name);
     return rc;
 }
 
 static void
 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
     gnutls_datum_t psk_key = { NULL, 0 };
 
     native->async_timer = 0;
 
     if (rc != pcmk_rc_ok) {
         lrmd_tls_connection_destroy(lrmd);
         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
                  CRM_XS " rc=%d",
                  native->server, native->port, pcmk_rc_str(rc), rc);
         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
         return;
     }
 
     /* The TCP connection was successful, so establish the TLS connection.
      * @TODO make this async to avoid blocking code in client
      */
 
     native->sock = sock;
 
     rc = lrmd__init_remote_key(&psk_key);
     if (rc != pcmk_rc_ok) {
         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
                  CRM_XS " rc=%d",
                  native->server, native->port, pcmk_rc_str(rc), rc);
         lrmd_tls_connection_destroy(lrmd);
         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
         return;
     }
 
     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
     gnutls_free(psk_key.data);
 
     native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
                                                         GNUTLS_CRD_PSK,
                                                         native->psk_cred_c);
     if (native->remote->tls_session == NULL) {
         lrmd_tls_connection_destroy(lrmd);
         report_async_connection_result(lrmd, -EPROTO);
         return;
     }
 
     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
         crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
                  native->server, native->port);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
         native->remote->tls_session = NULL;
         lrmd_tls_connection_destroy(lrmd);
         report_async_connection_result(lrmd, -EKEYREJECTED);
         return;
     }
 
     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
              native->server, native->port);
     rc = add_tls_to_mainloop(lrmd, true);
     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
 }
 
 static int
 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
 {
     int rc;
     int timer_id = 0;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     lrmd_gnutls_global_init();
     native->sock = -1;
     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
     if (rc != pcmk_rc_ok) {
         crm_warn("Pacemaker Remote connection to %s:%s failed: %s "
                  CRM_XS " rc=%d",
                  native->server, native->port, pcmk_rc_str(rc), rc);
         return pcmk_rc2legacy(rc);
     }
     native->async_timer = timer_id;
     return pcmk_ok;
 }
 
 static int
 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
 {
     int rc;
 
     lrmd_private_t *native = lrmd->lrmd_private;
     gnutls_datum_t psk_key = { NULL, 0 };
 
     lrmd_gnutls_global_init();
 
     native->sock = -1;
     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
                               &(native->sock), NULL, NULL);
     if (rc != pcmk_rc_ok) {
         crm_warn("Pacemaker Remote connection to %s:%s failed: %s "
                  CRM_XS " rc=%d",
                  native->server, native->port, pcmk_rc_str(rc), rc);
         lrmd_tls_connection_destroy(lrmd);
         return -ENOTCONN;
     }
 
     rc = lrmd__init_remote_key(&psk_key);
     if (rc != pcmk_rc_ok) {
         lrmd_tls_connection_destroy(lrmd);
         return pcmk_rc2legacy(rc);
     }
 
     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
     gnutls_free(psk_key.data);
 
     native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
                                                         GNUTLS_CRD_PSK,
                                                         native->psk_cred_c);
     if (native->remote->tls_session == NULL) {
         lrmd_tls_connection_destroy(lrmd);
         return -EPROTO;
     }
 
     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
         crm_err("Session creation for %s:%d failed", native->server, native->port);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
         native->remote->tls_session = NULL;
         lrmd_tls_connection_destroy(lrmd);
         return -EKEYREJECTED;
     }
 
     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
              native->port);
 
     if (fd) {
         *fd = native->sock;
     } else {
         add_tls_to_mainloop(lrmd, false);
     }
     return pcmk_ok;
 }
 #endif
 
 static int
 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
 {
     int rc = -ENOTCONN;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case pcmk__client_ipc:
             rc = lrmd_ipc_connect(lrmd, fd);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case pcmk__client_tls:
             rc = lrmd_tls_connect(lrmd, fd);
             break;
 #endif
         default:
             crm_err("Unsupported executor connection type (bug?): %d",
                     native->type);
             rc = -EPROTONOSUPPORT;
     }
 
     if (rc == pcmk_ok) {
         rc = lrmd_handshake(lrmd, name);
     }
 
     return rc;
 }
 
 static int
 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     CRM_CHECK(native && native->callback, return -EINVAL);
 
     switch (native->type) {
         case pcmk__client_ipc:
             /* fake async connection with ipc.  it should be fast
              * enough that we gain very little from async */
             rc = lrmd_api_connect(lrmd, name, NULL);
             if (!rc) {
                 report_async_connection_result(lrmd, rc);
             }
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case pcmk__client_tls:
             rc = lrmd_tls_connect_async(lrmd, timeout);
             if (rc) {
                 /* connection failed, report rc now */
                 report_async_connection_result(lrmd, rc);
             }
             break;
 #endif
         default:
             crm_err("Unsupported executor connection type (bug?): %d",
                     native->type);
             rc = -EPROTONOSUPPORT;
     }
 
     return rc;
 }
 
 static void
 lrmd_ipc_disconnect(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->source != NULL) {
         /* Attached to mainloop */
         mainloop_del_ipc_client(native->source);
         native->source = NULL;
         native->ipc = NULL;
 
     } else if (native->ipc) {
         /* Not attached to mainloop */
         crm_ipc_t *ipc = native->ipc;
 
         native->ipc = NULL;
         crm_ipc_close(ipc);
         crm_ipc_destroy(ipc);
     }
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_tls_disconnect(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->remote->tls_session) {
         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
         native->remote->tls_session = 0;
     }
 
     if (native->async_timer) {
         g_source_remove(native->async_timer);
         native->async_timer = 0;
     }
 
     if (native->source != NULL) {
         /* Attached to mainloop */
         mainloop_del_ipc_client(native->source);
         native->source = NULL;
 
     } else if (native->sock) {
         close(native->sock);
         native->sock = 0;
     }
 
     if (native->pending_notify) {
         g_list_free_full(native->pending_notify, lrmd_free_xml);
         native->pending_notify = NULL;
     }
 }
 #endif
 
 static int
 lrmd_api_disconnect(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
     int rc = pcmk_ok;
 
     crm_info("Disconnecting %s %s executor connection",
              pcmk__client_type_str(native->type),
              (native->remote_nodename? native->remote_nodename : "local"));
     switch (native->type) {
         case pcmk__client_ipc:
             lrmd_ipc_disconnect(lrmd);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case pcmk__client_tls:
             lrmd_tls_disconnect(lrmd);
             break;
 #endif
         default:
             crm_err("Unsupported executor connection type (bug?): %d",
                     native->type);
             rc = -EPROTONOSUPPORT;
     }
 
     free(native->token);
     native->token = NULL;
 
     free(native->peer_version);
     native->peer_version = NULL;
     return rc;
 }
 
 static int
 lrmd_api_register_rsc(lrmd_t * lrmd,
                       const char *rsc_id,
                       const char *class,
                       const char *provider, const char *type, enum lrmd_call_options options)
 {
     int rc = pcmk_ok;
     xmlNode *data = NULL;
 
     if (!class || !type || !rsc_id) {
         return -EINVAL;
     }
     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
         && (provider == NULL)) {
         return -EINVAL;
     }
 
     data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     crm_xml_add(data, F_LRMD_CLASS, class);
     crm_xml_add(data, F_LRMD_PROVIDER, provider);
     crm_xml_add(data, F_LRMD_TYPE, type);
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
     free_xml(data);
 
     return rc;
 }
 
 static int
 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
     free_xml(data);
 
     return rc;
 }
 
 lrmd_rsc_info_t *
 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
                   const char *provider, const char *type)
 {
     lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
 
     CRM_ASSERT(rsc_info);
     if (rsc_id) {
         rsc_info->id = strdup(rsc_id);
         CRM_ASSERT(rsc_info->id);
     }
     if (standard) {
         rsc_info->standard = strdup(standard);
         CRM_ASSERT(rsc_info->standard);
     }
     if (provider) {
         rsc_info->provider = strdup(provider);
         CRM_ASSERT(rsc_info->provider);
     }
     if (type) {
         rsc_info->type = strdup(type);
         CRM_ASSERT(rsc_info->type);
     }
     return rsc_info;
 }
 
 lrmd_rsc_info_t *
 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
 {
     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
                              rsc_info->provider, rsc_info->type);
 }
 
 void
 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
 {
     if (!rsc_info) {
         return;
     }
     free(rsc_info->id);
     free(rsc_info->type);
     free(rsc_info->standard);
     free(rsc_info->provider);
     free(rsc_info);
 }
 
 static lrmd_rsc_info_t *
 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
 {
     lrmd_rsc_info_t *rsc_info = NULL;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
     xmlNode *output = NULL;
     const char *class = NULL;
     const char *provider = NULL;
     const char *type = NULL;
 
     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
     free_xml(data);
 
     if (!output) {
         return NULL;
     }
 
     class = crm_element_value(output, F_LRMD_CLASS);
     provider = crm_element_value(output, F_LRMD_PROVIDER);
     type = crm_element_value(output, F_LRMD_TYPE);
 
     if (!class || !type) {
         free_xml(output);
         return NULL;
     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
                && !provider) {
         free_xml(output);
         return NULL;
     }
 
     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
     free_xml(output);
     return rsc_info;
 }
 
 void
 lrmd_free_op_info(lrmd_op_info_t *op_info)
 {
     if (op_info) {
         free(op_info->rsc_id);
         free(op_info->action);
         free(op_info->interval_ms_s);
         free(op_info->timeout_ms_s);
         free(op_info);
     }
 }
 
 static int
 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
                            enum lrmd_call_options options, GList **output)
 {
     xmlNode *data = NULL;
     xmlNode *output_xml = NULL;
     int rc = pcmk_ok;
 
     if (output == NULL) {
         return -EINVAL;
     }
     *output = NULL;
 
     // Send request
     if (rsc_id) {
         data = create_xml_node(NULL, F_LRMD_RSC);
         crm_xml_add(data, F_LRMD_ORIGIN, __func__);
         crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     }
     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
                            timeout_ms, options, TRUE);
     if (data) {
         free_xml(data);
     }
 
     // Process reply
     if ((rc != pcmk_ok) || (output_xml == NULL)) {
         return rc;
     }
     for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
          (rsc_xml != NULL) && (rc == pcmk_ok);
          rsc_xml = crm_next_same_xml(rsc_xml)) {
 
         rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
         if (rsc_id == NULL) {
             crm_err("Could not parse recurring operation information from executor");
             continue;
         }
         for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
              op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {
 
             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
 
             if (op_info == NULL) {
                 rc = -ENOMEM;
                 break;
             }
             op_info->rsc_id = strdup(rsc_id);
             op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
             op_info->interval_ms_s = crm_element_value_copy(op_xml,
                                                             F_LRMD_RSC_INTERVAL);
             op_info->timeout_ms_s = crm_element_value_copy(op_xml,
                                                            F_LRMD_TIMEOUT);
             *output = g_list_prepend(*output, op_info);
         }
     }
     free_xml(output_xml);
     return rc;
 }
 
 
 static void
 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     native->callback = callback;
 }
 
 void
 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     native->proxy_callback = callback;
     native->proxy_callback_userdata = userdata;
 }
 
 void
 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->proxy_callback) {
         crm_log_xml_trace(msg, "PROXY_INBOUND");
         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
     }
 }
 
 int
 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
 {
     if (lrmd == NULL) {
         return -ENOTCONN;
     }
     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
 
     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
     return lrmd_send_xml_no_reply(lrmd, msg);
 }
 
 static int
 stonith_get_metadata(const char *provider, const char *type, char **output)
 {
     int rc = pcmk_ok;
     stonith_t *stonith_api = stonith_api_new();
 
     if (stonith_api == NULL) {
         crm_err("Could not get fence agent meta-data: API memory allocation failed");
         return -ENOMEM;
     }
 
     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
                                      provider, output, 0);
     if ((rc == pcmk_ok) && (*output == NULL)) {
         rc = -EIO;
     }
     stonith_api->cmds->free(stonith_api);
     return rc;
 }
 
 static int
 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
                       const char *type, char **output,
                       enum lrmd_call_options options)
 {
     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
                                            output, options, NULL);
 }
 
 static int
 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
                              const char *provider, const char *type,
                              char **output, enum lrmd_call_options options,
                              lrmd_key_value_t *params)
 {
     svc_action_t *action = NULL;
     GHashTable *params_table = NULL;
 
     if (!standard || !type) {
         lrmd_key_value_freeall(params);
         return -EINVAL;
     }
 
     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         lrmd_key_value_freeall(params);
         return stonith_get_metadata(provider, type, output);
     }
 
     params_table = pcmk__strkey_table(free, free);
     for (const lrmd_key_value_t *param = params; param; param = param->next) {
         g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
     }
-    action = resources_action_create(type, standard, provider, type,
-                                     CRMD_ACTION_METADATA, 0,
-                                     CRMD_METADATA_CALL_TIMEOUT, params_table,
-                                     0);
+    action = services__create_resource_action(type, standard, provider, type,
+                                              CRMD_ACTION_METADATA, 0,
+                                              CRMD_METADATA_CALL_TIMEOUT,
+                                              params_table, 0);
     lrmd_key_value_freeall(params);
 
     if (action == NULL) {
-        crm_err("Unable to retrieve meta-data for %s:%s:%s",
-                standard, provider, type);
+        return -ENOMEM;
+    }
+    if (action->rc != PCMK_OCF_OK) {
+        services_action_free(action);
         return -EINVAL;
     }
 
     if (!services_action_sync(action)) {
         crm_err("Failed to retrieve meta-data for %s:%s:%s",
                 standard, provider, type);
         services_action_free(action);
         return -EIO;
     }
 
     if (!action->stdout_data) {
         crm_err("Failed to receive meta-data for %s:%s:%s",
                 standard, provider, type);
         services_action_free(action);
         return -EIO;
     }
 
     *output = strdup(action->stdout_data);
     services_action_free(action);
 
     return pcmk_ok;
 }
 
 static int
 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
               const char *userdata, guint interval_ms,
               int timeout,      /* ms */
               int start_delay,  /* ms */
               enum lrmd_call_options options, lrmd_key_value_t * params)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
     lrmd_key_value_t *tmp = NULL;
 
     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
 
     for (tmp = params; tmp; tmp = tmp->next) {
         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
     }
 
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
     free_xml(data);
 
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 /* timeout is in ms */
 static int
 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
                     int timeout, lrmd_key_value_t *params)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
     lrmd_key_value_t *tmp = NULL;
 
     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
     crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
     crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
 
     for (tmp = params; tmp; tmp = tmp->next) {
         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
     }
 
     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
                            lrmd_opt_notify_orig_only, TRUE);
     free_xml(data);
 
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 static int
 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
                 guint interval_ms)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
     free_xml(data);
     return rc;
 }
 
 static int
 list_stonith_agents(lrmd_list_t ** resources)
 {
     int rc = 0;
     stonith_t *stonith_api = stonith_api_new();
     stonith_key_value_t *stonith_resources = NULL;
     stonith_key_value_t *dIter = NULL;
 
     if (stonith_api == NULL) {
         crm_err("Could not list fence agents: API memory allocation failed");
         return -ENOMEM;
     }
     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
                                    &stonith_resources, 0);
     stonith_api->cmds->free(stonith_api);
 
     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
         rc++;
         if (resources) {
             *resources = lrmd_list_add(*resources, dIter->value);
         }
     }
 
     stonith_key_value_freeall(stonith_resources, 1, 0);
     return rc;
 }
 
 static int
 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
                      const char *provider)
 {
     int rc = 0;
     int stonith_count = 0; // Initially, whether to include stonith devices
 
     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         stonith_count = 1;
 
     } else {
         GList *gIter = NULL;
         GList *agents = resources_list_agents(class, provider);
 
         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
             rc++;
         }
         g_list_free_full(agents, free);
 
         if (!class) {
             stonith_count = 1;
         }
     }
 
     if (stonith_count) {
         // Now, if stonith devices are included, how many there are
         stonith_count = list_stonith_agents(resources);
         if (stonith_count > 0) {
             rc += stonith_count;
         }
     }
     if (rc == 0) {
         crm_notice("No agents found for class %s", class);
         rc = -EPROTONOSUPPORT;
     }
     return rc;
 }
 
 static bool
 does_provider_have_agent(const char *agent, const char *provider, const char *class)
 {
     bool found = false;
     GList *agents = NULL;
     GList *gIter2 = NULL;
 
     agents = resources_list_agents(class, provider);
     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
             found = true;
         }
     }
     g_list_free_full(agents, free);
     return found;
 }
 
 static int
 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
 {
     int rc = pcmk_ok;
     char *provider = NULL;
     GList *ocf_providers = NULL;
     GList *gIter = NULL;
 
     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
 
     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
         provider = gIter->data;
         if (!agent || does_provider_have_agent(agent, provider,
                                                PCMK_RESOURCE_CLASS_OCF)) {
             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
             rc++;
         }
     }
 
     g_list_free_full(ocf_providers, free);
     return rc;
 }
 
 static int
 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
 {
     int rc = 0;
     GList *standards = NULL;
     GList *gIter = NULL;
 
     standards = resources_list_standards();
 
     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
         rc++;
     }
 
     if (list_stonith_agents(NULL) > 0) {
         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
         rc++;
     }
 
     g_list_free_full(standards, free);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Create an executor API object
  *
  * \param[out] api       Will be set to newly created API object (it is the
  *                       caller's responsibility to free this value with
  *                       lrmd_api_delete() if this function succeeds)
  * \param[in]  nodename  If the object will be used for a remote connection,
  *                       the node name to use in cluster for remote executor
  * \param[in]  server    If the object will be used for a remote connection,
  *                       the resolvable host name to connect to
  * \param[in]  port      If the object will be used for a remote connection,
  *                       port number on \p server to connect to
  *
  * \return Standard Pacemaker return code
  * \note If the caller leaves one of \p nodename or \p server NULL, the other's
  *       value will be used for both. If the caller leaves both NULL, an API
  *       object will be created for a local executor connection.
  */
 int
 lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
 {
     lrmd_private_t *pvt = NULL;
 
     if (api == NULL) {
         return EINVAL;
     }
     *api = NULL;
 
     // Allocate all memory needed
 
     *api = calloc(1, sizeof(lrmd_t));
     if (*api == NULL) {
         return ENOMEM;
     }
 
     pvt = calloc(1, sizeof(lrmd_private_t));
     if (pvt == NULL) {
         lrmd_api_delete(*api);
         *api = NULL;
         return ENOMEM;
     }
     (*api)->lrmd_private = pvt;
 
     // @TODO Do we need to do this for local connections?
     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
 
     (*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
 
     if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
         lrmd_api_delete(*api);
         *api = NULL;
         return ENOMEM;
     }
 
     // Set methods
     (*api)->cmds->connect = lrmd_api_connect;
     (*api)->cmds->connect_async = lrmd_api_connect_async;
     (*api)->cmds->is_connected = lrmd_api_is_connected;
     (*api)->cmds->poke_connection = lrmd_api_poke_connection;
     (*api)->cmds->disconnect = lrmd_api_disconnect;
     (*api)->cmds->register_rsc = lrmd_api_register_rsc;
     (*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
     (*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
     (*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
     (*api)->cmds->set_callback = lrmd_api_set_callback;
     (*api)->cmds->get_metadata = lrmd_api_get_metadata;
     (*api)->cmds->exec = lrmd_api_exec;
     (*api)->cmds->cancel = lrmd_api_cancel;
     (*api)->cmds->list_agents = lrmd_api_list_agents;
     (*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
     (*api)->cmds->list_standards = lrmd_api_list_standards;
     (*api)->cmds->exec_alert = lrmd_api_exec_alert;
     (*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
 
     if ((nodename == NULL) && (server == NULL)) {
         pvt->type = pcmk__client_ipc;
     } else {
 #ifdef HAVE_GNUTLS_GNUTLS_H
         if (nodename == NULL) {
             nodename = server;
         } else if (server == NULL) {
             server = nodename;
         }
         pvt->type = pcmk__client_tls;
         pvt->remote_nodename = strdup(nodename);
         pvt->server = strdup(server);
         if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
             lrmd_api_delete(*api);
             *api = NULL;
             return ENOMEM;
         }
         pvt->port = port;
         if (pvt->port == 0) {
             pvt->port = crm_default_remote_port();
         }
 #else
         crm_err("Cannot communicate with Pacemaker Remote "
                 "because GnuTLS is not enabled for this build");
         lrmd_api_delete(*api);
         *api = NULL;
         return EOPNOTSUPP;
 #endif
     }
     return pcmk_rc_ok;
 }
 
 lrmd_t *
 lrmd_api_new(void)
 {
     lrmd_t *api = NULL;
 
     CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
     return api;
 }
 
 lrmd_t *
 lrmd_remote_api_new(const char *nodename, const char *server, int port)
 {
     lrmd_t *api = NULL;
 
     CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
     return api;
 }
 
 void
 lrmd_api_delete(lrmd_t * lrmd)
 {
     if (!lrmd) {
         return;
     }
     lrmd->cmds->disconnect(lrmd);       /* no-op if already disconnected */
     free(lrmd->cmds);
     if (lrmd->lrmd_private) {
         lrmd_private_t *native = lrmd->lrmd_private;
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
         free(native->server);
 #endif
         free(native->remote_nodename);
         free(native->remote);
         free(native->token);
         free(native->peer_version);
     }
 
     free(lrmd->lrmd_private);
     free(lrmd);
 }
diff --git a/lib/services/services.c b/lib/services/services.c
index 670d4f531f..7409953c4d 100644
--- a/lib/services/services.c
+++ b/lib/services/services.c
@@ -1,1157 +1,1141 @@
 /*
  * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <stdio.h>
 #include <errno.h>
 #include <unistd.h>
 #include <dirent.h>
 #include <fcntl.h>
 
 #include <crm/crm.h>
 #include <crm/common/mainloop.h>
 #include <crm/services.h>
 #include <crm/services_internal.h>
 #include <crm/stonith-ng.h>
 #include <crm/msg_xml.h>
 #include "services_private.h"
 #include "services_lsb.h"
 
 #if SUPPORT_UPSTART
 #  include <upstart.h>
 #endif
 
 #if SUPPORT_SYSTEMD
 #  include <systemd.h>
 #endif
 
 #if SUPPORT_NAGIOS
 #  include <services_nagios.h>
 #endif
 
 /* TODO: Develop a rollover strategy */
 
 static int operations = 0;
 static GHashTable *recurring_actions = NULL;
 
 /* ops waiting to run async because of conflicting active
  * pending ops */
 static GList *blocked_ops = NULL;
 
 /* ops currently active (in-flight) */
 static GList *inflight_ops = NULL;
 
 static void handle_blocked_ops(void);
 
 /*!
  * \brief Find first service class that can provide a specified agent
  *
  * \param[in] agent  Name of agent to search for
  *
  * \return Service class if found, NULL otherwise
  *
  * \note The priority is LSB, then systemd, then upstart. It would be preferable
  *       to put systemd first, but LSB merely requires a file existence check,
  *       while systemd requires contacting D-Bus.
  */
 const char *
 resources_find_service_class(const char *agent)
 {
     if (services__lsb_agent_exists(agent)) {
         return PCMK_RESOURCE_CLASS_LSB;
     }
 
 #if SUPPORT_SYSTEMD
     if (systemd_unit_exists(agent)) {
         return PCMK_RESOURCE_CLASS_SYSTEMD;
     }
 #endif
 
 #if SUPPORT_UPSTART
     if (upstart_job_exists(agent)) {
         return PCMK_RESOURCE_CLASS_UPSTART;
     }
 #endif
     return NULL;
 }
 
 static inline void
 init_recurring_actions(void)
 {
     if (recurring_actions == NULL) {
         recurring_actions = pcmk__strkey_table(NULL, NULL);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether op is in-flight systemd or upstart op
  *
  * \param[in] op  Operation to check
  *
  * \return TRUE if op is in-flight systemd or upstart op
  */
 static inline gboolean
 inflight_systemd_or_upstart(svc_action_t *op)
 {
     return pcmk__strcase_any_of(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD,
                            PCMK_RESOURCE_CLASS_UPSTART, NULL) &&
            g_list_find(inflight_ops, op) != NULL;
 }
 
 /*!
  * \internal
  * \brief Expand "service" alias to an actual resource class
  *
  * \param[in] rsc       Resource name (for logging only)
  * \param[in] standard  Resource class as configured
  * \param[in] agent     Agent name to look for
  *
  * \return Newly allocated string with actual resource class
  *
  * \note The caller is responsible for calling free() on the result.
  */
 static char *
 expand_resource_class(const char *rsc, const char *standard, const char *agent)
 {
     char *expanded_class = NULL;
 
     if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0) {
         const char *found_class = resources_find_service_class(agent);
 
         if (found_class) {
             crm_debug("Found %s agent %s for %s", found_class, agent, rsc);
             expanded_class = strdup(found_class);
         } else {
             crm_info("Assuming resource class lsb for agent %s for %s",
                      agent, rsc);
             expanded_class = strdup(PCMK_RESOURCE_CLASS_LSB);
         }
     } else {
         expanded_class = strdup(standard);
     }
     CRM_ASSERT(expanded_class);
     return expanded_class;
 }
 
-#if SUPPORT_NAGIOS
-/*!
- * \brief Duplicate a file path, inserting a prefix if not absolute
- *
- * \param[in] filename  File path to duplicate
- * \param[in] dirname   If filename is not absolute, prefix to add
- *
- * \return Newly allocated memory with full path
- */
-static char *
-dup_file_path(const char *filename, const char *dirname)
-{
-    return (*filename == '/')? strdup(filename)
-           : crm_strdup_printf("%s/%s", dirname, filename);
-}
-#endif
 
 svc_action_t *
 services__create_resource_action(const char *name, const char *standard,
                         const char *provider, const char *agent,
                         const char *action, guint interval_ms, int timeout,
                         GHashTable *params, enum svc_action_flags flags)
 {
     svc_action_t *op = NULL;
     uint32_t ra_caps = 0;
 
     /*
      * Do some up front sanity checks before we go off and
      * build the svc_action_t instance.
      */
 
     if (pcmk__str_empty(name)) {
         crm_err("Cannot create operation without resource name");
         goto return_error;
     }
 
     if (pcmk__str_empty(standard)) {
         crm_err("Cannot create operation for %s without resource class", name);
         goto return_error;
     }
     ra_caps = pcmk_get_ra_caps(standard);
 
     if (pcmk_is_set(ra_caps, pcmk_ra_cap_provider)
         && pcmk__str_empty(provider)) {
         crm_err("Cannot create operation for %s without provider", name);
         goto return_error;
     }
 
     if (pcmk__str_empty(agent)) {
         crm_err("Cannot create operation for %s without agent name", name);
         goto return_error;
     }
 
     if (pcmk__str_empty(action)) {
         crm_err("Cannot create operation for %s without operation name", name);
         goto return_error;
     }
 
     /*
      * Sanity checks passed, proceed!
      */
 
     op = calloc(1, sizeof(svc_action_t));
     op->opaque = calloc(1, sizeof(svc_action_private_t));
     op->rsc = strdup(name);
     op->interval_ms = interval_ms;
     op->timeout = timeout;
     op->standard = expand_resource_class(name, standard, agent);
     op->agent = strdup(agent);
     op->sequence = ++operations;
     op->flags = flags;
     op->id = pcmk__op_key(name, action, interval_ms);
 
     if (pcmk_is_set(ra_caps, pcmk_ra_cap_status)
         && pcmk__str_eq(action, "monitor", pcmk__str_casei)) {
 
         op->action = strdup("status");
     } else {
         op->action = strdup(action);
     }
 
     if (pcmk_is_set(ra_caps, pcmk_ra_cap_provider)) {
         op->provider = strdup(provider);
     }
 
     if (pcmk_is_set(ra_caps, pcmk_ra_cap_params)) {
         op->params = params;
         params = NULL; // so we don't free them in this function
     }
 
     if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
         char *dirs = strdup(OCF_RA_PATH);
         char *dir = NULL;
         char *buf = NULL;
         struct stat st;
 
         if (pcmk__str_empty(dirs)) {
             free(dirs);
             services__handle_exec_error(op, ENOMEM);
             return op;
         }
 
         for (dir = strtok(dirs, ":"); dir != NULL; dir = strtok(NULL, ":")) {
             buf = crm_strdup_printf("%s/%s/%s", dir, provider, agent);
             if (stat(buf, &st) == 0) {
                 break;
             }
             free(buf);
             buf = NULL;
         }
 
         free(dirs);
 
         if (buf) {
             op->opaque->exec = buf;
         } else {
             services__handle_exec_error(op, ENOENT);
             return op;
         }
 
         op->opaque->args[0] = strdup(op->opaque->exec);
         op->opaque->args[1] = strdup(op->action);
 
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
-        op->opaque->exec = services__lsb_agent_path(op->agent);
+        op->opaque->exec = pcmk__full_path(op->agent, LSB_ROOT_DIR);
         op->opaque->args[0] = strdup(op->opaque->exec);
         op->opaque->args[1] = strdup(op->action);
 
 #if SUPPORT_SYSTEMD
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
         op->opaque->exec = strdup("systemd-dbus");
 #endif
 #if SUPPORT_UPSTART
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
         op->opaque->exec = strdup("upstart-dbus");
 #endif
 #if SUPPORT_NAGIOS
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
-        op->opaque->exec = dup_file_path(op->agent, NAGIOS_PLUGIN_DIR);
+        op->opaque->exec = pcmk__full_path(op->agent, NAGIOS_PLUGIN_DIR);
         op->opaque->args[0] = strdup(op->opaque->exec);
 
         if (pcmk__str_eq(op->action, "monitor", pcmk__str_casei) && (op->interval_ms == 0)) {
             /* Invoke --version for a nagios probe */
             op->opaque->args[1] = strdup("--version");
 
         } else if (op->params) {
             GHashTableIter iter;
             char *key = NULL;
             char *value = NULL;
             int index = 1;
             static int args_size = sizeof(op->opaque->args) / sizeof(char *);
 
             g_hash_table_iter_init(&iter, op->params);
 
             while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value) &&
                    index <= args_size - 3) {
 
                 if (pcmk__str_eq(key, XML_ATTR_CRM_VERSION, pcmk__str_casei) || strstr(key, CRM_META "_")) {
                     continue;
                 }
                 op->opaque->args[index++] = crm_strdup_printf("--%s", key);
                 op->opaque->args[index++] = strdup(value);
             }
         }
 
         // Nagios actions don't need to keep the parameters
         if (op->params != NULL) {
             g_hash_table_destroy(op->params);
             op->params = NULL;
         }
 #endif
     } else {
         crm_err("Unknown resource standard: %s", op->standard);
         services__handle_exec_error(op, ENOENT);
     }
 
   return_error:
     if(params) {
         g_hash_table_destroy(params);
     }
 
     return op;
 }
 
 svc_action_t *
 resources_action_create(const char *name, const char *standard,
                         const char *provider, const char *agent,
                         const char *action, guint interval_ms, int timeout,
                         GHashTable *params, enum svc_action_flags flags)
 {
     svc_action_t *op = services__create_resource_action(name, standard,
                             provider, agent, action, interval_ms, timeout,
                             params, flags);
     if (op == NULL || op->rc != 0) {
         services_action_free(op);
         return NULL;
     } else {
         return op;
     }
 }
 
 svc_action_t *
 services_action_create_generic(const char *exec, const char *args[])
 {
     svc_action_t *op;
     unsigned int cur_arg;
 
     op = calloc(1, sizeof(*op));
     op->opaque = calloc(1, sizeof(svc_action_private_t));
 
     op->opaque->exec = strdup(exec);
     op->opaque->args[0] = strdup(exec);
 
     for (cur_arg = 1; args && args[cur_arg - 1]; cur_arg++) {
         op->opaque->args[cur_arg] = strdup(args[cur_arg - 1]);
 
         if (cur_arg == PCMK__NELEM(op->opaque->args) - 1) {
             crm_err("svc_action_t args list not long enough for '%s' execution request.", exec);
             break;
         }
     }
 
     return op;
 }
 
 /*!
  * \brief Create an alert agent action
  *
  * \param[in] id        Alert ID
  * \param[in] exec      Path to alert agent executable
  * \param[in] timeout   Action timeout
  * \param[in] params    Parameters to use with action
  * \param[in] sequence  Action sequence number
  * \param[in] cb_data   Data to pass to callback function
  *
  * \return New action on success, NULL on error
  * \note It is the caller's responsibility to free cb_data.
  *       The caller should not free params explicitly.
  */
 svc_action_t *
 services_alert_create(const char *id, const char *exec, int timeout,
                       GHashTable *params, int sequence, void *cb_data)
 {
     svc_action_t *action = services_action_create_generic(exec, NULL);
 
     CRM_ASSERT(action);
     action->timeout = timeout;
     action->id = strdup(id);
     action->params = params;
     action->sequence = sequence;
     action->cb_data = cb_data;
     return action;
 }
 
 /*!
  * \brief Set the user and group that an action will execute as
  *
  * \param[in,out] action  Action to modify
  * \param[in]     user    Name of user to execute action as
  * \param[in]     group   Name of group to execute action as
  *
  * \return pcmk_ok on success, -errno otherwise
  *
  * \note This will have no effect unless the process executing the action runs
  *       as root, and the action is not a systemd or upstart action.
  *       We could implement this for systemd by adding User= and Group= to
  *       [Service] in the override file, but that seems more likely to cause
  *       problems than be useful.
  */
 int
 services_action_user(svc_action_t *op, const char *user)
 {
     CRM_CHECK((op != NULL) && (user != NULL), return -EINVAL);
     return crm_user_lookup(user, &(op->opaque->uid), &(op->opaque->gid));
 }
 
 /*!
  * \brief Execute an alert agent action
  *
  * \param[in] action  Action to execute
  * \param[in] cb      Function to call when action completes
  *
  * \return TRUE if the library will free action, FALSE otherwise
  *
  * \note If this function returns FALSE, it is the caller's responsibility to
  *       free the action with services_action_free().
  */
 gboolean
 services_alert_async(svc_action_t *action, void (*cb)(svc_action_t *op))
 {
     action->synchronous = false;
     action->opaque->callback = cb;
     return services_os_action_execute(action);
 }
 
 #if SUPPORT_DBUS
 /*!
  * \internal
  * \brief Update operation's pending DBus call, unreferencing old one if needed
  *
  * \param[in,out] op       Operation to modify
  * \param[in]     pending  Pending call to set
  */
 void
 services_set_op_pending(svc_action_t *op, DBusPendingCall *pending)
 {
     if (op->opaque->pending && (op->opaque->pending != pending)) {
         if (pending) {
             crm_info("Lost pending %s DBus call (%p)", op->id, op->opaque->pending);
         } else {
             crm_trace("Done with pending %s DBus call (%p)", op->id, op->opaque->pending);
         }
         dbus_pending_call_unref(op->opaque->pending);
     }
     op->opaque->pending = pending;
     if (pending) {
         crm_trace("Updated pending %s DBus call (%p)", op->id, pending);
     } else {
         crm_trace("Cleared pending %s DBus call", op->id);
     }
 }
 #endif
 
 void
 services_action_cleanup(svc_action_t * op)
 {
     if ((op == NULL) || (op->opaque == NULL)) {
         return;
     }
 
 #if SUPPORT_DBUS
     if(op->opaque->timerid != 0) {
         crm_trace("Removing timer for call %s to %s", op->action, op->rsc);
         g_source_remove(op->opaque->timerid);
         op->opaque->timerid = 0;
     }
 
     if(op->opaque->pending) {
         if (dbus_pending_call_get_completed(op->opaque->pending)) {
             // This should never be the case
             crm_warn("Result of %s op %s was unhandled",
                      op->standard, op->id);
         } else {
             crm_debug("Will ignore any result of canceled %s op %s",
                       op->standard, op->id);
         }
         dbus_pending_call_cancel(op->opaque->pending);
         services_set_op_pending(op, NULL);
     }
 #endif
 
     if (op->opaque->stderr_gsource) {
         mainloop_del_fd(op->opaque->stderr_gsource);
         op->opaque->stderr_gsource = NULL;
     }
 
     if (op->opaque->stdout_gsource) {
         mainloop_del_fd(op->opaque->stdout_gsource);
         op->opaque->stdout_gsource = NULL;
     }
 }
 
 void
 services_action_free(svc_action_t * op)
 {
     unsigned int i;
 
     if (op == NULL) {
         return;
     }
 
     /* The operation should be removed from all tracking lists by this point.
      * If it's not, we have a bug somewhere, so bail. That may lead to a
      * memory leak, but it's better than a use-after-free segmentation fault.
      */
     CRM_CHECK(g_list_find(inflight_ops, op) == NULL, return);
     CRM_CHECK(g_list_find(blocked_ops, op) == NULL, return);
     CRM_CHECK((recurring_actions == NULL)
               || (g_hash_table_lookup(recurring_actions, op->id) == NULL),
               return);
 
     services_action_cleanup(op);
 
     if (op->opaque->repeat_timer) {
         g_source_remove(op->opaque->repeat_timer);
         op->opaque->repeat_timer = 0;
     }
 
     free(op->id);
     free(op->opaque->exec);
 
     for (i = 0; i < PCMK__NELEM(op->opaque->args); i++) {
         free(op->opaque->args[i]);
     }
 
     free(op->opaque);
     free(op->rsc);
     free(op->action);
 
     free(op->standard);
     free(op->agent);
     free(op->provider);
 
     free(op->stdout_data);
     free(op->stderr_data);
 
     if (op->params) {
         g_hash_table_destroy(op->params);
         op->params = NULL;
     }
 
     free(op);
 }
 
 gboolean
 cancel_recurring_action(svc_action_t * op)
 {
     crm_info("Cancelling %s operation %s", op->standard, op->id);
 
     if (recurring_actions) {
         g_hash_table_remove(recurring_actions, op->id);
     }
 
     if (op->opaque->repeat_timer) {
         g_source_remove(op->opaque->repeat_timer);
         op->opaque->repeat_timer = 0;
     }
 
     return TRUE;
 }
 
 /*!
  * \brief Cancel a recurring action
  *
  * \param[in] name         Name of resource that operation is for
  * \param[in] action       Name of operation to cancel
  * \param[in] interval_ms  Interval of operation to cancel
  *
  * \return TRUE if action was successfully cancelled, FALSE otherwise
  */
 gboolean
 services_action_cancel(const char *name, const char *action, guint interval_ms)
 {
     gboolean cancelled = FALSE;
     char *id = pcmk__op_key(name, action, interval_ms);
     svc_action_t *op = NULL;
 
     /* We can only cancel a recurring action */
     init_recurring_actions();
     op = g_hash_table_lookup(recurring_actions, id);
     if (op == NULL) {
         goto done;
     }
 
     /* Tell operation_finalize() not to reschedule the operation */
     op->cancel = TRUE;
 
     /* Stop tracking it as a recurring operation, and stop its repeat timer */
     cancel_recurring_action(op);
 
     /* If the op has a PID, it's an in-flight child process, so kill it.
      *
      * Whether the kill succeeds or fails, the main loop will send the op to
      * operation_finished() (and thus operation_finalize()) when the process
      * goes away.
      */
     if (op->pid != 0) {
         crm_info("Terminating in-flight op %s[%d] early because it was cancelled",
                  id, op->pid);
         cancelled = mainloop_child_kill(op->pid);
         if (cancelled == FALSE) {
             crm_err("Termination of %s[%d] failed", id, op->pid);
         }
         goto done;
     }
 
 #if SUPPORT_DBUS
     // In-flight systemd and upstart ops don't have a pid
     if (inflight_systemd_or_upstart(op)) {
         inflight_ops = g_list_remove(inflight_ops, op);
 
         /* This will cause any result that comes in later to be discarded, so we
          * don't call the callback and free the operation twice.
          */
         services_action_cleanup(op);
     }
 #endif
 
     // The rest of this is essentially equivalent to operation_finalize(),
     // except without calling handle_blocked_ops()
 
     // Report operation as cancelled
     op->status = PCMK_EXEC_CANCELLED;
     if (op->opaque->callback) {
         op->opaque->callback(op);
     }
 
     blocked_ops = g_list_remove(blocked_ops, op);
     services_action_free(op);
     cancelled = TRUE;
     // @TODO Initiate handle_blocked_ops() asynchronously
 
 done:
     free(id);
     return cancelled;
 }
 
 gboolean
 services_action_kick(const char *name, const char *action, guint interval_ms)
 {
     svc_action_t * op = NULL;
     char *id = pcmk__op_key(name, action, interval_ms);
 
     init_recurring_actions();
     op = g_hash_table_lookup(recurring_actions, id);
     free(id);
 
     if (op == NULL) {
         return FALSE;
     }
 
 
     if (op->pid || inflight_systemd_or_upstart(op)) {
         return TRUE;
     } else {
         if (op->opaque->repeat_timer) {
             g_source_remove(op->opaque->repeat_timer);
             op->opaque->repeat_timer = 0;
         }
         recurring_action_timer(op);
         return TRUE;
     }
 
 }
 
 /*!
  * \internal
  * \brief Add a new recurring operation, checking for duplicates
  *
  * \param[in] op               Operation to add
  *
  * \return TRUE if duplicate found (and reschedule), FALSE otherwise
  */
 static gboolean
 handle_duplicate_recurring(svc_action_t * op)
 {
     svc_action_t * dup = NULL;
 
     /* check for duplicates */
     dup = g_hash_table_lookup(recurring_actions, op->id);
 
     if (dup && (dup != op)) {
         /* update user data */
         if (op->opaque->callback) {
             dup->opaque->callback = op->opaque->callback;
             dup->cb_data = op->cb_data;
             op->cb_data = NULL;
         }
         /* immediately execute the next interval */
         if (dup->pid != 0) {
             if (op->opaque->repeat_timer) {
                 g_source_remove(op->opaque->repeat_timer);
                 op->opaque->repeat_timer = 0;
             }
             recurring_action_timer(dup);
         }
         /* free the duplicate */
         services_action_free(op);
         return TRUE;
     }
 
     return FALSE;
 }
 
 inline static gboolean
 action_exec_helper(svc_action_t * op)
 {
     /* Whether a/synchronous must be decided (op->synchronous) beforehand. */
     if (op->standard
         && (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_UPSTART) == 0)) {
 #if SUPPORT_UPSTART
         return upstart_job_exec(op);
 #endif
     } else if (op->standard && strcasecmp(op->standard,
                                           PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
 #if SUPPORT_SYSTEMD
         return systemd_unit_exec(op);
 #endif
     } else {
         return services_os_action_execute(op);
     }
     /* The 'op' has probably been freed if the execution functions return TRUE
        for the asynchronous 'op'. */
     /* Avoid using the 'op' in here. */
 
     return FALSE;
 }
 
 void
 services_add_inflight_op(svc_action_t * op)
 {
     if (op == NULL) {
         return;
     }
 
     CRM_ASSERT(op->synchronous == FALSE);
 
     /* keep track of ops that are in-flight to avoid collisions in the same namespace */
     if (op->rsc) {
         inflight_ops = g_list_append(inflight_ops, op);
     }
 }
 
 /*!
  * \internal
  * \brief Stop tracking an operation that completed
  *
  * \param[in] op  Operation to stop tracking
  */
 void
 services_untrack_op(svc_action_t *op)
 {
     /* Op is no longer in-flight or blocked */
     inflight_ops = g_list_remove(inflight_ops, op);
     blocked_ops = g_list_remove(blocked_ops, op);
 
     /* Op is no longer blocking other ops, so check if any need to run */
     handle_blocked_ops();
 }
 
 gboolean
 services_action_async_fork_notify(svc_action_t * op,
                                   void (*action_callback) (svc_action_t *),
                                   void (*action_fork_callback) (svc_action_t *))
 {
     op->synchronous = false;
     if (action_callback) {
         op->opaque->callback = action_callback;
     }
     if (action_fork_callback) {
         op->opaque->fork_callback = action_fork_callback;
     }
 
     if (op->interval_ms > 0) {
         init_recurring_actions();
         if (handle_duplicate_recurring(op) == TRUE) {
             /* entry rescheduled, dup freed */
             /* exit early */
             return TRUE;
         }
         g_hash_table_replace(recurring_actions, op->id, op);
     }
 
     if (!pcmk_is_set(op->flags, SVC_ACTION_NON_BLOCKED)
         && op->rsc && is_op_blocked(op->rsc)) {
         blocked_ops = g_list_append(blocked_ops, op);
         return TRUE;
     }
 
     return action_exec_helper(op);
 }
 
 gboolean
 services_action_async(svc_action_t * op,
                       void (*action_callback) (svc_action_t *))
 {
     return services_action_async_fork_notify(op, action_callback, NULL);
 }
 
 static gboolean processing_blocked_ops = FALSE;
 
 gboolean
 is_op_blocked(const char *rsc)
 {
     GList *gIter = NULL;
     svc_action_t *op = NULL;
 
     for (gIter = inflight_ops; gIter != NULL; gIter = gIter->next) {
         op = gIter->data;
         if (pcmk__str_eq(op->rsc, rsc, pcmk__str_casei)) {
             return TRUE;
         }
     }
 
     return FALSE;
 }
 
 static void
 handle_blocked_ops(void)
 {
     GList *executed_ops = NULL;
     GList *gIter = NULL;
     svc_action_t *op = NULL;
     gboolean res = FALSE;
 
     if (processing_blocked_ops) {
         /* avoid nested calling of this function */
         return;
     }
 
     processing_blocked_ops = TRUE;
 
     /* n^2 operation here, but blocked ops are incredibly rare. this list
      * will be empty 99% of the time. */
     for (gIter = blocked_ops; gIter != NULL; gIter = gIter->next) {
         op = gIter->data;
         if (is_op_blocked(op->rsc)) {
             continue;
         }
         executed_ops = g_list_append(executed_ops, op);
         res = action_exec_helper(op);
         if (res == FALSE) {
             op->status = PCMK_EXEC_ERROR;
             /* this can cause this function to be called recursively
              * which is why we have processing_blocked_ops static variable */
             operation_finalize(op);
         }
     }
 
     for (gIter = executed_ops; gIter != NULL; gIter = gIter->next) {
         op = gIter->data;
         blocked_ops = g_list_remove(blocked_ops, op);
     }
     g_list_free(executed_ops);
 
     processing_blocked_ops = FALSE;
 }
 
 static gboolean
 action_get_metadata(svc_action_t *op)
 {
     const char *class = op->standard;
 
     if (op->agent == NULL) {
         crm_err("meta-data requested without specifying agent");
         return FALSE;
     }
 
     if (class == NULL) {
         crm_err("meta-data requested for agent %s without specifying class",
                 op->agent);
         return FALSE;
     }
 
     if (!strcmp(class, PCMK_RESOURCE_CLASS_SERVICE)) {
         class = resources_find_service_class(op->agent);
     }
 
     if (class == NULL) {
         crm_err("meta-data requested for %s, but could not determine class",
                 op->agent);
         return FALSE;
     }
 
     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) {
         return (services__get_lsb_metadata(op->agent, &op->stdout_data) >= 0);
     }
 
 #if SUPPORT_NAGIOS
     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
         return services__get_nagios_metadata(op->agent, &op->stdout_data) >= 0;
     }
 #endif
 
     return action_exec_helper(op);
 }
 
 gboolean
 services_action_sync(svc_action_t * op)
 {
     gboolean rc = TRUE;
 
     if (op == NULL) {
         crm_trace("No operation to execute");
         return FALSE;
     }
 
     op->synchronous = true;
 
     if (pcmk__str_eq(op->action, "meta-data", pcmk__str_casei)) {
         /* Synchronous meta-data operations are handled specially. Since most
          * resource classes don't provide any meta-data, it has to be
          * synthesized from available information about the agent.
          *
          * services_action_async() doesn't treat meta-data actions specially, so
          * it will result in an error for classes that don't support the action.
          */
         rc = action_get_metadata(op);
     } else {
         rc = action_exec_helper(op);
     }
     crm_trace(" > " PCMK__OP_FMT ": %s = %d",
               op->rsc, op->action, op->interval_ms, op->opaque->exec, op->rc);
     if (op->stdout_data) {
         crm_trace(" >  stdout: %s", op->stdout_data);
     }
     if (op->stderr_data) {
         crm_trace(" >  stderr: %s", op->stderr_data);
     }
     return rc;
 }
 
 GList *
 get_directory_list(const char *root, gboolean files, gboolean executable)
 {
     return services_os_get_directory_list(root, files, executable);
 }
 
 GList *
 resources_list_standards(void)
 {
     GList *standards = NULL;
 
     standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_OCF));
     standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_LSB));
     standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_SERVICE));
 
 #if SUPPORT_SYSTEMD
     {
         GList *agents = systemd_unit_listall();
 
         if (agents != NULL) {
             standards = g_list_append(standards,
                                       strdup(PCMK_RESOURCE_CLASS_SYSTEMD));
             g_list_free_full(agents, free);
         }
     }
 #endif
 
 #if SUPPORT_UPSTART
     {
         GList *agents = upstart_job_listall();
 
         if (agents != NULL) {
             standards = g_list_append(standards,
                                       strdup(PCMK_RESOURCE_CLASS_UPSTART));
             g_list_free_full(agents, free);
         }
     }
 #endif
 
 #if SUPPORT_NAGIOS
     {
         GList *agents = services__list_nagios_agents();
 
         if (agents != NULL) {
             standards = g_list_append(standards,
                                       strdup(PCMK_RESOURCE_CLASS_NAGIOS));
             g_list_free_full(agents, free);
         }
     }
 #endif
 
     return standards;
 }
 
 GList *
 resources_list_providers(const char *standard)
 {
     if (pcmk_is_set(pcmk_get_ra_caps(standard), pcmk_ra_cap_provider)) {
         return resources_os_list_ocf_providers();
     }
 
     return NULL;
 }
 
 GList *
 resources_list_agents(const char *standard, const char *provider)
 {
     if ((standard == NULL)
         || (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0)) {
 
         GList *tmp1;
         GList *tmp2;
         GList *result = services__list_lsb_agents();
 
         if (standard == NULL) {
             tmp1 = result;
             tmp2 = resources_os_list_ocf_agents(NULL);
             if (tmp2) {
                 result = g_list_concat(tmp1, tmp2);
             }
         }
 #if SUPPORT_SYSTEMD
         tmp1 = result;
         tmp2 = systemd_unit_listall();
         if (tmp2) {
             result = g_list_concat(tmp1, tmp2);
         }
 #endif
 
 #if SUPPORT_UPSTART
         tmp1 = result;
         tmp2 = upstart_job_listall();
         if (tmp2) {
             result = g_list_concat(tmp1, tmp2);
         }
 #endif
 
         return result;
 
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
         return resources_os_list_ocf_agents(provider);
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
         return services__list_lsb_agents();
 #if SUPPORT_SYSTEMD
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
         return systemd_unit_listall();
 #endif
 #if SUPPORT_UPSTART
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
         return upstart_job_listall();
 #endif
 #if SUPPORT_NAGIOS
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
         return services__list_nagios_agents();
 #endif
     }
 
     return NULL;
 }
 
 gboolean
 resources_agent_exists(const char *standard, const char *provider, const char *agent)
 {
     GList *standards = NULL;
     GList *providers = NULL;
     GList *iter = NULL;
     gboolean rc = FALSE;
     gboolean has_providers = FALSE;
 
     standards = resources_list_standards();
     for (iter = standards; iter != NULL; iter = iter->next) {
         if (pcmk__str_eq(iter->data, standard, pcmk__str_none)) {
             rc = TRUE;
             break;
         }
     }
 
     if (rc == FALSE) {
         goto done;
     }
 
     rc = FALSE;
 
     has_providers = pcmk_is_set(pcmk_get_ra_caps(standard), pcmk_ra_cap_provider);
     if (has_providers == TRUE && provider != NULL) {
         providers = resources_list_providers(standard);
         for (iter = providers; iter != NULL; iter = iter->next) {
             if (pcmk__str_eq(iter->data, provider, pcmk__str_none)) {
                 rc = TRUE;
                 break;
             }
         }
     } else if (has_providers == FALSE && provider == NULL) {
         rc = TRUE;
     }
 
     if (rc == FALSE) {
         goto done;
     }
 
     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
         if (services__lsb_agent_exists(agent)) {
             rc = TRUE;
 #if SUPPORT_SYSTEMD
         } else if (systemd_unit_exists(agent)) {
             rc = TRUE;
 #endif
 
 #if SUPPORT_UPSTART
         } else if (upstart_job_exists(agent)) {
             rc = TRUE;
 #endif
         } else {
             rc = FALSE;
         }
 
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
         rc = services__ocf_agent_exists(provider, agent);
 
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) {
         rc = services__lsb_agent_exists(agent);
 
 #if SUPPORT_SYSTEMD
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
         rc = systemd_unit_exists(agent);
 #endif
 
 #if SUPPORT_UPSTART
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_casei)) {
         rc = upstart_job_exists(agent);
 #endif
 
 #if SUPPORT_NAGIOS
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
         rc = services__nagios_agent_exists(agent);
 #endif
 
     } else {
         rc = FALSE;
     }
 
 done:
     g_list_free(standards);
     g_list_free(providers);
     return rc;
 }
diff --git a/lib/services/services_lsb.c b/lib/services/services_lsb.c
index 81f449acf5..a3ad431445 100644
--- a/lib/services/services_lsb.c
+++ b/lib/services/services_lsb.c
@@ -1,288 +1,281 @@
 /*
- * Copyright 2010-2020 the Pacemaker project contributors
+ * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <stdio.h>
 #include <errno.h>
 #include <sys/stat.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include "services_private.h"
 #include "services_lsb.h"
 
 #define lsb_metadata_template  \
     "<?xml version='1.0'?>\n"                                           \
     "<!DOCTYPE resource-agent SYSTEM 'ra-api-1.dtd'>\n"                 \
     "<resource-agent name='%s' version='" PCMK_DEFAULT_AGENT_VERSION "'>\n" \
     "  <version>1.0</version>\n"                                        \
     "  <longdesc lang='en'>\n"                                          \
     "%s"                                                                \
     "  </longdesc>\n"                                                   \
     "  <shortdesc lang='en'>%s</shortdesc>\n"                           \
     "  <parameters>\n"                                                  \
     "  </parameters>\n"                                                 \
     "  <actions>\n"                                                     \
     "    <action name='meta-data'    timeout='5' />\n"                  \
     "    <action name='start'        timeout='15' />\n"                 \
     "    <action name='stop'         timeout='15' />\n"                 \
     "    <action name='status'       timeout='15' />\n"                 \
     "    <action name='restart'      timeout='15' />\n"                 \
     "    <action name='force-reload' timeout='15' />\n"                 \
     "    <action name='monitor'      timeout='15' interval='15' />\n"   \
     "  </actions>\n"                                                    \
     "  <special tag='LSB'>\n"                                           \
     "    <Provides>%s</Provides>\n"                                     \
     "    <Required-Start>%s</Required-Start>\n"                         \
     "    <Required-Stop>%s</Required-Stop>\n"                           \
     "    <Should-Start>%s</Should-Start>\n"                             \
     "    <Should-Stop>%s</Should-Stop>\n"                               \
     "    <Default-Start>%s</Default-Start>\n"                           \
     "    <Default-Stop>%s</Default-Stop>\n"                             \
     "  </special>\n"                                                    \
     "</resource-agent>\n"
 
 /* See "Comment Conventions for Init Scripts" in the LSB core specification at:
  * http://refspecs.linuxfoundation.org/lsb.shtml
  */
 #define LSB_INITSCRIPT_INFOBEGIN_TAG "### BEGIN INIT INFO"
 #define LSB_INITSCRIPT_INFOEND_TAG "### END INIT INFO"
 #define PROVIDES    "# Provides:"
 #define REQ_START   "# Required-Start:"
 #define REQ_STOP    "# Required-Stop:"
 #define SHLD_START  "# Should-Start:"
 #define SHLD_STOP   "# Should-Stop:"
 #define DFLT_START  "# Default-Start:"
 #define DFLT_STOP   "# Default-Stop:"
 #define SHORT_DSCR  "# Short-Description:"
 #define DESCRIPTION "# Description:"
 
 #define lsb_meta_helper_free_value(m)           \
     do {                                        \
         if ((m) != NULL) {                      \
             xmlFree(m);                         \
             (m) = NULL;                         \
         }                                       \
     } while(0)
 
 /*!
  * \internal
  * \brief Grab an LSB header value
  *
  * \param[in]     line    Line read from LSB init script
  * \param[in,out] value   If not set, will be set to XML-safe copy of value
  * \param[in]     prefix  Set value if line starts with this pattern
  *
  * \return TRUE if value was set, FALSE otherwise
  */
 static inline gboolean
 lsb_meta_helper_get_value(const char *line, char **value, const char *prefix)
 {
     if (!*value && pcmk__starts_with(line, prefix)) {
         *value = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST line+strlen(prefix));
         return TRUE;
     }
     return FALSE;
 }
 
 #define DESC_MAX 2048
 
 int
 services__get_lsb_metadata(const char *type, char **output)
 {
     char ra_pathname[PATH_MAX] = { 0, };
     FILE *fp = NULL;
     char buffer[1024] = { 0, };
     char *provides = NULL;
     char *req_start = NULL;
     char *req_stop = NULL;
     char *shld_start = NULL;
     char *shld_stop = NULL;
     char *dflt_start = NULL;
     char *dflt_stop = NULL;
     char *s_dscrpt = NULL;
     char *xml_l_dscrpt = NULL;
     int offset = 0;
     bool in_header = FALSE;
     char description[DESC_MAX] = { 0, };
 
     if (type[0] == '/') {
         snprintf(ra_pathname, sizeof(ra_pathname), "%s", type);
     } else {
         snprintf(ra_pathname, sizeof(ra_pathname), "%s/%s",
                  LSB_ROOT_DIR, type);
     }
 
     crm_trace("Looking into %s", ra_pathname);
     fp = fopen(ra_pathname, "r");
     if (fp == NULL) {
         return -errno;
     }
 
     /* Enter into the LSB-compliant comment block */
     while (fgets(buffer, sizeof(buffer), fp)) {
 
         // Ignore lines up to and including the block delimiter
         if (pcmk__starts_with(buffer, LSB_INITSCRIPT_INFOBEGIN_TAG)) {
             in_header = TRUE;
             continue;
         }
         if (!in_header) {
             continue;
         }
 
         /* Assume each of the following eight arguments contain one line */
         if (lsb_meta_helper_get_value(buffer, &provides, PROVIDES)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &req_start, REQ_START)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &req_stop, REQ_STOP)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &shld_start, SHLD_START)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &shld_stop, SHLD_STOP)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &dflt_start, DFLT_START)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &dflt_stop, DFLT_STOP)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &s_dscrpt, SHORT_DSCR)) {
             continue;
         }
 
         /* Long description may cross multiple lines */
         if ((offset == 0) // haven't already found long description
             && pcmk__starts_with(buffer, DESCRIPTION)) {
             bool processed_line = TRUE;
 
             // Get remainder of description line itself
             offset += snprintf(description, DESC_MAX, "%s",
                                buffer + strlen(DESCRIPTION));
 
             // Read any continuation lines of the description
             buffer[0] = '\0';
             while (fgets(buffer, sizeof(buffer), fp)) {
                 if (pcmk__starts_with(buffer, "#  ")
                     || pcmk__starts_with(buffer, "#\t")) {
                     /* '#' followed by a tab or more than one space indicates a
                      * continuation of the long description.
                      */
                     offset += snprintf(description + offset, DESC_MAX - offset,
                                        "%s", buffer + 1);
                 } else {
                     /* This line is not part of the long description,
                      * so continue with normal processing.
                      */
                     processed_line = FALSE;
                     break;
                 }
             }
 
             // Make long description safe to use in XML
             xml_l_dscrpt = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST(description));
 
             if (processed_line) {
                 // We grabbed the line into the long description
                 continue;
             }
         }
 
         // Stop if we leave the header block
         if (pcmk__starts_with(buffer, LSB_INITSCRIPT_INFOEND_TAG)) {
             break;
         }
         if (buffer[0] != '#') {
             break;
         }
     }
     fclose(fp);
 
     *output = crm_strdup_printf(lsb_metadata_template, type,
                                 (xml_l_dscrpt? xml_l_dscrpt : type),
                                 (s_dscrpt? s_dscrpt : type),
                                 (provides? provides : ""),
                                 (req_start? req_start : ""),
                                 (req_stop? req_stop : ""),
                                 (shld_start? shld_start : ""),
                                 (shld_stop? shld_stop : ""),
                                 (dflt_start? dflt_start : ""),
                                 (dflt_stop? dflt_stop : ""));
 
     lsb_meta_helper_free_value(xml_l_dscrpt);
     lsb_meta_helper_free_value(s_dscrpt);
     lsb_meta_helper_free_value(provides);
     lsb_meta_helper_free_value(req_start);
     lsb_meta_helper_free_value(req_stop);
     lsb_meta_helper_free_value(shld_start);
     lsb_meta_helper_free_value(shld_stop);
     lsb_meta_helper_free_value(dflt_start);
     lsb_meta_helper_free_value(dflt_stop);
 
     crm_trace("Created fake metadata: %llu",
               (unsigned long long) strlen(*output));
     return pcmk_ok;
 }
 
 GList *
 services__list_lsb_agents(void)
 {
     return services_os_get_directory_list(LSB_ROOT_DIR, TRUE, TRUE);
 }
 
-char *
-services__lsb_agent_path(const char *agent)
-{
-    return (*agent == '/')? strdup(agent)
-        : crm_strdup_printf("%s/%s", LSB_ROOT_DIR, agent);
-}
-
 bool
 services__lsb_agent_exists(const char *agent)
 {
     bool rc = FALSE;
     struct stat st;
-    char *path = services__lsb_agent_path(agent);
+    char *path = pcmk__full_path(agent, LSB_ROOT_DIR);
 
     rc = (stat(path, &st) == 0);
     free(path);
     return rc;
 }
 
 /* The remaining functions below are not used by the Pacemaker code base, and
  * are provided for API compatibility only.
  *
  * @TODO They should be removed or renamed.
  */
 
 svc_action_t *
 services_action_create(const char *name, const char *action,
                        guint interval_ms, int timeout)
 {
     return resources_action_create(name, PCMK_RESOURCE_CLASS_LSB, NULL, name,
                                    action, interval_ms, timeout, NULL, 0);
 }
 
 GList *
 services_list(void)
 {
     return resources_list_agents(PCMK_RESOURCE_CLASS_LSB, NULL);
 }
 
diff --git a/lib/services/services_lsb.h b/lib/services/services_lsb.h
index df9d8b356a..b595dba8b0 100644
--- a/lib/services/services_lsb.h
+++ b/lib/services/services_lsb.h
@@ -1,18 +1,17 @@
 /*
  * Copyright 2010-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef SERVICES_LSB__H
 #  define SERVICES_LSB__H
 
 G_GNUC_INTERNAL int services__get_lsb_metadata(const char *type, char **output);
 G_GNUC_INTERNAL GList *services__list_lsb_agents(void);
-G_GNUC_INTERNAL char *services__lsb_agent_path(const char *agent);
 G_GNUC_INTERNAL bool services__lsb_agent_exists(const char *agent);
 
 #endif
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 141cbacf3f..1d69a868c3 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1,2135 +1,2155 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm_resource.h>
 #include <crm/lrmd_internal.h>
 #include <crm/common/cmdline_internal.h>
 #include <crm/common/lists_internal.h>
 #include <crm/common/output.h>
 #include <pacemaker-internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/stonith-ng.h>
 #include <crm/common/ipc_controld.h>
 #include <crm/cib/internal.h>
 
 #define SUMMARY "crm_resource - perform tasks related to Pacemaker cluster resources"
 
 enum rsc_command {
     cmd_none = 0,           // No command option given (yet)
     cmd_ban,
     cmd_cleanup,
     cmd_clear,
     cmd_colocations,
     cmd_colocations_deep,
     cmd_cts,
     cmd_delete,
     cmd_delete_param,
     cmd_digests,
     cmd_execute_agent,
     cmd_fail,
     cmd_get_param,
     cmd_get_property,
     cmd_list_active_ops,
     cmd_list_agents,
     cmd_list_all_ops,
     cmd_list_alternatives,
     cmd_list_instances,
     cmd_list_providers,
     cmd_list_resources,
     cmd_list_standards,
     cmd_locate,
     cmd_metadata,
     cmd_move,
     cmd_query_raw_xml,
     cmd_query_xml,
     cmd_refresh,
     cmd_restart,
     cmd_set_param,
     cmd_set_property,
     cmd_wait,
     cmd_why,
 };
 
 struct {
     enum rsc_command rsc_cmd;     // crm_resource command to perform
 
     // Infrastructure that given command needs to work
     gboolean require_cib;         // Whether command requires CIB IPC
     int cib_options;              // Options to use with CIB IPC calls
     gboolean require_crmd;        // Whether command requires controller IPC
     gboolean require_dataset;     // Whether command requires populated data set
     gboolean require_resource;    // Whether command requires resource specified
     gboolean require_node;        // Whether command requires node specified
     int find_flags;               // Flags to use when searching for resource
 
     // Command-line option values
     gchar *rsc_id;                // Value of --resource
     gchar *rsc_type;              // Value of --resource-type
     gboolean force;               // --force was given
     gboolean clear_expired;       // --expired was given
     gboolean recursive;           // --recursive was given
     gboolean promoted_role_only;  // --promoted was given
     gchar *host_uname;            // Value of --node
     gchar *interval_spec;         // Value of --interval
     gchar *move_lifetime;         // Value of --lifetime
     gchar *operation;             // Value of --operation
     const char *attr_set_type;    // Instance, meta, or utilization attribute
     gchar *prop_id;               // --nvpair (attribute XML ID)
     char *prop_name;              // Attribute name
     gchar *prop_set;              // --set-name (attribute block XML ID)
     gchar *prop_value;            // --parameter-value (attribute value)
     int timeout_ms;               // Parsed from --timeout value
     char *agent_spec;             // Standard and/or provider and/or agent
     gchar *xml_file;              // Value of (deprecated) --xml-file
     int check_level;              // Optional value of --validate or --force-check
 
     // Resource configuration specified via command-line arguments
     gboolean cmdline_config;      // Resource configuration was via arguments
     char *v_agent;                // Value of --agent
     char *v_class;                // Value of --class
     char *v_provider;             // Value of --provider
     GHashTable *cmdline_params;   // Resource parameters specified
 
     // Positional command-line arguments
     gchar **remainder;            // Positional arguments as given
     GHashTable *override_params;  // Resource parameter values that override config
 } options = {
     .attr_set_type = XML_TAG_ATTR_SETS,
     .check_level = -1,
     .cib_options = cib_sync_call,
     .require_cib = TRUE,
     .require_dataset = TRUE,
     .require_resource = TRUE,
 };
 
 #if 0
 // @COMPAT @TODO enable this at next backward compatibility break
 #define SET_COMMAND(cmd) do {                                               \
         if (options.rsc_cmd != cmd_none) {                                  \
             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,             \
                         "Only one command option may be specified");        \
             return FALSE;                                                   \
         }                                                                   \
         options.rsc_cmd = (cmd);                                            \
     } while (0)
 #else
 #define SET_COMMAND(cmd) do { options.rsc_cmd = (cmd); } while (0)
 #endif
 
 gboolean agent_provider_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean class_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean cleanup_refresh_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean delete_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean expired_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean list_agents_cb(const gchar *option_name, const gchar *optarg,
                         gpointer data, GError **error);
 gboolean list_providers_cb(const gchar *option_name, const gchar *optarg,
                            gpointer data, GError **error);
 gboolean list_standards_cb(const gchar *option_name, const gchar *optarg,
                            gpointer data, GError **error);
 gboolean list_alternatives_cb(const gchar *option_name, const gchar *optarg,
                               gpointer data, GError **error);
 gboolean metadata_cb(const gchar *option_name, const gchar *optarg,
                      gpointer data, GError **error);
 gboolean option_cb(const gchar *option_name, const gchar *optarg,
                    gpointer data, GError **error);
 gboolean fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean flag_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean get_param_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean list_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean set_delete_param_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean set_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean timeout_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean validate_or_force_cb(const gchar *option_name, const gchar *optarg,
                               gpointer data, GError **error);
 gboolean restart_cb(const gchar *option_name, const gchar *optarg,
                     gpointer data, GError **error);
 gboolean digests_cb(const gchar *option_name, const gchar *optarg,
                     gpointer data, GError **error);
 gboolean wait_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean why_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 
 static crm_exit_t exit_code = CRM_EX_OK;
 static pcmk__output_t *out = NULL;
 static pcmk__common_args_t *args = NULL;
 
 // Things that should be cleaned up on exit
 static GError *error = NULL;
 static GMainLoop *mainloop = NULL;
 static cib_t *cib_conn = NULL;
 static pcmk_ipc_api_t *controld_api = NULL;
 static pe_working_set_t *data_set = NULL;
 
 #define MESSAGE_TIMEOUT_S 60
 
 #define INDENT "                                    "
 
 static pcmk__supported_format_t formats[] = {
     PCMK__SUPPORTED_FORMAT_NONE,
     PCMK__SUPPORTED_FORMAT_TEXT,
     PCMK__SUPPORTED_FORMAT_XML,
     { NULL, NULL, NULL }
 };
 
 // Clean up and exit
 static crm_exit_t
 bye(crm_exit_t ec)
 {
     pcmk__output_and_clear_error(error, out);
 
     if (out != NULL) {
         out->finish(out, ec, true, NULL);
         pcmk__output_free(out);
     }
 
     if (cib_conn != NULL) {
         cib_t *save_cib_conn = cib_conn;
 
         cib_conn = NULL; // Ensure we can't free this twice
         save_cib_conn->cmds->signoff(save_cib_conn);
         cib_delete(save_cib_conn);
     }
     if (controld_api != NULL) {
         pcmk_ipc_api_t *save_controld_api = controld_api;
 
         controld_api = NULL; // Ensure we can't free this twice
         pcmk_free_ipc_api(save_controld_api);
     }
     if (mainloop != NULL) {
         g_main_loop_unref(mainloop);
         mainloop = NULL;
     }
     pe_free_working_set(data_set);
     data_set = NULL;
     crm_exit(ec);
     return ec;
 }
 
 static void
 quit_main_loop(crm_exit_t ec)
 {
     exit_code = ec;
     if (mainloop != NULL) {
         GMainLoop *mloop = mainloop;
 
         mainloop = NULL; // Don't re-enter this block
         pcmk_quit_main_loop(mloop, 10);
         g_main_loop_unref(mloop);
     }
 }
 
 static gboolean
 resource_ipc_timeout(gpointer data)
 {
     // Start with newline because "Waiting for ..." message doesn't have one
     if (error != NULL) {
         g_clear_error(&error);
     }
 
     g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_TIMEOUT,
                 "Aborting because no messages received in %d seconds", MESSAGE_TIMEOUT_S);
 
     quit_main_loop(CRM_EX_TIMEOUT);
     return FALSE;
 }
 
 static void
 controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
                           crm_exit_t status, void *event_data, void *user_data)
 {
     switch (event_type) {
         case pcmk_ipc_event_disconnect:
             if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
                 crm_info("Connection to controller was terminated");
             }
             quit_main_loop(exit_code);
             break;
 
         case pcmk_ipc_event_reply:
             if (status != CRM_EX_OK) {
                 out->err(out, "Error: bad reply from controller: %s",
                          crm_exit_str(status));
                 pcmk_disconnect_ipc(api);
                 quit_main_loop(status);
             } else {
                 if ((pcmk_controld_api_replies_expected(api) == 0)
                     && mainloop && g_main_loop_is_running(mainloop)) {
                     out->info(out, "... got reply (done)");
                     crm_debug("Got all the replies we expected");
                     pcmk_disconnect_ipc(api);
                     quit_main_loop(CRM_EX_OK);
                 } else {
                     out->info(out, "... got reply");
                 }
             }
             break;
 
         default:
             break;
     }
 }
 
 static void
 start_mainloop(pcmk_ipc_api_t *capi)
 {
     unsigned int count = pcmk_controld_api_replies_expected(capi);
 
     if (count > 0) {
         out->info(out, "Waiting for %d %s from the controller",
                   count, pcmk__plural_alt(count, "reply", "replies"));
         exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
         mainloop = g_main_loop_new(NULL, FALSE);
         g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
         g_main_loop_run(mainloop);
     }
 }
 
 static int
 compare_id(gconstpointer a, gconstpointer b)
 {
     return strcmp((const char *)a, (const char *)b);
 }
 
 static GList *
 build_constraint_list(xmlNode *root)
 {
     GList *retval = NULL;
     xmlNode *cib_constraints = NULL;
     xmlXPathObjectPtr xpathObj = NULL;
     int ndx = 0;
 
     cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, root);
     xpathObj = xpath_search(cib_constraints, "//" XML_CONS_TAG_RSC_LOCATION);
 
     for (ndx = 0; ndx < numXpathResults(xpathObj); ndx++) {
         xmlNode *match = getXpathResult(xpathObj, ndx);
         retval = g_list_insert_sorted(retval, (gpointer) ID(match), compare_id);
     }
 
     freeXpathObject(xpathObj);
     return retval;
 }
 
 /* short option letters still available: eEJkKXyYZ */
 
 static GOptionEntry query_entries[] = {
     { "list", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
       "List all cluster resources with status",
       NULL },
     { "list-raw", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
       "List IDs of all instantiated resources (individual members\n"
       INDENT "rather than groups etc.)",
       NULL },
     { "list-cts", 'c', G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
       NULL,
       NULL },
     { "list-operations", 'O', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
       "List active resource operations, optionally filtered by\n"
       INDENT "--resource and/or --node",
       NULL },
     { "list-all-operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
       "List all resource operations, optionally filtered by\n"
       INDENT "--resource and/or --node",
       NULL },
     { "list-standards", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
       list_standards_cb,
       "List supported standards",
       NULL },
     { "list-ocf-providers", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
       list_providers_cb,
       "List all available OCF providers",
       NULL },
     { "list-agents", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
       list_agents_cb,
       "List all agents available for the named standard and/or provider",
       "STD:PROV" },
     { "list-ocf-alternatives", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
       list_alternatives_cb,
       "List all available providers for the named OCF agent",
       "AGENT" },
     { "show-metadata", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
       metadata_cb,
       "Show the metadata for the named class:provider:agent",
       "SPEC" },
     { "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
       "Show XML configuration of resource (after any template expansion)",
       NULL },
     { "query-xml-raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
       "Show XML configuration of resource (before any template expansion)",
       NULL },
     { "get-parameter", 'g', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
       "Display named parameter for resource (use instance attribute\n"
       INDENT "unless --meta or --utilization is specified)",
       "PARAM" },
     { "get-property", 'G', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
       "Display named property of resource ('class', 'type', or 'provider') "
       "(requires --resource)",
       "PROPERTY" },
     { "locate", 'W', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
       "Show node(s) currently running resource",
       NULL },
     { "stack", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
       "Display the (co)location constraints that apply to a resource\n"
       INDENT "and the resources is it colocated with",
       NULL },
     { "constraints", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
       "Display the (co)location constraints that apply to a resource",
       NULL },
     { "why", 'Y', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, why_cb,
       "Show why resources are not running, optionally filtered by\n"
       INDENT "--resource and/or --node",
       NULL },
 
     { NULL }
 };
 
 static GOptionEntry command_entries[] = {
     { "validate", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
       validate_or_force_cb,
       "Validate resource configuration by calling agent's validate-all\n"
       INDENT "action. The configuration may be specified either by giving an\n"
       INDENT "existing resource name with -r, or by specifying --class,\n"
       INDENT "--agent, and --provider arguments, along with any number of\n"
       INDENT "--option arguments. An optional LEVEL argument can be given\n"
       INDENT "to control the level of checking performed.",
       "LEVEL" },
     { "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
       "If resource has any past failures, clear its history and fail\n"
       INDENT "count. Optionally filtered by --resource, --node, --operation\n"
       INDENT "and --interval (otherwise all). --operation and --interval\n"
       INDENT "apply to fail counts, but entire history is always clear, to\n"
       INDENT "allow current state to be rechecked. If the named resource is\n"
       INDENT "part of a group, or one numbered instance of a clone or bundled\n"
       INDENT "resource, the clean-up applies to the whole collective resource\n"
       INDENT "unless --force is given.",
       NULL },
     { "refresh", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
       "Delete resource's history (including failures) so its current state\n"
       INDENT "is rechecked. Optionally filtered by --resource and --node\n"
       INDENT "(otherwise all). If the named resource is part of a group, or one\n"
       INDENT "numbered instance of a clone or bundled resource, the refresh\n"
       INDENT "applies to the whole collective resource unless --force is given.",
       NULL },
     { "set-parameter", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
       "Set named parameter for resource (requires -v). Use instance\n"
       INDENT "attribute unless --meta or --utilization is specified.",
       "PARAM" },
     { "delete-parameter", 'd', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
       "Delete named parameter for resource. Use instance attribute\n"
       INDENT "unless --meta or --utilization is specified.",
       "PARAM" },
     { "set-property", 'S', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, set_prop_cb,
       "Set named property of resource ('class', 'type', or 'provider') "
       "(requires -r, -t, -v)",
       "PROPERTY" },
 
     { NULL }
 };
 
 static GOptionEntry location_entries[] = {
     { "move", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
       "Create a constraint to move resource. If --node is specified,\n"
       INDENT "the constraint will be to move to that node, otherwise it\n"
       INDENT "will be to ban the current node. Unless --force is specified\n"
       INDENT "this will return an error if the resource is already running\n"
       INDENT "on the specified node. If --force is specified, this will\n"
       INDENT "always ban the current node.\n"
       INDENT "Optional: --lifetime, --promoted. NOTE: This may prevent the\n"
       INDENT "resource from running on its previous location until the\n"
       INDENT "implicit constraint expires or is removed with --clear.",
       NULL },
     { "ban", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
       "Create a constraint to keep resource off a node.\n"
       INDENT "Optional: --node, --lifetime, --promoted.\n"
       INDENT "NOTE: This will prevent the resource from running on the\n"
       INDENT "affected node until the implicit constraint expires or is\n"
       INDENT "removed with --clear. If --node is not specified, it defaults\n"
       INDENT "to the node currently running the resource for primitives\n"
       INDENT "and groups, or the promoted instance of promotable clones with\n"
       INDENT "promoted-max=1 (all other situations result in an error as\n"
       INDENT "there is no sane default).",
       NULL },
     { "clear", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
       "Remove all constraints created by the --ban and/or --move\n"
       INDENT "commands. Requires: --resource. Optional: --node, --promoted,\n"
       INDENT "--expired. If --node is not specified, all constraints created\n"
       INDENT "by --ban and --move will be removed for the named resource. If\n"
       INDENT "--node and --force are specified, any constraint created by\n"
       INDENT "--move will be cleared, even if it is not for the specified\n"
       INDENT "node. If --expired is specified, only those constraints whose\n"
       INDENT "lifetimes have expired will be removed.",
       NULL },
     { "expired", 'e', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, expired_cb,
       "Modifies the --clear argument to remove constraints with\n"
       INDENT "expired lifetimes.",
       NULL },
     { "lifetime", 'u', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.move_lifetime,
       "Lifespan (as ISO 8601 duration) of created constraints (with\n"
       INDENT "-B, -M) see https://en.wikipedia.org/wiki/ISO_8601#Durations)",
       "TIMESPEC" },
     { "promoted", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
       &options.promoted_role_only,
       "Limit scope of command to promoted role (with -B, -M, -U). For\n"
       INDENT "-B and -M, previously promoted instances may remain\n"
       INDENT "active in the unpromoted role.",
       NULL },
 
     // Deprecated since 2.1.0
     { "master", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
       &options.promoted_role_only,
       "Deprecated: Use --promoted instead", NULL },
 
     { NULL }
 };
 
 static GOptionEntry advanced_entries[] = {
     { "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, delete_cb,
       "(Advanced) Delete a resource from the CIB. Required: -t",
       NULL },
     { "fail", 'F', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, fail_cb,
       "(Advanced) Tell the cluster this resource has failed",
       NULL },
     { "restart", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, restart_cb,
       "(Advanced) Tell the cluster to restart this resource and\n"
       INDENT "anything that depends on it",
       NULL },
     { "wait", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, wait_cb,
       "(Advanced) Wait until the cluster settles into a stable state",
       NULL },
     { "digests", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, digests_cb,
       "(Advanced) Show parameter hashes that Pacemaker uses to detect\n"
       INDENT "configuration changes (only accurate if there is resource\n"
       INDENT "history on the specified node). Required: --resource, --node.\n"
       INDENT "Optional: any NAME=VALUE parameters will be used to override\n"
       INDENT "the configuration (to see what the hash would be with those\n"
       INDENT "changes).",
       NULL },
     { "force-demote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
       validate_or_force_cb,
       "(Advanced) Bypass the cluster and demote a resource on the local\n"
       INDENT "node. Unless --force is specified, this will refuse to do so if\n"
       INDENT "the cluster believes the resource is a clone instance already\n"
       INDENT "running on the local node.",
       NULL },
     { "force-stop", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
       validate_or_force_cb,
       "(Advanced) Bypass the cluster and stop a resource on the local node",
       NULL },
     { "force-start", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
       validate_or_force_cb,
       "(Advanced) Bypass the cluster and start a resource on the local\n"
       INDENT "node. Unless --force is specified, this will refuse to do so if\n"
       INDENT "the cluster believes the resource is a clone instance already\n"
       INDENT "running on the local node.",
       NULL },
     { "force-promote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
       validate_or_force_cb,
       "(Advanced) Bypass the cluster and promote a resource on the local\n"
       INDENT "node. Unless --force is specified, this will refuse to do so if\n"
       INDENT "the cluster believes the resource is a clone instance already\n"
       INDENT "running on the local node.",
       NULL },
     { "force-check", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
       validate_or_force_cb,
       "(Advanced) Bypass the cluster and check the state of a resource on\n"
       INDENT "the local node. An optional LEVEL argument can be given\n"
       INDENT "to control the level of checking performed.",
       "LEVEL" },
 
     { NULL }
 };
 
 static GOptionEntry addl_entries[] = {
     { "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.host_uname,
       "Node name",
       "NAME" },
     { "recursive", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.recursive,
       "Follow colocation chains when using --set-parameter",
       NULL },
     { "resource-type", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_type,
       "Resource XML element (primitive, group, etc.) (with -D)",
       "ELEMENT" },
     { "parameter-value", 'v', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_value,
       "Value to use with -p",
       "PARAM" },
     { "meta", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
       "Use resource meta-attribute instead of instance attribute\n"
       INDENT "(with -p, -g, -d)",
       NULL },
     { "utilization", 'z', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
       "Use resource utilization attribute instead of instance attribute\n"
       INDENT "(with -p, -g, -d)",
       NULL },
     { "operation", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.operation,
       "Operation to clear instead of all (with -C -r)",
       "OPERATION" },
     { "interval", 'I', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.interval_spec,
       "Interval of operation to clear (default 0) (with -C -r -n)",
       "N" },
     { "class", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, class_cb,
       "The standard the resource agent conforms to (for example, ocf).\n"
       INDENT "Use with --agent, --provider, --option, and --validate.",
       "CLASS" },
     { "agent", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
       "The agent to use (for example, IPaddr). Use with --class,\n"
       INDENT "--provider, --option, and --validate.",
       "AGENT" },
     { "provider", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
       "The vendor that supplies the resource agent (for example,\n"
       INDENT "heartbeat). Use with --class, --agent, --option, and --validate.",
       "PROVIDER" },
     { "option", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, option_cb,
       "Specify a device configuration parameter as NAME=VALUE (may be\n"
       INDENT "specified multiple times). Use with --validate and without the\n"
       INDENT "-r option.",
       "PARAM" },
     { "set-name", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_set,
       "(Advanced) XML ID of attributes element to use (with -p, -d)",
       "ID" },
     { "nvpair", 'i', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_id,
       "(Advanced) XML ID of nvpair element to use (with -p, -d)",
       "ID" },
     { "timeout", 'T', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, timeout_cb,
       "(Advanced) Abort if command does not finish in this time (with\n"
       INDENT "--restart, --wait, --force-*)",
       "N" },
     { "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
       "If making CIB changes, do so regardless of quorum. See help for\n"
       INDENT "individual commands for additional behavior.",
       NULL },
     { "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_FILENAME, &options.xml_file,
       NULL,
       "FILE" },
     { "host-uname", 'H', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.host_uname,
       NULL,
       "HOST" },
 
     { NULL }
 };
 
 gboolean
 agent_provider_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.cmdline_config = TRUE;
     options.require_resource = FALSE;
 
     if (pcmk__str_eq(option_name, "--provider", pcmk__str_casei)) {
         if (options.v_provider) {
            free(options.v_provider);
         }
         options.v_provider = strdup(optarg);
     } else {
         if (options.v_agent) {
            free(options.v_agent);
         }
         options.v_agent = strdup(optarg);
     }
 
     return TRUE;
 }
 
 gboolean
 attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (pcmk__str_any_of(option_name, "-m", "--meta", NULL)) {
         options.attr_set_type = XML_TAG_META_SETS;
     } else if (pcmk__str_any_of(option_name, "-z", "--utilization", NULL)) {
         options.attr_set_type = XML_TAG_UTILIZATION;
     }
 
     return TRUE;
 }
 
 gboolean
 class_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (options.v_class != NULL) {
         free(options.v_class);
     }
 
     options.v_class = strdup(optarg);
 
     options.cmdline_config = TRUE;
     options.require_resource = FALSE;
     return TRUE;
 }
 
 gboolean
 cleanup_refresh_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (pcmk__str_any_of(option_name, "-C", "--cleanup", NULL)) {
         SET_COMMAND(cmd_cleanup);
     } else {
         SET_COMMAND(cmd_refresh);
     }
 
     options.require_resource = FALSE;
     if (getenv("CIB_file") == NULL) {
         options.require_crmd = TRUE;
     }
     options.find_flags = pe_find_renamed|pe_find_anon;
     return TRUE;
 }
 
 gboolean
 delete_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.require_dataset = FALSE;
     SET_COMMAND(cmd_delete);
     options.find_flags = pe_find_renamed|pe_find_any;
     return TRUE;
 }
 
 gboolean
 expired_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.clear_expired = TRUE;
     options.require_resource = FALSE;
     return TRUE;
 }
 
 static void
 get_agent_spec(const gchar *optarg)
 {
     options.require_cib = FALSE;
     options.require_dataset = FALSE;
     options.require_resource = FALSE;
     if (options.agent_spec != NULL) {
         free(options.agent_spec);
         options.agent_spec = NULL;
     }
     if (optarg != NULL) {
         options.agent_spec = strdup(optarg);
     }
 }
 
 gboolean
 list_agents_cb(const gchar *option_name, const gchar *optarg, gpointer data,
                GError **error)
 {
     SET_COMMAND(cmd_list_agents);
     get_agent_spec(optarg);
     return TRUE;
 }
 
 gboolean
 list_providers_cb(const gchar *option_name, const gchar *optarg, gpointer data,
                   GError **error)
 {
     SET_COMMAND(cmd_list_providers);
     get_agent_spec(optarg);
     return TRUE;
 }
 
 gboolean
 list_standards_cb(const gchar *option_name, const gchar *optarg, gpointer data,
                   GError **error)
 {
     SET_COMMAND(cmd_list_standards);
     options.require_cib = FALSE;
     options.require_dataset = FALSE;
     options.require_resource = FALSE;
     return TRUE;
 }
 
 gboolean
 list_alternatives_cb(const gchar *option_name, const gchar *optarg,
                      gpointer data, GError **error)
 {
     SET_COMMAND(cmd_list_alternatives);
     get_agent_spec(optarg);
     return TRUE;
 }
 
 gboolean
 metadata_cb(const gchar *option_name, const gchar *optarg, gpointer data,
             GError **error)
 {
     SET_COMMAND(cmd_metadata);
     get_agent_spec(optarg);
     return TRUE;
 }
 
 gboolean
 option_cb(const gchar *option_name, const gchar *optarg, gpointer data,
           GError **error)
 {
     char *name = NULL;
     char *value = NULL;
 
     if (pcmk__scan_nvpair(optarg, &name, &value) != 2) {
         return FALSE;
     }
     if (options.cmdline_params == NULL) {
         options.cmdline_params = pcmk__strkey_table(free, free);
     }
     g_hash_table_replace(options.cmdline_params, name, value);
     return TRUE;
 }
 
 gboolean
 fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.require_crmd = TRUE;
     options.require_node = TRUE;
     SET_COMMAND(cmd_fail);
     return TRUE;
 }
 
 gboolean
 flag_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (pcmk__str_any_of(option_name, "-U", "--clear", NULL)) {
         options.find_flags = pe_find_renamed|pe_find_anon;
         SET_COMMAND(cmd_clear);
     } else if (pcmk__str_any_of(option_name, "-B", "--ban", NULL)) {
         options.find_flags = pe_find_renamed|pe_find_anon;
         SET_COMMAND(cmd_ban);
     } else if (pcmk__str_any_of(option_name, "-M", "--move", NULL)) {
         options.find_flags = pe_find_renamed|pe_find_anon;
         SET_COMMAND(cmd_move);
     } else if (pcmk__str_any_of(option_name, "-q", "--query-xml", NULL)) {
         options.find_flags = pe_find_renamed|pe_find_any;
         SET_COMMAND(cmd_query_xml);
     } else if (pcmk__str_any_of(option_name, "-w", "--query-xml-raw", NULL)) {
         options.find_flags = pe_find_renamed|pe_find_any;
         SET_COMMAND(cmd_query_raw_xml);
     } else if (pcmk__str_any_of(option_name, "-W", "--locate", NULL)) {
         options.find_flags = pe_find_renamed|pe_find_anon;
         SET_COMMAND(cmd_locate);
     } else if (pcmk__str_any_of(option_name, "-A", "--stack", NULL)) {
         options.find_flags = pe_find_renamed|pe_find_anon;
         SET_COMMAND(cmd_colocations_deep);
     } else {
         options.find_flags = pe_find_renamed|pe_find_anon;
         SET_COMMAND(cmd_colocations);
     }
 
     return TRUE;
 }
 
 gboolean
 get_param_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (pcmk__str_any_of(option_name, "-g", "--get-parameter", NULL)) {
         SET_COMMAND(cmd_get_param);
     } else {
         SET_COMMAND(cmd_get_property);
     }
 
     if (options.prop_name) {
         free(options.prop_name);
     }
 
     options.prop_name = strdup(optarg);
     options.find_flags = pe_find_renamed|pe_find_any;
     return TRUE;
 }
 
 gboolean
 list_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (pcmk__str_any_of(option_name, "-c", "--list-cts", NULL)) {
         SET_COMMAND(cmd_cts);
     } else if (pcmk__str_any_of(option_name, "-L", "--list", NULL)) {
         SET_COMMAND(cmd_list_resources);
     } else if (pcmk__str_any_of(option_name, "-l", "--list-raw", NULL)) {
         SET_COMMAND(cmd_list_instances);
     } else if (pcmk__str_any_of(option_name, "-O", "--list-operations", NULL)) {
         SET_COMMAND(cmd_list_active_ops);
     } else {
         SET_COMMAND(cmd_list_all_ops);
     }
 
     options.require_resource = FALSE;
     return TRUE;
 }
 
 gboolean
 set_delete_param_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (pcmk__str_any_of(option_name, "-p", "--set-parameter", NULL)) {
         SET_COMMAND(cmd_set_param);
     } else {
         SET_COMMAND(cmd_delete_param);
     }
 
     if (options.prop_name) {
         free(options.prop_name);
     }
 
     options.prop_name = strdup(optarg);
     options.find_flags = pe_find_renamed|pe_find_any;
     return TRUE;
 }
 
 gboolean
 set_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.require_dataset = FALSE;
 
     if (options.prop_name) {
         free(options.prop_name);
     }
 
     options.prop_name = strdup(optarg);
     SET_COMMAND(cmd_set_property);
     options.find_flags = pe_find_renamed|pe_find_any;
     return TRUE;
 }
 
 gboolean
 timeout_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.timeout_ms = crm_get_msec(optarg);
     return TRUE;
 }
 
 gboolean
 validate_or_force_cb(const gchar *option_name, const gchar *optarg,
                      gpointer data, GError **error)
 {
     SET_COMMAND(cmd_execute_agent);
     if (options.operation) {
         g_free(options.operation);
     }
     options.operation = g_strdup(option_name + 2); // skip "--"
     options.find_flags = pe_find_renamed|pe_find_anon;
     if (options.override_params == NULL) {
         options.override_params = pcmk__strkey_table(free, free);
     }
 
     if (optarg != NULL) {
         if (pcmk__scan_min_int(optarg, &options.check_level, 0) != pcmk_rc_ok) {
             g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
                         "Invalid check level setting: %s", optarg);
             return FALSE;
         }
     }
 
     return TRUE;
 }
 
 gboolean
 restart_cb(const gchar *option_name, const gchar *optarg, gpointer data,
            GError **error)
 {
     SET_COMMAND(cmd_restart);
     options.find_flags = pe_find_renamed|pe_find_anon;
     return TRUE;
 }
 
 gboolean
 digests_cb(const gchar *option_name, const gchar *optarg, gpointer data,
            GError **error)
 {
     SET_COMMAND(cmd_digests);
     options.find_flags = pe_find_renamed|pe_find_anon;
     if (options.override_params == NULL) {
         options.override_params = pcmk__strkey_table(free, free);
     }
     options.require_node = TRUE;
     options.require_dataset = TRUE;
     return TRUE;
 }
 
 gboolean
 wait_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     SET_COMMAND(cmd_wait);
     options.require_resource = FALSE;
     options.require_dataset = FALSE;
     return TRUE;
 }
 
 gboolean
 why_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.require_resource = FALSE;
     SET_COMMAND(cmd_why);
     options.find_flags = pe_find_renamed|pe_find_anon;
     return TRUE;
 }
 
 static int
 ban_or_move(pcmk__output_t *out, pe_resource_t *rsc, const char *move_lifetime)
 {
     int rc = pcmk_rc_ok;
     pe_node_t *current = NULL;
     unsigned int nactive = 0;
 
     CRM_CHECK(rsc != NULL, return EINVAL);
 
     current = pe__find_active_requires(rsc, &nactive);
 
     if (nactive == 1) {
         rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
                               cib_conn, options.cib_options, options.promoted_role_only);
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         int count = 0;
         GList *iter = NULL;
 
         current = NULL;
         for(iter = rsc->children; iter; iter = iter->next) {
             pe_resource_t *child = (pe_resource_t *)iter->data;
             enum rsc_role_e child_role = child->fns->state(child, TRUE);
 
             if (child_role == RSC_ROLE_PROMOTED) {
                 count++;
                 current = pe__current_node(child);
             }
         }
 
         if(count == 1 && current) {
             rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
                                   cib_conn, options.cib_options, options.promoted_role_only);
 
         } else {
             rc = EINVAL;
             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                         "Resource '%s' not moved: active in %d locations (promoted in %d).\n"
                         "To prevent '%s' from running on a specific location, "
                         "specify a node."
                         "To prevent '%s' from being promoted at a specific "
                         "location, specify a node and the --promoted option.",
                         options.rsc_id, nactive, count, options.rsc_id, options.rsc_id);
         }
 
     } else {
         rc = EINVAL;
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     "Resource '%s' not moved: active in %d locations.\n"
                     "To prevent '%s' from running on a specific location, "
                     "specify a node.",
                     options.rsc_id, nactive, options.rsc_id);
     }
 
     return rc;
 }
 
 static void
 cleanup(pcmk__output_t *out, pe_resource_t *rsc)
 {
     int rc = pcmk_rc_ok;
 
     if (options.force == FALSE) {
         rsc = uber_parent(rsc);
     }
 
     crm_debug("Erasing failures of %s (%s requested) on %s",
               rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
     rc = cli_resource_delete(controld_api, options.host_uname, rsc, options.operation,
                              options.interval_spec, TRUE, data_set, options.force);
 
     if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
         // Show any reasons why resource might stay stopped
         cli_resource_check(out, cib_conn, rsc);
     }
 
     if (rc == pcmk_rc_ok) {
         start_mainloop(controld_api);
     }
 }
 
 static int
 clear_constraints(pcmk__output_t *out, xmlNodePtr *cib_xml_copy)
 {
     GList *before = NULL;
     GList *after = NULL;
     GList *remaining = NULL;
     GList *ele = NULL;
     pe_node_t *dest = NULL;
     int rc = pcmk_rc_ok;
 
     if (!out->is_quiet(out)) {
         before = build_constraint_list(data_set->input);
     }
 
     if (options.clear_expired) {
         rc = cli_resource_clear_all_expired(data_set->input, cib_conn, options.cib_options,
                                             options.rsc_id, options.host_uname,
                                             options.promoted_role_only);
 
     } else if (options.host_uname) {
         dest = pe_find_node(data_set->nodes, options.host_uname);
         if (dest == NULL) {
             rc = pcmk_rc_node_unknown;
             if (!out->is_quiet(out)) {
                 g_list_free(before);
             }
             return rc;
         }
         rc = cli_resource_clear(options.rsc_id, dest->details->uname, NULL,
                                 cib_conn, options.cib_options, TRUE, options.force);
 
     } else {
         rc = cli_resource_clear(options.rsc_id, NULL, data_set->nodes,
                                 cib_conn, options.cib_options, TRUE, options.force);
     }
 
     if (!out->is_quiet(out)) {
         rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local | cib_sync_call);
         rc = pcmk_legacy2rc(rc);
 
         if (rc != pcmk_rc_ok) {
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         "Could not get modified CIB: %s\n", pcmk_strerror(rc));
             g_list_free(before);
             free_xml(*cib_xml_copy);
             *cib_xml_copy = NULL;
             return rc;
         }
 
         data_set->input = *cib_xml_copy;
         cluster_status(data_set);
 
         after = build_constraint_list(data_set->input);
         remaining = pcmk__subtract_lists(before, after, (GCompareFunc) strcmp);
 
         for (ele = remaining; ele != NULL; ele = ele->next) {
             out->info(out, "Removing constraint: %s", (char *) ele->data);
         }
 
         g_list_free(before);
         g_list_free(after);
         g_list_free(remaining);
     }
 
     return rc;
 }
 
 static int
 delete(void)
 {
     int rc = pcmk_rc_ok;
     xmlNode *msg_data = NULL;
 
     if (options.rsc_type == NULL) {
         rc = ENXIO;
         g_set_error(&error, PCMK__RC_ERROR, rc,
                     "You need to specify a resource type with -t");
         return rc;
     }
 
     msg_data = create_xml_node(NULL, options.rsc_type);
     crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);
 
     rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
                                 options.cib_options);
     rc = pcmk_legacy2rc(rc);
     free_xml(msg_data);
     return rc;
 }
 
 static int
 list_agents(pcmk__output_t *out, const char *agent_spec)
 {
     int rc = pcmk_rc_ok;
     char *provider = strchr(agent_spec, ':');
-    lrmd_t *lrmd_conn = lrmd_api_new();
+    lrmd_t *lrmd_conn = NULL;
     lrmd_list_t *list = NULL;
 
+    rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
+    if (rc != pcmk_rc_ok) {
+        goto error;
+    }
+
     if (provider) {
         *provider++ = 0;
     }
 
     rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, agent_spec, provider);
 
     if (rc > 0) {
         rc = out->message(out, "agents-list", list, agent_spec, provider);
     } else {
         rc = pcmk_rc_error;
     }
 
+error:
     if (rc != pcmk_rc_ok) {
         if (provider == NULL) {
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         "No agents found for standard '%s'", agent_spec);
         } else {
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         "No agents found for standard '%s' and provider '%s'",
                         agent_spec, provider);
         }
     }
 
     lrmd_api_delete(lrmd_conn);
     return rc;
 }
 
 static int
 list_providers(pcmk__output_t *out, const char *agent_spec)
 {
     int rc;
     const char *text = NULL;
-    lrmd_t *lrmd_conn = lrmd_api_new();
+    lrmd_t *lrmd_conn = NULL;
     lrmd_list_t *list = NULL;
 
+    rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
+    if (rc != pcmk_rc_ok) {
+        goto error;
+    }
+
     switch (options.rsc_cmd) {
         case cmd_list_alternatives:
             rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);
 
             if (rc > 0) {
                 rc = out->message(out, "alternatives-list", list, agent_spec);
             } else {
                 rc = pcmk_rc_error;
             }
 
             text = "OCF providers";
             break;
         case cmd_list_standards:
             rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
 
             if (rc > 0) {
                 rc = out->message(out, "standards-list", list);
             } else {
                 rc = pcmk_rc_error;
             }
 
             text = "standards";
             break;
         case cmd_list_providers:
             rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);
 
             if (rc > 0) {
                 rc = out->message(out, "providers-list", list, agent_spec);
             } else {
                 rc = pcmk_rc_error;
             }
 
             text = "OCF providers";
             break;
         default:
             g_set_error(&error, PCMK__RC_ERROR, pcmk_rc_error, "Bug");
             lrmd_api_delete(lrmd_conn);
             return pcmk_rc_error;
     }
 
+error:
     if (rc != pcmk_rc_ok) {
         if (agent_spec != NULL) {
             rc = ENXIO;
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         "No %s found for %s", text, agent_spec);
 
         } else {
             rc = ENXIO;
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         "No %s found", text);
         }
     }
 
     lrmd_api_delete(lrmd_conn);
     return rc;
 }
 
 static int
 populate_working_set(xmlNodePtr *cib_xml_copy)
 {
     int rc = pcmk_rc_ok;
 
     if (options.xml_file != NULL) {
         *cib_xml_copy = filename2xml(options.xml_file);
         if (*cib_xml_copy == NULL) {
             rc = pcmk_rc_cib_corrupt;
         }
     } else {
         rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local | cib_sync_call);
         rc = pcmk_legacy2rc(rc);
     }
 
     if (rc == pcmk_rc_ok) {
         data_set = pe_new_working_set();
         if (data_set == NULL) {
             rc = ENOMEM;
         } else {
             pe__set_working_set_flags(data_set,
                                       pe_flag_no_counts|pe_flag_no_compat);
             data_set->priv = out;
             rc = update_working_set_xml(data_set, cib_xml_copy);
         }
     }
 
     if (rc != pcmk_rc_ok) {
         free_xml(*cib_xml_copy);
         *cib_xml_copy = NULL;
         return rc;
     }
 
     cluster_status(data_set);
     return pcmk_rc_ok;
 }
 
 static int
 refresh(pcmk__output_t *out)
 {
     int rc = pcmk_rc_ok;
     const char *router_node = options.host_uname;
     int attr_options = pcmk__node_attr_none;
 
     if (options.host_uname) {
         pe_node_t *node = pe_find_node(data_set->nodes, options.host_uname);
 
         if (pe__is_guest_or_remote_node(node)) {
             node = pe__current_node(node->details->remote_rsc);
             if (node == NULL) {
                 rc = ENXIO;
                 g_set_error(&error, PCMK__RC_ERROR, rc,
                             "No cluster connection to Pacemaker Remote node %s detected",
                             options.host_uname);
                 return rc;
             }
             router_node = node->details->uname;
             attr_options |= pcmk__node_attr_remote;
         }
     }
 
     if (controld_api == NULL) {
         out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
                   options.host_uname? options.host_uname : "all nodes");
         rc = pcmk_rc_ok;
         return rc;
     }
 
     crm_debug("Re-checking the state of all resources on %s", options.host_uname?options.host_uname:"all nodes");
 
     rc = pcmk__node_attr_request_clear(NULL, options.host_uname,
                                        NULL, NULL, NULL,
                                        NULL, attr_options);
 
     if (pcmk_controld_api_reprobe(controld_api, options.host_uname,
                                   router_node) == pcmk_rc_ok) {
         start_mainloop(controld_api);
     }
 
     return rc;
 }
 
 static void
 refresh_resource(pcmk__output_t *out, pe_resource_t *rsc)
 {
     int rc = pcmk_rc_ok;
 
     if (options.force == FALSE) {
         rsc = uber_parent(rsc);
     }
 
     crm_debug("Re-checking the state of %s (%s requested) on %s",
               rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
     rc = cli_resource_delete(controld_api, options.host_uname, rsc, NULL,
                              0, FALSE, data_set, options.force);
 
     if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
         // Show any reasons why resource might stay stopped
         cli_resource_check(out, cib_conn, rsc);
     }
 
     if (rc == pcmk_rc_ok) {
         start_mainloop(controld_api);
     }
 }
 
 static int
 set_property(void)
 {
     int rc = pcmk_rc_ok;
     xmlNode *msg_data = NULL;
 
     if (pcmk__str_empty(options.rsc_type)) {
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     "Must specify -t with resource type");
         rc = ENXIO;
         return rc;
 
     } else if (pcmk__str_empty(options.prop_value)) {
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     "Must supply -v with new value");
         rc = ENXIO;
         return rc;
     }
 
     CRM_LOG_ASSERT(options.prop_name != NULL);
 
     msg_data = create_xml_node(NULL, options.rsc_type);
     crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);
     crm_xml_add(msg_data, options.prop_name, options.prop_value);
 
     rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
                                 options.cib_options);
     rc = pcmk_legacy2rc(rc);
     free_xml(msg_data);
 
     return rc;
 }
 
 static int
 show_metadata(pcmk__output_t *out, const char *agent_spec)
 {
     int rc = pcmk_rc_ok;
     char *standard = NULL;
     char *provider = NULL;
     char *type = NULL;
     char *metadata = NULL;
-    lrmd_t *lrmd_conn = lrmd_api_new();
+    lrmd_t *lrmd_conn = NULL;
+
+    rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
+    if (rc != pcmk_rc_ok) {
+        g_set_error(&error, PCMK__RC_ERROR, rc,
+                    "Could not create executor connection");
+        lrmd_api_delete(lrmd_conn);
+        return rc;
+    }
 
     rc = crm_parse_agent_spec(agent_spec, &standard, &provider, &type);
     rc = pcmk_legacy2rc(rc);
 
     if (rc == pcmk_rc_ok) {
         rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
                                            provider, type,
                                            &metadata, 0);
         rc = pcmk_legacy2rc(rc);
 
         if (metadata) {
             out->output_xml(out, "metadata", metadata);
         } else {
             /* We were given a validly formatted spec, but it doesn't necessarily
              * match up with anything that exists.  Use ENXIO as the return code
              * here because that maps to an exit code of CRM_EX_NOSUCH, which
              * probably is the most common reason to get here.
              */
             rc = ENXIO;
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         "Metadata query for %s failed: %s",
                         agent_spec, pcmk_rc_str(rc));
         }
     } else {
         rc = ENXIO;
         g_set_error(&error, PCMK__RC_ERROR, rc,
                     "'%s' is not a valid agent specification", agent_spec);
     }
 
     lrmd_api_delete(lrmd_conn);
     return rc;
 }
 
 static void
 validate_cmdline_config(void)
 {
     // Cannot use both --resource and command-line resource configuration
     if (options.rsc_id != NULL) {
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     "--resource cannot be used with --class, --agent, and --provider");
 
     // Not all commands support command-line resource configuration
     } else if (options.rsc_cmd != cmd_execute_agent) {
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     "--class, --agent, and --provider can only be used with "
                     "--validate and --force-*");
 
     // Not all of --class, --agent, and --provider need to be given.  Not all
     // classes support the concept of a provider.  Check that what we were given
     // is valid.
     } else if (pcmk__str_eq(options.v_class, "stonith", pcmk__str_none)) {
         if (options.v_provider != NULL) {
             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                         "stonith does not support providers");
 
         } else if (stonith_agent_exists(options.v_agent, 0) == FALSE) {
             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                         "%s is not a known stonith agent", options.v_agent ? options.v_agent : "");
         }
 
     } else if (resources_agent_exists(options.v_class, options.v_provider, options.v_agent) == FALSE) {
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     "%s:%s:%s is not a known resource",
                     options.v_class ? options.v_class : "",
                     options.v_provider ? options.v_provider : "",
                     options.v_agent ? options.v_agent : "");
     }
 
     if (error != NULL) {
         return;
     }
 
     if (options.cmdline_params == NULL) {
         options.cmdline_params = pcmk__strkey_table(free, free);
     }
     options.require_resource = FALSE;
     options.require_dataset = FALSE;
     options.require_cib = FALSE;
 }
 
 static GOptionContext *
 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     GOptionContext *context = NULL;
 
     GOptionEntry extra_prog_entries[] = {
         { "quiet", 'Q', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &(args->quiet),
           "Be less descriptive in output.",
           NULL },
         { "resource", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_id,
           "Resource ID",
           "ID" },
         { G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &options.remainder,
           NULL,
           NULL },
 
         { NULL }
     };
 
     const char *description = "Examples:\n\n"
                               "List the available OCF agents:\n\n"
                               "\t# crm_resource --list-agents ocf\n\n"
                               "List the available OCF agents from the linux-ha project:\n\n"
                               "\t# crm_resource --list-agents ocf:heartbeat\n\n"
                               "Move 'myResource' to a specific node:\n\n"
                               "\t# crm_resource --resource myResource --move --node altNode\n\n"
                               "Allow (but not force) 'myResource' to move back to its original "
                               "location:\n\n"
                               "\t# crm_resource --resource myResource --clear\n\n"
                               "Stop 'myResource' (and anything that depends on it):\n\n"
                               "\t# crm_resource --resource myResource --set-parameter target-role "
                               "--meta --parameter-value Stopped\n\n"
                               "Tell the cluster not to manage 'myResource' (the cluster will not "
                               "attempt to start or stop the\n"
                               "resource under any circumstances; useful when performing maintenance "
                               "tasks on a resource):\n\n"
                               "\t# crm_resource --resource myResource --set-parameter is-managed "
                               "--meta --parameter-value false\n\n"
                               "Erase the operation history of 'myResource' on 'aNode' (the cluster "
                               "will 'forget' the existing\n"
                               "resource state, including any errors, and attempt to recover the"
                               "resource; useful when a resource\n"
                               "had failed permanently and has been repaired by an administrator):\n\n"
                               "\t# crm_resource --resource myResource --cleanup --node aNode\n\n";
 
     context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
     g_option_context_set_description(context, description);
 
     /* Add the -Q option, which cannot be part of the globally supported options
      * because some tools use that flag for something else.
      */
     pcmk__add_main_args(context, extra_prog_entries);
 
     pcmk__add_arg_group(context, "queries", "Queries:",
                         "Show query help", query_entries);
     pcmk__add_arg_group(context, "commands", "Commands:",
                         "Show command help", command_entries);
     pcmk__add_arg_group(context, "locations", "Locations:",
                         "Show location help", location_entries);
     pcmk__add_arg_group(context, "advanced", "Advanced:",
                         "Show advanced option help", advanced_entries);
     pcmk__add_arg_group(context, "additional", "Additional Options:",
                         "Show additional options", addl_entries);
     return context;
 }
 
 int
 main(int argc, char **argv)
 {
     xmlNode *cib_xml_copy = NULL;
     pe_resource_t *rsc = NULL;
     pe_node_t *node = NULL;
     int rc = pcmk_rc_ok;
 
     GOptionGroup *output_group = NULL;
     gchar **processed_args = NULL;
     GOptionContext *context = NULL;
 
     /*
      * Parse command line arguments
      */
 
     args = pcmk__new_common_args(SUMMARY);
     processed_args = pcmk__cmdline_preproc(argv, "GHINSTdginpstuvx");
     context = build_arg_context(args, &output_group);
 
     pcmk__register_formats(output_group, formats);
     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
         exit_code = CRM_EX_USAGE;
         goto done;
     }
 
     pcmk__cli_init_logging("crm_resource", args->verbosity);
 
     rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
     if (rc != pcmk_rc_ok) {
         exit_code = CRM_EX_ERROR;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
                     args->output_ty, pcmk_rc_str(rc));
         goto done;
     }
 
     pe__register_messages(out);
     crm_resource_register_messages(out);
     lrmd__register_messages(out);
     pcmk__register_lib_messages(out);
 
     out->quiet = args->quiet;
 
     crm_log_args(argc, argv);
 
     /*
      * Validate option combinations
      */
 
     // If the user didn't explicitly specify a command, list resources
     if (options.rsc_cmd == cmd_none) {
         options.rsc_cmd = cmd_list_resources;
         options.require_resource = FALSE;
     }
 
     // --expired without --clear/-U doesn't make sense
     if (options.clear_expired && (options.rsc_cmd != cmd_clear)) {
         exit_code = CRM_EX_USAGE;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "--expired requires --clear or -U");
         goto done;
     }
 
     if ((options.remainder != NULL) && (options.override_params != NULL)) {
         // Commands that use positional arguments will create override_params
         for (gchar **s = options.remainder; *s; s++) {
             char *name = calloc(1, strlen(*s));
             char *value = calloc(1, strlen(*s));
             int rc = sscanf(*s, "%[^=]=%s", name, value);
 
             if (rc == 2) {
                 g_hash_table_replace(options.override_params, name, value);
 
             } else {
                 exit_code = CRM_EX_USAGE;
                 g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                             "Error parsing '%s' as a name=value pair",
                             argv[optind]);
                 free(value);
                 free(name);
                 goto done;
             }
         }
 
     } else if (options.remainder != NULL) {
         gchar **strv = NULL;
         gchar *msg = NULL;
         int i = 1;
         int len = 0;
 
         for (gchar **s = options.remainder; *s; s++) {
             len++;
         }
 
         CRM_ASSERT(len > 0);
 
         strv = calloc(len, sizeof(char *));
         strv[0] = strdup("non-option ARGV-elements:");
 
         for (gchar **s = options.remainder; *s; s++) {
             strv[i] = crm_strdup_printf("[%d of %d] %s\n", i, len, *s);
             i++;
         }
 
         exit_code = CRM_EX_USAGE;
         msg = g_strjoinv("", strv);
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
         g_free(msg);
 
         for(i = 0; i < len; i++) {
             free(strv[i]);
         }
         free(strv);
 
         goto done;
     }
 
     if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
         /* Kind of a hack to display XML lists using a real tag instead of <list>.  This just
          * saves from having to write custom messages to build the lists around all these things
          */
         switch (options.rsc_cmd) {
             case cmd_execute_agent:
             case cmd_list_resources:
             case cmd_query_xml:
             case cmd_query_raw_xml:
             case cmd_list_active_ops:
             case cmd_list_all_ops:
             case cmd_colocations:
             case cmd_colocations_deep:
                 pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
                 break;
 
             default:
                 pcmk__force_args(context, &error, "%s --xml-substitute", g_get_prgname());
                 break;
         }
     } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
         if (options.rsc_cmd == cmd_colocations || options.rsc_cmd == cmd_colocations_deep ||
             options.rsc_cmd == cmd_list_resources) {
             pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
         }
     }
 
     if (args->version) {
         out->version(out, false);
         goto done;
     }
 
     if (options.cmdline_config) {
         /* A resource configuration was given on the command line. Sanity-check
          * the values and set error if they don't make sense.
          */
         validate_cmdline_config();
         if (error != NULL) {
             exit_code = CRM_EX_USAGE;
             goto done;
         }
 
     } else if (options.cmdline_params != NULL) {
         // @COMPAT @TODO error out here when we can break backward compatibility
         g_hash_table_destroy(options.cmdline_params);
         options.cmdline_params = NULL;
     }
 
     if (options.require_resource && (options.rsc_id == NULL)) {
         exit_code = CRM_EX_USAGE;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                     "Must supply a resource id with -r");
         goto done;
     }
     if (options.require_node && (options.host_uname == NULL)) {
         exit_code = CRM_EX_USAGE;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                     "Must supply a node name with -N");
         goto done;
     }
 
     /*
      * Set up necessary connections
      */
 
     if (options.force) {
         crm_debug("Forcing...");
         cib__set_call_options(options.cib_options, crm_system_name,
                               cib_quorum_override);
     }
 
     if (options.find_flags && options.rsc_id) {
         options.require_dataset = TRUE;
     }
 
     // Establish a connection to the CIB if needed
     if (options.require_cib) {
         cib_conn = cib_new();
         if ((cib_conn == NULL) || (cib_conn->cmds == NULL)) {
             exit_code = CRM_EX_DISCONNECT;
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         "Could not create CIB connection");
             goto done;
         }
         rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
         rc = pcmk_legacy2rc(rc);
         if (rc != pcmk_rc_ok) {
             exit_code = pcmk_rc2exitc(rc);
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         "Could not connect to the CIB: %s", pcmk_rc_str(rc));
             goto done;
         }
     }
 
     /* Populate working set from XML file if specified or CIB query otherwise */
     if (options.require_dataset) {
         rc = populate_working_set(&cib_xml_copy);
         if (rc != pcmk_rc_ok) {
             exit_code = pcmk_rc2exitc(rc);
             goto done;
         }
     }
 
     // If command requires that resource exist if specified, find it
     if (options.find_flags && options.rsc_id) {
         rsc = pe_find_resource_with_flags(data_set->resources, options.rsc_id,
                                           options.find_flags);
         if (rsc == NULL) {
             exit_code = CRM_EX_NOSUCH;
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         "Resource '%s' not found", options.rsc_id);
             goto done;
         }
     }
 
     // If user supplied a node name, check whether it exists
     if ((options.host_uname != NULL) && (data_set != NULL)) {
         node = pe_find_node(data_set->nodes, options.host_uname);
     }
 
     // Establish a connection to the controller if needed
     if (options.require_crmd) {
         rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
         if (rc != pcmk_rc_ok) {
             exit_code = pcmk_rc2exitc(rc);
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         "Error connecting to the controller: %s", pcmk_rc_str(rc));
             goto done;
         }
         pcmk_register_ipc_callback(controld_api, controller_event_callback,
                                    NULL);
         rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
         if (rc != pcmk_rc_ok) {
             exit_code = pcmk_rc2exitc(rc);
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         "Error connecting to the controller: %s", pcmk_rc_str(rc));
             goto done;
         }
     }
 
     /*
      * Handle requested command
      */
 
     switch (options.rsc_cmd) {
         case cmd_list_resources: {
             GList *all = NULL;
             all = g_list_prepend(all, (gpointer) "*");
             rc = out->message(out, "resource-list", data_set,
                               pcmk_show_inactive_rscs | pcmk_show_rsc_only | pcmk_show_pending,
                               TRUE, all, all, FALSE);
             g_list_free(all);
 
             if (rc == pcmk_rc_no_output) {
                 rc = ENXIO;
             }
             break;
         }
 
         case cmd_list_instances:
             rc = out->message(out, "resource-names-list", data_set->resources);
 
             if (rc != pcmk_rc_ok) {
                 rc = ENXIO;
             }
 
             break;
 
         case cmd_list_standards:
         case cmd_list_providers:
         case cmd_list_alternatives:
             rc = list_providers(out, options.agent_spec);
             break;
 
         case cmd_list_agents:
             rc = list_agents(out, options.agent_spec);
             break;
 
         case cmd_metadata:
             rc = show_metadata(out, options.agent_spec);
             break;
 
         case cmd_restart:
             /* We don't pass data_set because rsc needs to stay valid for the
              * entire lifetime of cli_resource_restart(), but it will reset and
              * update the working set multiple times, so it needs to use its own
              * copy.
              */
             rc = cli_resource_restart(out, rsc, options.host_uname,
                                       options.move_lifetime, options.timeout_ms,
                                       cib_conn, options.cib_options,
                                       options.promoted_role_only,
                                       options.force);
             break;
 
         case cmd_wait:
             rc = wait_till_stable(out, options.timeout_ms, cib_conn);
             break;
 
         case cmd_execute_agent:
             if (options.cmdline_config) {
                 exit_code = cli_resource_execute_from_params(out, NULL,
                     options.v_class, options.v_provider, options.v_agent,
                     options.operation, options.cmdline_params,
                     options.override_params, options.timeout_ms,
                     args->verbosity, options.force, options.check_level);
             } else {
                 exit_code = cli_resource_execute(rsc, options.rsc_id,
                     options.operation, options.override_params,
                     options.timeout_ms, cib_conn, data_set,
                     args->verbosity, options.force, options.check_level);
             }
             goto done;
 
         case cmd_digests:
             node = pe_find_node(data_set->nodes, options.host_uname);
             if (node == NULL) {
                 rc = pcmk_rc_node_unknown;
             } else {
                 rc = pcmk__resource_digests(out, rsc, node,
                                             options.override_params, data_set);
             }
             break;
 
         case cmd_colocations:
             rc = out->message(out, "stacks-constraints", rsc, data_set, false);
             break;
 
         case cmd_colocations_deep:
             rc = out->message(out, "stacks-constraints", rsc, data_set, true);
             break;
 
         case cmd_cts:
             rc = pcmk_rc_ok;
             g_list_foreach(data_set->resources, (GFunc) cli_resource_print_cts, out);
             cli_resource_print_cts_constraints(data_set);
             break;
 
         case cmd_fail:
             rc = cli_resource_fail(controld_api, options.host_uname,
                                    options.rsc_id, data_set);
             if (rc == pcmk_rc_ok) {
                 start_mainloop(controld_api);
             }
             break;
 
         case cmd_list_active_ops:
             rc = cli_resource_print_operations(options.rsc_id,
                                                options.host_uname, TRUE,
                                                data_set);
             break;
 
         case cmd_list_all_ops:
             rc = cli_resource_print_operations(options.rsc_id,
                                                options.host_uname, FALSE,
                                                data_set);
             break;
 
         case cmd_locate: {
             GList *nodes = cli_resource_search(rsc, options.rsc_id, data_set);
             rc = out->message(out, "resource-search-list", nodes, options.rsc_id);
             g_list_free_full(nodes, free);
             break;
         }
 
         case cmd_query_xml:
             rc = cli_resource_print(rsc, data_set, TRUE);
             break;
 
         case cmd_query_raw_xml:
             rc = cli_resource_print(rsc, data_set, FALSE);
             break;
 
         case cmd_why:
             if ((options.host_uname != NULL) && (node == NULL)) {
                 rc = pcmk_rc_node_unknown;
             } else {
                 rc = out->message(out, "resource-reasons-list", cib_conn,
                                   data_set->resources, rsc, node);
             }
             break;
 
         case cmd_clear:
             rc = clear_constraints(out, &cib_xml_copy);
             break;
 
         case cmd_move:
             if (options.host_uname == NULL) {
                 rc = ban_or_move(out, rsc, options.move_lifetime);
             } else {
                 rc = cli_resource_move(rsc, options.rsc_id, options.host_uname,
                                        options.move_lifetime, cib_conn,
                                        options.cib_options, data_set,
                                        options.promoted_role_only,
                                        options.force);
             }
 
             if (rc == EINVAL) {
                 exit_code = CRM_EX_USAGE;
                 goto done;
             }
 
             break;
 
         case cmd_ban:
             if (options.host_uname == NULL) {
                 rc = ban_or_move(out, rsc, options.move_lifetime);
             } else if (node == NULL) {
                 rc = pcmk_rc_node_unknown;
             } else {
                 rc = cli_resource_ban(out, options.rsc_id, node->details->uname,
                                       options.move_lifetime, NULL, cib_conn,
                                       options.cib_options,
                                       options.promoted_role_only);
             }
 
             if (rc == EINVAL) {
                 exit_code = CRM_EX_USAGE;
                 goto done;
             }
 
             break;
 
         case cmd_get_property:
             rc = out->message(out, "property-list", rsc, options.prop_name);
             if (rc == pcmk_rc_no_output) {
                 rc = ENXIO;
             }
 
             break;
 
         case cmd_set_property:
             rc = set_property();
             break;
 
         case cmd_get_param: {
             unsigned int count = 0;
             GHashTable *params = NULL;
             pe_node_t *current = pe__find_active_on(rsc, &count, NULL);
             bool free_params = true;
 
             if (count > 1) {
                 out->err(out, "%s is active on more than one node,"
                          " returning the default value for %s", rsc->id, crm_str(options.prop_name));
                 current = NULL;
             }
 
             crm_debug("Looking up %s in %s", options.prop_name, rsc->id);
 
             if (pcmk__str_eq(options.attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
                 params = pe_rsc_params(rsc, current, data_set);
                 free_params = false;
 
             } else if (pcmk__str_eq(options.attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
                 params = pcmk__strkey_table(free, free);
                 get_meta_attributes(params, rsc, current, data_set);
 
             } else {
                 params = pcmk__strkey_table(free, free);
                 pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_UTILIZATION, NULL, params,
                                            NULL, FALSE, data_set);
             }
 
             rc = out->message(out, "attribute-list", rsc, options.prop_name, params);
             if (free_params) {
                 g_hash_table_destroy(params);
             }
             break;
         }
 
         case cmd_set_param:
             if (pcmk__str_empty(options.prop_value)) {
                 exit_code = CRM_EX_USAGE;
                 g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                             "You need to supply a value with the -v option");
                 goto done;
             }
 
             /* coverity[var_deref_model] False positive */
             rc = cli_resource_update_attribute(rsc, options.rsc_id,
                                                options.prop_set,
                                                options.attr_set_type,
                                                options.prop_id,
                                                options.prop_name,
                                                options.prop_value,
                                                options.recursive, cib_conn,
                                                options.cib_options, data_set,
                                                options.force);
             break;
 
         case cmd_delete_param:
             /* coverity[var_deref_model] False positive */
             rc = cli_resource_delete_attribute(rsc, options.rsc_id,
                                                options.prop_set,
                                                options.attr_set_type,
                                                options.prop_id,
                                                options.prop_name, cib_conn,
                                                options.cib_options, data_set,
                                                options.force);
             break;
 
         case cmd_cleanup:
             if (rsc == NULL) {
                 rc = cli_cleanup_all(controld_api, options.host_uname,
                                      options.operation, options.interval_spec,
                                      data_set);
                 if (rc == pcmk_rc_ok) {
                     start_mainloop(controld_api);
                 }
             } else {
                 cleanup(out, rsc);
             }
             break;
 
         case cmd_refresh:
             if (rsc == NULL) {
                 rc = refresh(out);
             } else {
                 refresh_resource(out, rsc);
             }
             break;
 
         case cmd_delete:
             rc = delete();
             break;
 
         default:
             exit_code = CRM_EX_USAGE;
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         "Unimplemented command: %d", (int) options.rsc_cmd);
             goto done;
     }
 
     /* Convert rc into an exit code. */
     if (rc != pcmk_rc_ok && rc != pcmk_rc_no_output) {
         if (rc == pcmk_rc_no_quorum) {
             g_prefix_error(&error, "To ignore quorum, use the force option.\n");
         }
 
         exit_code = pcmk_rc2exitc(rc);
     }
 
     /*
      * Clean up and exit
      */
 
 done:
     /* When we get here, exit_code has been set one of two ways - either at one of
      * the spots where there's a "goto done" (which itself could have happened either
      * directly or by calling pcmk_rc2exitc), or just up above after any of the break
      * statements.
      *
      * Thus, we can use just exit_code here to decide what to do.
      */
     if (exit_code != CRM_EX_OK && exit_code != CRM_EX_USAGE) {
         if (error != NULL) {
             char *msg = crm_strdup_printf("%s\nError performing operation: %s",
                                           error->message, crm_exit_str(exit_code));
             g_clear_error(&error);
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
             free(msg);
         } else {
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         "Error performing operation: %s", crm_exit_str(exit_code));
         }
     }
 
     g_free(options.host_uname);
     g_free(options.interval_spec);
     g_free(options.move_lifetime);
     g_free(options.operation);
     g_free(options.prop_id);
     free(options.prop_name);
     g_free(options.prop_set);
     g_free(options.prop_value);
     g_free(options.rsc_id);
     g_free(options.rsc_type);
     free(options.agent_spec);
     free(options.v_agent);
     free(options.v_class);
     free(options.v_provider);
     g_free(options.xml_file);
     g_strfreev(options.remainder);
 
     if (options.override_params != NULL) {
         g_hash_table_destroy(options.override_params);
     }
 
     /* options.cmdline_params does not need to be destroyed here.  See the
      * comments in cli_resource_execute_from_params.
      */
 
     g_strfreev(processed_args);
     g_option_context_free(context);
 
     return bye(exit_code);
 }
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 13b78b6b93..582687c493 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -1,1992 +1,1981 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm_resource.h>
 #include <crm/common/ipc_controld.h>
 #include <crm/common/lists_internal.h>
+#include <crm/services_internal.h>
 
 resource_checks_t *
 cli_check_resource(pe_resource_t *rsc, char *role_s, char *managed)
 {
     pe_resource_t *parent = uber_parent(rsc);
     resource_checks_t *rc = calloc(1, sizeof(resource_checks_t));
 
     if (role_s) {
         enum rsc_role_e role = text2role(role_s);
 
         if (role == RSC_ROLE_STOPPED) {
             rc->flags |= rsc_remain_stopped;
         } else if (pcmk_is_set(parent->flags, pe_rsc_promotable) &&
                    (role == RSC_ROLE_UNPROMOTED)) {
             rc->flags |= rsc_unpromotable;
         }
     }
 
     if (managed && !crm_is_true(managed)) {
         rc->flags |= rsc_unmanaged;
     }
 
     if (rsc->lock_node) {
         rc->lock_node = rsc->lock_node->details->uname;
     }
 
     rc->rsc = rsc;
     return rc;
 }
 
 static GList *
 build_node_info_list(pe_resource_t *rsc)
 {
     GList *retval = NULL;
 
     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
         pe_resource_t *child = (pe_resource_t *) iter->data;
 
         for (GList *iter2 = child->running_on; iter2 != NULL; iter2 = iter2->next) {
             pe_node_t *node = (pe_node_t *) iter2->data;
             node_info_t *ni = calloc(1, sizeof(node_info_t));
             ni->node_name = node->details->uname;
             ni->promoted = pcmk_is_set(rsc->flags, pe_rsc_promotable) &&
                            child->fns->state(child, TRUE) == RSC_ROLE_PROMOTED;
 
             retval = g_list_prepend(retval, ni);
         }
     }
 
     return retval;
 }
 
 GList *
 cli_resource_search(pe_resource_t *rsc, const char *requested_name,
                     pe_working_set_t *data_set)
 {
     GList *retval = NULL;
     pe_resource_t *parent = uber_parent(rsc);
 
     if (pe_rsc_is_clone(rsc)) {
         retval = build_node_info_list(rsc);
 
     /* The anonymous clone children's common ID is supplied */
     } else if (pe_rsc_is_clone(parent)
                && !pcmk_is_set(rsc->flags, pe_rsc_unique)
                && rsc->clone_name
                && pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
                && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {
 
         retval = build_node_info_list(parent);
 
     } else if (rsc->running_on != NULL) {
         for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
             pe_node_t *node = (pe_node_t *) iter->data;
             node_info_t *ni = calloc(1, sizeof(node_info_t));
             ni->node_name = node->details->uname;
             ni->promoted = (rsc->fns->state(rsc, TRUE) == RSC_ROLE_PROMOTED);
 
             retval = g_list_prepend(retval, ni);
         }
     }
 
     return retval;
 }
 
 #define XPATH_MAX 1024
 
 // \return Standard Pacemaker return code
 static int
 find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
                    const char *rsc, const char *attr_set_type, const char *set_name,
                    const char *attr_id, const char *attr_name, char **value)
 {
     int offset = 0;
     int rc = pcmk_rc_ok;
     xmlNode *xml_search = NULL;
     char *xpath_string = NULL;
 
     if(value) {
         *value = NULL;
     }
 
     if(the_cib == NULL) {
         return ENOTCONN;
     }
 
     xpath_string = calloc(1, XPATH_MAX);
     offset +=
         snprintf(xpath_string + offset, XPATH_MAX - offset, "%s", get_object_path("resources"));
 
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//*[@id=\"%s\"]", rsc);
 
     if (attr_set_type) {
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s", attr_set_type);
         if (set_name) {
             offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "[@id=\"%s\"]", set_name);
         }
     }
 
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//nvpair[");
     if (attr_id) {
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@id=\"%s\"", attr_id);
     }
 
     if (attr_name) {
         if (attr_id) {
             offset += snprintf(xpath_string + offset, XPATH_MAX - offset, " and ");
         }
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@name=\"%s\"", attr_name);
     }
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "]");
     CRM_LOG_ASSERT(offset > 0);
 
     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
     rc = pcmk_legacy2rc(rc);
 
     if (rc != pcmk_rc_ok) {
         goto done;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_has_children(xml_search)) {
         xmlNode *child = NULL;
 
         rc = ENOTUNIQ;
         out->info(out, "Multiple attributes match name=%s", attr_name);
 
         for (child = pcmk__xml_first_child(xml_search); child != NULL;
              child = pcmk__xml_next(child)) {
             out->info(out, "  Value: %s \t(id=%s)",
                       crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
         }
 
         out->spacer(out);
 
     } else if(value) {
         const char *tmp = crm_element_value(xml_search, attr);
 
         if (tmp) {
             *value = strdup(tmp);
         }
     }
 
   done:
     free(xpath_string);
     free_xml(xml_search);
     return rc;
 }
 
 /* PRIVATE. Use the find_matching_attr_resources instead. */
 static void
 find_matching_attr_resources_recursive(pcmk__output_t *out, GList/* <pe_resource_t*> */ ** result,
                                        pe_resource_t * rsc, const char * rsc_id,
                                        const char * attr_set, const char * attr_set_type,
                                        const char * attr_id, const char * attr_name,
                                        cib_t * cib, const char * cmd, int depth)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = clone_strip(rsc->id);
     char *local_attr_id = NULL;
 
     /* visit the children */
     for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
         find_matching_attr_resources_recursive(out, result, (pe_resource_t*)gIter->data,
                                                rsc_id, attr_set, attr_set_type,
                                                attr_id, attr_name, cib, cmd, depth+1);
         /* do it only once for clones */
         if(pe_clone == rsc->variant) {
             break;
         }
     }
 
     rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
                             attr_set, attr_id, attr_name, &local_attr_id);
     /* Post-order traversal. 
      * The root is always on the list and it is the last item. */
     if((0 == depth) || (pcmk_rc_ok == rc)) {
         /* push the head */
         *result = g_list_append(*result, rsc);
     }
 
     free(local_attr_id);
     free(lookup_id);
 }
 
 
 /* The result is a linearized pre-ordered tree of resources. */
 static GList/*<pe_resource_t*>*/ *
 find_matching_attr_resources(pcmk__output_t *out, pe_resource_t * rsc,
                              const char * rsc_id, const char * attr_set,
                              const char * attr_set_type, const char * attr_id,
                              const char * attr_name, cib_t * cib, const char * cmd,
                              gboolean force)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = NULL;
     char *local_attr_id = NULL;
     GList * result = NULL;
     /* If --force is used, update only the requested resource (clone or primitive).
      * Otherwise, if the primitive has the attribute, use that.
      * Otherwise use the clone. */
     if(force == TRUE) {
         return g_list_append(result, rsc);
     }
     if(rsc->parent && pe_clone == rsc->parent->variant) {
         int rc = pcmk_rc_ok;
         char *local_attr_id = NULL;
         rc = find_resource_attr(out, cib, XML_ATTR_ID, rsc_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &local_attr_id);
         free(local_attr_id);
 
         if(rc != pcmk_rc_ok) {
             rsc = rsc->parent;
             out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
                       cmd, attr_name, rsc->id, rsc_id);
         }
         return g_list_append(result, rsc);
     } else if(rsc->parent == NULL && rsc->children && pe_clone == rsc->variant) {
         pe_resource_t *child = rsc->children->data;
 
         if(child->variant == pe_native) {
             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
             rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
                                     attr_set, attr_id, attr_name, &local_attr_id);
 
             if(rc == pcmk_rc_ok) {
                 rsc = child;
                 out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
                           attr_name, lookup_id, cmd, rsc_id);
             }
 
             free(local_attr_id);
             free(lookup_id);
         }
         return g_list_append(result, rsc);
     }
     /* If the resource is a group ==> children inherit the attribute if defined. */
     find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
                                            attr_set_type, attr_id, attr_name,
                                            cib, cmd, 0);
     return result;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_set_type,
                               const char *attr_id, const char *attr_name,
                               const char *attr_value, gboolean recursive,
                               cib_t *cib, int cib_options,
                               pe_working_set_t *data_set, gboolean force)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     static bool need_init = TRUE;
 
     char *local_attr_id = NULL;
     char *local_attr_set = NULL;
 
     GList/*<pe_resource_t*>*/ *resources = NULL;
     const char *common_attr_id = attr_id;
 
     if (attr_id == NULL && force == FALSE) {
         find_resource_attr (out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
                             NULL, NULL, attr_name, NULL);
     }
 
     if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
         if (force == FALSE) {
             rc = find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id,
                                     XML_TAG_META_SETS, attr_set, attr_id,
                                     attr_name, &local_attr_id);
             if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
                 out->err(out, "WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)",
                          uber_parent(rsc)->id, attr_name, local_attr_id);
                 out->err(out, "         Delete '%s' first or use the force option to override",
                          local_attr_id);
             }
             free(local_attr_id);
             if (rc == pcmk_rc_ok) {
                 return ENOTUNIQ;
             }
         }
         resources = g_list_append(resources, rsc);
 
     } else {
         resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
                                                  attr_id, attr_name, cib, "update", force);
     }
 
     /* If either attr_set or attr_id is specified,
      * one clearly intends to modify a single resource.
      * It is the last item on the resource list.*/
     for(GList *gIter = (attr_set||attr_id) ? g_list_last(resources) : resources
             ; gIter; gIter = gIter->next) {
         char *lookup_id = NULL;
 
         xmlNode *xml_top = NULL;
         xmlNode *xml_obj = NULL;
         local_attr_id = NULL;
         local_attr_set = NULL;
 
         rsc = (pe_resource_t*)gIter->data;
         attr_id = common_attr_id;
 
         lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
         rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &local_attr_id);
 
         if (rc == pcmk_rc_ok) {
             crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
             attr_id = local_attr_id;
 
         } else if (rc != ENXIO) {
             free(lookup_id);
             free(local_attr_id);
             g_list_free(resources);
             return rc;
 
         } else {
             const char *tag = crm_element_name(rsc->xml);
 
             if (attr_set == NULL) {
                 local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
                                                    attr_set_type);
                 attr_set = local_attr_set;
             }
             if (attr_id == NULL) {
                 local_attr_id = crm_strdup_printf("%s-%s", attr_set, attr_name);
                 attr_id = local_attr_id;
             }
 
             xml_top = create_xml_node(NULL, tag);
             crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);
 
             xml_obj = create_xml_node(xml_top, attr_set_type);
             crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
         }
 
         xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value);
         if (xml_top == NULL) {
             xml_top = xml_obj;
         }
 
         crm_log_xml_debug(xml_top, "Update");
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
         rc = pcmk_legacy2rc(rc);
 
         if (rc == pcmk_rc_ok) {
             out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s", lookup_id, local_attr_id,
                       attr_set ? " set=" : "", attr_set ? attr_set : "",
                       attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
         }
 
         free_xml(xml_top);
 
         free(lookup_id);
         free(local_attr_id);
         free(local_attr_set);
 
         if(recursive && pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
             GList *lpc = NULL;
 
             if(need_init) {
                 xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 
                 need_init = FALSE;
                 unpack_constraints(cib_constraints, data_set);
 
                 pe__clear_resource_flags_on_all(data_set, pe_rsc_allocating);
             }
 
             crm_debug("Looking for dependencies %p", rsc->rsc_cons_lhs);
             pe__set_resource_flags(rsc, pe_rsc_allocating);
             for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
                 pcmk__colocation_t *cons = (pcmk__colocation_t *) lpc->data;
                 pe_resource_t *peer = cons->rsc_lh;
 
                 crm_debug("Checking %s %d", cons->id, cons->score);
                 if (cons->score > 0 && !pcmk_is_set(peer->flags, pe_rsc_allocating)) {
                     /* Don't get into colocation loops */
                     crm_debug("Setting %s=%s for dependent resource %s", attr_name, attr_value, peer->id);
                     cli_resource_update_attribute(peer, peer->id, NULL, attr_set_type,
                                                   NULL, attr_name, attr_value, recursive,
                                                   cib, cib_options, data_set, force);
                 }
             }
         }
     }
     g_list_free(resources);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_set_type,
                               const char *attr_id, const char *attr_name,
                               cib_t *cib, int cib_options,
                               pe_working_set_t *data_set, gboolean force)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     GList/*<pe_resource_t*>*/ *resources = NULL;
 
     if (attr_id == NULL && force == FALSE) {
         find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
                            NULL, NULL, attr_name, NULL);
     }
 
     if(pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
         resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
                                                  attr_id, attr_name, cib, "delete", force);
     } else {
         resources = g_list_append(resources, rsc);
     }
 
     for(GList *gIter = resources; gIter; gIter = gIter->next) {
         char *lookup_id = NULL;
         xmlNode *xml_obj = NULL;
         char *local_attr_id = NULL;
 
         rsc = (pe_resource_t*)gIter->data;
 
         lookup_id = clone_strip(rsc->id);
         rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &local_attr_id);
 
         if (rc == ENXIO) {
             free(lookup_id);
             rc = pcmk_rc_ok;
             continue;
 
         } else if (rc != pcmk_rc_ok) {
             free(lookup_id);
             g_list_free(resources);
             return rc;
         }
 
         if (attr_id == NULL) {
             attr_id = local_attr_id;
         }
 
         xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, NULL);
         crm_log_xml_debug(xml_obj, "Delete");
 
         CRM_ASSERT(cib);
         rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
         rc = pcmk_legacy2rc(rc);
 
         if (rc == pcmk_rc_ok) {
             out->info(out, "Deleted '%s' option: id=%s%s%s%s%s", lookup_id, local_attr_id,
                       attr_set ? " set=" : "", attr_set ? attr_set : "",
                       attr_name ? " name=" : "", attr_name ? attr_name : "");
         }
 
         free(lookup_id);
         free_xml(xml_obj);
         free(local_attr_id);
     }
     g_list_free(resources);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
                 const char *host_uname, const char *rsc_id, pe_working_set_t *data_set)
 {
     pcmk__output_t *out = data_set->priv;
     const char *router_node = host_uname;
     const char *rsc_api_id = NULL;
     const char *rsc_long_id = NULL;
     const char *rsc_class = NULL;
     const char *rsc_provider = NULL;
     const char *rsc_type = NULL;
     bool cib_only = false;
     pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
         out->err(out, "Resource %s not found", rsc_id);
         return ENXIO;
 
     } else if (rsc->variant != pe_native) {
         out->err(out, "We can only process primitive resources, not %s", rsc_id);
         return EINVAL;
     }
 
     rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
     rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
     if ((rsc_class == NULL) || (rsc_type == NULL)) {
         out->err(out, "Resource %s does not have a class and type", rsc_id);
         return EINVAL;
     }
 
     {
         pe_node_t *node = pe_find_node(data_set->nodes, host_uname);
 
         if (node == NULL) {
             out->err(out, "Node %s not found", host_uname);
             return pcmk_rc_node_unknown;
         }
 
         if (!(node->details->online)) {
             if (do_fail_resource) {
                 out->err(out, "Node %s is not online", host_uname);
                 return ENOTCONN;
             } else {
                 cib_only = true;
             }
         }
         if (!cib_only && pe__is_guest_or_remote_node(node)) {
             node = pe__current_node(node->details->remote_rsc);
             if (node == NULL) {
                 out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
                          host_uname);
                 return ENOTCONN;
             }
             router_node = node->details->uname;
         }
     }
 
     if (rsc->clone_name) {
         rsc_api_id = rsc->clone_name;
         rsc_long_id = rsc->id;
     } else {
         rsc_api_id = rsc->id;
     }
     if (do_fail_resource) {
         return pcmk_controld_api_fail(controld_api, host_uname, router_node,
                                       rsc_api_id, rsc_long_id,
                                       rsc_class, rsc_provider, rsc_type);
     } else {
         return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
                                          rsc_api_id, rsc_long_id, rsc_class,
                                          rsc_provider, rsc_type, cib_only);
     }
 }
 
 /*!
  * \internal
  * \brief Get resource name as used in failure-related node attributes
  *
  * \param[in] rsc  Resource to check
  *
  * \return Newly allocated string containing resource's fail name
  * \note The caller is responsible for freeing the result.
  */
 static inline char *
 rsc_fail_name(pe_resource_t *rsc)
 {
     const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
 
     return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
                   const char *rsc_id, pe_working_set_t *data_set)
 {
     int rc = pcmk_rc_ok;
 
     /* Erase the resource's entire LRM history in the CIB, even if we're only
      * clearing a single operation's fail count. If we erased only entries for a
      * single operation, we might wind up with a wrong idea of the current
      * resource state, and we might not re-probe the resource.
      */
     rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, data_set);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     crm_trace("Processing %d mainloop inputs",
               pcmk_controld_api_replies_expected(controld_api));
     while (g_main_context_iteration(NULL, FALSE)) {
         crm_trace("Processed mainloop input, %d still remaining",
                   pcmk_controld_api_replies_expected(controld_api));
     }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
                    const char *node_name, const char *rsc_id, const char *operation,
                    const char *interval_spec, pe_working_set_t *data_set)
 {
     int rc = pcmk_rc_ok;
     const char *failed_value = NULL;
     const char *failed_id = NULL;
     const char *interval_ms_s = NULL;
     GHashTable *rscs = NULL;
     GHashTableIter iter;
 
     /* Create a hash table to use as a set of resources to clean. This lets us
      * clean each resource only once (per node) regardless of how many failed
      * operations it has.
      */
     rscs = pcmk__strkey_table(NULL, NULL);
 
     // Normalize interval to milliseconds for comparison to history entry
     if (operation) {
         interval_ms_s = crm_strdup_printf("%u",
                                           crm_parse_interval_spec(interval_spec));
     }
 
     for (xmlNode *xml_op = pcmk__xml_first_child(data_set->failed);
          xml_op != NULL;
          xml_op = pcmk__xml_next(xml_op)) {
 
         failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
         if (failed_id == NULL) {
             // Malformed history entry, should never happen
             continue;
         }
 
         // No resource specified means all resources match
         if (rsc_id) {
             pe_resource_t *fail_rsc = pe_find_resource_with_flags(data_set->resources,
                                                                   failed_id,
                                                                   pe_find_renamed|pe_find_anon);
 
             if (!fail_rsc || !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
                 continue;
             }
         }
 
         // Host name should always have been provided by this point
         failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
         if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
             continue;
         }
 
         // No operation specified means all operations match
         if (operation) {
             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
             if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
                 continue;
             }
 
             // Interval (if operation was specified) defaults to 0 (not all)
             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
             if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
                 continue;
             }
         }
 
         g_hash_table_add(rscs, (gpointer) failed_id);
     }
 
     g_hash_table_iter_init(&iter, rscs);
     while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
         crm_debug("Erasing failures of %s on %s", failed_id, node_name);
         rc = clear_rsc_history(controld_api, node_name, failed_id, data_set);
         if (rc != pcmk_rc_ok) {
             return rc;
         }
     }
     g_hash_table_destroy(rscs);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_fail_attrs(pe_resource_t *rsc, const char *operation,
                      const char *interval_spec, pe_node_t *node)
 {
     int rc = pcmk_rc_ok;
     int attr_options = pcmk__node_attr_none;
     char *rsc_name = rsc_fail_name(rsc);
 
     if (pe__is_guest_or_remote_node(node)) {
         attr_options |= pcmk__node_attr_remote;
     }
     rc = pcmk__node_attr_request_clear(NULL, node->details->uname, rsc_name,
                                        operation, interval_spec, NULL,
                                        attr_options);
     free(rsc_name);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
                     pe_resource_t *rsc, const char *operation,
                     const char *interval_spec, bool just_failures,
                     pe_working_set_t *data_set, gboolean force)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     pe_node_t *node = NULL;
 
     if (rsc == NULL) {
         return ENXIO;
 
     } else if (rsc->children) {
         GList *lpc = NULL;
 
         for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
             pe_resource_t *child = (pe_resource_t *) lpc->data;
 
             rc = cli_resource_delete(controld_api, host_uname, child, operation,
                                      interval_spec, just_failures, data_set,
                                      force);
             if (rc != pcmk_rc_ok) {
                 return rc;
             }
         }
         return pcmk_rc_ok;
 
     } else if (host_uname == NULL) {
         GList *lpc = NULL;
         GList *nodes = g_hash_table_get_values(rsc->known_on);
 
         if(nodes == NULL && force) {
             nodes = pcmk__copy_node_list(data_set->nodes, false);
 
         } else if(nodes == NULL && rsc->exclusive_discover) {
             GHashTableIter iter;
             pe_node_t *node = NULL;
 
             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
             while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
                 if(node->weight >= 0) {
                     nodes = g_list_prepend(nodes, node);
                 }
             }
 
         } else if(nodes == NULL) {
             nodes = g_hash_table_get_values(rsc->allowed_nodes);
         }
 
         for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
             node = (pe_node_t *) lpc->data;
 
             if (node->details->online) {
                 rc = cli_resource_delete(controld_api, node->details->uname,
                                          rsc, operation, interval_spec,
                                          just_failures, data_set, force);
             }
             if (rc != pcmk_rc_ok) {
                 g_list_free(nodes);
                 return rc;
             }
         }
 
         g_list_free(nodes);
         return pcmk_rc_ok;
     }
 
     node = pe_find_node(data_set->nodes, host_uname);
 
     if (node == NULL) {
         out->err(out, "Unable to clean up %s because node %s not found",
                  rsc->id, host_uname);
         return ENODEV;
     }
 
     if (!node->details->rsc_discovery_enabled) {
         out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
                  rsc->id, host_uname);
         return EOPNOTSUPP;
     }
 
     if (controld_api == NULL) {
         out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
                  rsc->id, host_uname);
         return pcmk_rc_ok;
     }
 
     rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Unable to clean up %s failures on %s: %s",
                  rsc->id, host_uname, pcmk_rc_str(rc));
         return rc;
     }
 
     if (just_failures) {
         rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
                                 interval_spec, data_set);
     } else {
         rc = clear_rsc_history(controld_api, host_uname, rsc->id, data_set);
     }
     if (rc != pcmk_rc_ok) {
         out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
                  rsc->id, host_uname, pcmk_strerror(rc));
     } else {
         out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
     }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
                 const char *operation, const char *interval_spec,
                 pe_working_set_t *data_set)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     int attr_options = pcmk__node_attr_none;
     const char *display_name = node_name? node_name : "all nodes";
 
     if (controld_api == NULL) {
         out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
                   display_name);
         return rc;
     }
 
     if (node_name) {
         pe_node_t *node = pe_find_node(data_set->nodes, node_name);
 
         if (node == NULL) {
             out->err(out, "Unknown node: %s", node_name);
             return ENXIO;
         }
         if (pe__is_guest_or_remote_node(node)) {
             attr_options |= pcmk__node_attr_remote;
         }
     }
 
     rc = pcmk__node_attr_request_clear(NULL, node_name, NULL, operation,
                                        interval_spec, NULL, attr_options);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Unable to clean up all failures on %s: %s",
                  display_name, pcmk_rc_str(rc));
         return rc;
     }
 
     if (node_name) {
         rc = clear_rsc_failures(out, controld_api, node_name, NULL,
                                 operation, interval_spec, data_set);
         if (rc != pcmk_rc_ok) {
             out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
                      node_name, pcmk_strerror(rc));
             return rc;
         }
     } else {
         for (GList *iter = data_set->nodes; iter; iter = iter->next) {
             pe_node_t *node = (pe_node_t *) iter->data;
 
             rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
                                     operation, interval_spec, data_set);
             if (rc != pcmk_rc_ok) {
                 out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
                          pcmk_strerror(rc));
                 return rc;
             }
         }
     }
 
     out->info(out, "Cleaned up all resources on %s", display_name);
     return rc;
 }
 
 int
 cli_resource_check(pcmk__output_t *out, cib_t * cib_conn, pe_resource_t *rsc)
 {
     char *role_s = NULL;
     char *managed = NULL;
     pe_resource_t *parent = uber_parent(rsc);
     int rc = pcmk_rc_no_output;
     resource_checks_t *checks = NULL;
 
     find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);
 
     find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);
 
     checks = cli_check_resource(rsc, role_s, managed);
 
     if (checks->flags != 0 || checks->lock_node != NULL) {
         rc = out->message(out, "resource-check-list", checks);
     }
 
     free(role_s);
     free(managed);
     free(checks);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
                   const char *rsc_id, pe_working_set_t *data_set)
 {
     crm_notice("Failing %s on %s", rsc_id, host_uname);
     return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, data_set);
 }
 
 static GHashTable *
 generate_resource_params(pe_resource_t *rsc, pe_node_t *node,
                          pe_working_set_t *data_set)
 {
     GHashTable *params = NULL;
     GHashTable *meta = NULL;
     GHashTable *combined = NULL;
     GHashTableIter iter;
     char *key = NULL;
     char *value = NULL;
 
     combined = pcmk__strkey_table(free, free);
 
     params = pe_rsc_params(rsc, node, data_set);
     if (params != NULL) {
         g_hash_table_iter_init(&iter, params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             g_hash_table_insert(combined, strdup(key), strdup(value));
         }
     }
 
     meta = pcmk__strkey_table(free, free);
     get_meta_attributes(meta, rsc, node, data_set);
     if (meta != NULL) {
         g_hash_table_iter_init(&iter, meta);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             char *crm_name = crm_meta_name(key);
 
             g_hash_table_insert(combined, crm_name, strdup(value));
         }
         g_hash_table_destroy(meta);
     }
 
     return combined;
 }
 
 bool resource_is_running_on(pe_resource_t *rsc, const char *host)
 {
     bool found = TRUE;
     GList *hIter = NULL;
     GList *hosts = NULL;
 
     if(rsc == NULL) {
         return FALSE;
     }
 
     rsc->fns->location(rsc, &hosts, TRUE);
     for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
         pe_node_t *node = (pe_node_t *) hIter->data;
 
         if(strcmp(host, node->details->uname) == 0) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         } else if(strcmp(host, node->details->id) == 0) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         }
     }
 
     if(host != NULL) {
         crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
         found = FALSE;
 
     } else if(host == NULL && hosts == NULL) {
         crm_trace("Resource %s is not running\n", rsc->id);
         found = FALSE;
     }
 
   done:
     g_list_free(hosts);
     return found;
 }
 
 /*!
  * \internal
  * \brief Create a list of all resources active on host from a given list
  *
  * \param[in] host      Name of host to check whether resources are active
  * \param[in] rsc_list  List of resources to check
  *
  * \return New list of resources from list that are active on host
  */
 static GList *
 get_active_resources(const char *host, GList *rsc_list)
 {
     GList *rIter = NULL;
     GList *active = NULL;
 
     for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) rIter->data;
 
         /* Expand groups to their members, because if we're restarting a member
          * other than the first, we can't otherwise tell which resources are
          * stopping and starting.
          */
         if (rsc->variant == pe_group) {
             active = g_list_concat(active,
                                    get_active_resources(host, rsc->children));
         } else if (resource_is_running_on(rsc, host)) {
             active = g_list_append(active, strdup(rsc->id));
         }
     }
     return active;
 }
 
 static void dump_list(GList *items, const char *tag) 
 {
     int lpc = 0;
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
         lpc++;
     }
 }
 
 static void display_list(pcmk__output_t *out, GList *items, const char *tag)
 {
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         out->info(out, "%s%s", tag, (const char *)item->data);
     }
 }
 
 /*!
  * \internal
  * \brief Upgrade XML to latest schema version and use it as working set input
  *
  * This also updates the working set timestamp to the current time.
  *
  * \param[in] data_set   Working set instance to update
  * \param[in] xml        XML to use as input
  *
  * \return Standard Pacemaker return code
  * \note On success, caller is responsible for freeing memory allocated for
  *       data_set->now.
  * \todo This follows the example of other callers of cli_config_update()
  *       and returns ENOKEY ("Required key not available") if that fails,
  *       but perhaps pcmk_rc_schema_validation would be better in that case.
  */
 int
 update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml)
 {
     if (cli_config_update(xml, NULL, FALSE) == FALSE) {
         return ENOKEY;
     }
     data_set->input = *xml;
     data_set->now = crm_time_new(NULL);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Update a working set's XML input based on a CIB query
  *
  * \param[in] data_set   Data set instance to initialize
  * \param[in] cib        Connection to the CIB manager
  *
  * \return Standard Pacemaker return code
  * \note On success, caller is responsible for freeing memory allocated for
  *       data_set->input and data_set->now.
  */
 static int
 update_working_set_from_cib(pcmk__output_t *out, pe_working_set_t * data_set,
                             cib_t *cib)
 {
     xmlNode *cib_xml_copy = NULL;
     int rc = pcmk_rc_ok;
 
     rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
     rc = pcmk_legacy2rc(rc);
 
     if (rc != pcmk_rc_ok) {
         out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_strerror(rc), rc);
         return rc;
     }
     rc = update_working_set_xml(data_set, &cib_xml_copy);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Could not upgrade the current CIB XML");
         free_xml(cib_xml_copy);
         return rc;
     }
 
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
 {
     char *pid = NULL;
     char *shadow_file = NULL;
     cib_t *shadow_cib = NULL;
     int rc = pcmk_rc_ok;
 
     pcmk__output_t *out = data_set->priv;
 
     pe_reset_working_set(data_set);
     rc = update_working_set_from_cib(out, data_set, cib);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     if(simulate) {
         bool prev_quiet = false;
 
         pid = pcmk__getpid_s();
         shadow_cib = cib_shadow_new(pid);
         shadow_file = get_shadow_file(pid);
 
         if (shadow_cib == NULL) {
             out->err(out, "Could not create shadow cib: '%s'", pid);
             rc = ENXIO;
             goto done;
         }
 
         rc = write_xml_file(data_set->input, shadow_file, FALSE);
 
         if (rc < 0) {
             out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
             goto done;
         }
 
         rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
         rc = pcmk_legacy2rc(rc);
 
         if (rc != pcmk_rc_ok) {
             out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_strerror(rc), rc);
             goto done;
         }
 
         pcmk__schedule_actions(data_set, data_set->input, NULL);
 
         prev_quiet = out->is_quiet(out);
         out->quiet = true;
         run_simulation(data_set, shadow_cib, NULL);
         out->quiet = prev_quiet;
 
         rc = update_dataset(shadow_cib, data_set, FALSE);
 
     } else {
         cluster_status(data_set);
     }
 
   done:
     /* Do not free data_set->input here, we need rsc->xml to be valid later on */
     cib_delete(shadow_cib);
     free(pid);
 
     if(shadow_file) {
         unlink(shadow_file);
         free(shadow_file);
     }
 
     return rc;
 }
 
 static int
 max_delay_for_resource(pe_working_set_t * data_set, pe_resource_t *rsc) 
 {
     int delay = 0;
     int max_delay = 0;
 
     if(rsc && rsc->children) {
         GList *iter = NULL;
 
         for(iter = rsc->children; iter; iter = iter->next) {
             pe_resource_t *child = (pe_resource_t *)iter->data;
 
             delay = max_delay_for_resource(data_set, child);
             if(delay > max_delay) {
                 double seconds = delay / 1000.0;
                 crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
                 max_delay = delay;
             }
         }
 
     } else if(rsc) {
         char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
         pe_action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
         const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);
         long long result_ll;
 
         if ((pcmk__scan_ll(value, &result_ll, -1LL) == pcmk_rc_ok)
             && (result_ll >= 0) && (result_ll <= INT_MAX)) {
             max_delay = (int) result_ll;
         } else {
             max_delay = -1;
         }
         pe_free_action(stop);
     }
 
     return max_delay;
 }
 
 static int
 max_delay_in(pe_working_set_t * data_set, GList *resources) 
 {
     int max_delay = 0;
     GList *item = NULL;
 
     for (item = resources; item != NULL; item = item->next) {
         int delay = 0;
         pe_resource_t *rsc = pe_find_resource(data_set->resources, (const char *)item->data);
 
         delay = max_delay_for_resource(data_set, rsc);
 
         if(delay > max_delay) {
             double seconds = delay / 1000.0;
             crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
             max_delay = delay;
         }
     }
 
     return 5 + (max_delay / 1000);
 }
 
 #define waiting_for_starts(d, r, h) ((d != NULL) || \
                                     (!resource_is_running_on((r), (h))))
 
 /*!
  * \internal
  * \brief Restart a resource (on a particular host if requested).
  *
  * \param[in] rsc        The resource to restart
  * \param[in] host       The host to restart the resource on (or NULL for all)
  * \param[in] timeout_ms Consider failed if actions do not complete in this time
  *                       (specified in milliseconds, but a two-second
  *                       granularity is actually used; if 0, a timeout will be
  *                       calculated based on the resource timeout)
  * \param[in] cib        Connection to the CIB manager
  *
  * \return Standard Pacemaker return code (exits on certain failures)
  */
 int
 cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc, const char *host,
                      const char *move_lifetime, int timeout_ms, cib_t *cib,
                      int cib_options, gboolean promoted_role_only, gboolean force)
 {
     int rc = pcmk_rc_ok;
     int lpc = 0;
     int before = 0;
     int step_timeout_s = 0;
     int sleep_interval = 2;
     int timeout = timeout_ms / 1000;
 
     bool stop_via_ban = FALSE;
     char *rsc_id = NULL;
     char *orig_target_role = NULL;
 
     GList *list_delta = NULL;
     GList *target_active = NULL;
     GList *current_active = NULL;
     GList *restart_target_active = NULL;
 
     pe_working_set_t *data_set = NULL;
 
     if (!resource_is_running_on(rsc, host)) {
         const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
         if(host) {
             out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
         } else {
             out->err(out, "%s is not running anywhere and so cannot be restarted", id);
         }
         return ENXIO;
     }
 
     rsc_id = strdup(rsc->id);
     if ((pe_rsc_is_clone(rsc) || pe_bundle_replicas(rsc)) && host) {
         stop_via_ban = TRUE;
     }
 
     /*
       grab full cib
       determine originally active resources
       disable or ban
       poll cib and watch for affected resources to get stopped
       without --timeout, calculate the stop timeout for each step and wait for that
       if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
       if everything stopped, re-enable or un-ban
       poll cib and watch for affected resources to get started
       without --timeout, calculate the start timeout for each step and wait for that
       if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
       report success
 
       Optimizations:
       - use constraints to determine ordered list of affected resources
       - Allow a --no-deps option (aka. --force-restart)
     */
 
     data_set = pe_new_working_set();
     if (data_set == NULL) {
         crm_perror(LOG_ERR, "Could not allocate working set");
         rc = ENOMEM;
         goto done;
     }
 
     data_set->priv = out;
     pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
     rc = update_dataset(cib, data_set, FALSE);
 
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not get new resource list: %s (%d)", pcmk_strerror(rc), rc);
         goto done;
     }
 
     restart_target_active = get_active_resources(host, data_set->resources);
     current_active = get_active_resources(host, data_set->resources);
 
     dump_list(current_active, "Origin");
 
     if (stop_via_ban) {
         /* Stop the clone or bundle instance by banning it from the host */
         out->quiet = true;
         rc = cli_resource_ban(out, rsc_id, host, move_lifetime, NULL, cib,
                               cib_options, promoted_role_only);
 
     } else {
         /* Stop the resource by setting target-role to Stopped.
          * Remember any existing target-role so we can restore it later
          * (though it only makes any difference if it's Unpromoted).
          */
         char *lookup_id = clone_strip(rsc->id);
 
         find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
                            NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
         free(lookup_id);
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, XML_RSC_ATTR_TARGET_ROLE,
                                            RSC_STOPPED, FALSE, cib, cib_options,
                                            data_set, force);
     }
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not set target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
         if (current_active) {
             g_list_free_full(current_active, free);
         }
         if (restart_target_active) {
             g_list_free_full(restart_target_active, free);
         }
         goto done;
     }
 
     rc = update_dataset(cib, data_set, TRUE);
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not determine which resources would be stopped");
         goto failure;
     }
 
     target_active = get_active_resources(host, data_set->resources);
     dump_list(target_active, "Target");
 
     list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
     out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (list_delta != NULL) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
             rc = update_dataset(cib, data_set, FALSE);
             if(rc != pcmk_rc_ok) {
                 out->err(out, "Could not determine which resources were stopped");
                 goto failure;
             }
 
             if (current_active) {
                 g_list_free_full(current_active, free);
             }
             current_active = get_active_resources(host, data_set->resources);
             g_list_free(list_delta);
             list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
         if(before == g_list_length(list_delta)) {
             /* aborted during stop phase, print the contents of list_delta */
             out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
 
     }
 
     if (stop_via_ban) {
         rc = cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
 
     } else if (orig_target_role) {
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, XML_RSC_ATTR_TARGET_ROLE,
                                            orig_target_role, FALSE, cib,
                                            cib_options, data_set, force);
         free(orig_target_role);
         orig_target_role = NULL;
     } else {
         rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
                                            cib_options, data_set, force);
     }
 
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not unset target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
         goto done;
     }
 
     if (target_active) {
         g_list_free_full(target_active, free);
     }
     target_active = restart_target_active;
     list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
     out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (waiting_for_starts(list_delta, rsc, host)) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
 
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
 
             rc = update_dataset(cib, data_set, FALSE);
             if(rc != pcmk_rc_ok) {
                 out->err(out, "Could not determine which resources were started");
                 goto failure;
             }
 
             if (current_active) {
                 g_list_free_full(current_active, free);
             }
 
             /* It's OK if dependent resources moved to a different node,
              * so we check active resources on all nodes.
              */
             current_active = get_active_resources(NULL, data_set->resources);
             g_list_free(list_delta);
             list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         if(before == g_list_length(list_delta)) {
             /* aborted during start phase, print the contents of list_delta */
             out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
 
     }
 
     rc = pcmk_rc_ok;
     goto done;
 
   failure:
     if (stop_via_ban) {
         cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
     } else if (orig_target_role) {
         cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
                                       XML_RSC_ATTR_TARGET_ROLE, orig_target_role,
                                       FALSE, cib, cib_options, data_set, force);
         free(orig_target_role);
     } else {
         cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
                                       XML_RSC_ATTR_TARGET_ROLE, cib, cib_options,
                                       data_set, force);
     }
 
 done:
     if (list_delta) {
         g_list_free(list_delta);
     }
     if (current_active) {
         g_list_free_full(current_active, free);
     }
     if (target_active && (target_active != restart_target_active)) {
         g_list_free_full(target_active, free);
     }
     if (restart_target_active) {
         g_list_free_full(restart_target_active, free);
     }
     free(rsc_id);
     pe_free_working_set(data_set);
     return rc;
 }
 
 static inline bool action_is_pending(pe_action_t *action)
 {
     if (pcmk_any_flags_set(action->flags, pe_action_optional|pe_action_pseudo)
         || !pcmk_is_set(action->flags, pe_action_runnable)
         || pcmk__str_eq("notify", action->task, pcmk__str_casei)) {
         return false;
     }
     return true;
 }
 
 /*!
  * \internal
  * \brief Return TRUE if any actions in a list are pending
  *
  * \param[in] actions   List of actions to check
  *
  * \return TRUE if any actions in the list are pending, FALSE otherwise
  */
 static bool
 actions_are_pending(GList *actions)
 {
     GList *action;
 
     for (action = actions; action != NULL; action = action->next) {
         pe_action_t *a = (pe_action_t *)action->data;
         if (action_is_pending(a)) {
             crm_notice("Waiting for %s (flags=0x%.8x)", a->uuid, a->flags);
             return TRUE;
         }
     }
     return FALSE;
 }
 
 static void
 print_pending_actions(pcmk__output_t *out, GList *actions)
 {
     GList *action;
 
     out->info(out, "Pending actions:");
     for (action = actions; action != NULL; action = action->next) {
         pe_action_t *a = (pe_action_t *) action->data;
 
         if (!action_is_pending(a)) {
             continue;
         }
 
         if (a->node) {
             out->info(out, "\tAction %d: %s\ton %s", a->id, a->uuid, a->node->details->uname);
         } else {
             out->info(out, "\tAction %d: %s", a->id, a->uuid);
         }
     }
 }
 
 /* For --wait, timeout (in seconds) to use if caller doesn't specify one */
 #define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
 
 /* For --wait, how long to sleep between cluster state checks */
 #define WAIT_SLEEP_S (2)
 
 /*!
  * \internal
  * \brief Wait until all pending cluster actions are complete
  *
  * This waits until either the CIB's transition graph is idle or a timeout is
  * reached.
  *
  * \param[in] timeout_ms Consider failed if actions do not complete in this time
  *                       (specified in milliseconds, but one-second granularity
  *                       is actually used; if 0, a default will be used)
  * \param[in] cib        Connection to the CIB manager
  *
  * \return Standard Pacemaker return code
  */
 int
 wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
 {
     pe_working_set_t *data_set = NULL;
     int rc = pcmk_rc_ok;
     int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
     time_t expire_time = time(NULL) + timeout_s;
     time_t time_diff;
     bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
 
     data_set = pe_new_working_set();
     if (data_set == NULL) {
         return ENOMEM;
     }
     pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
 
     do {
 
         /* Abort if timeout is reached */
         time_diff = expire_time - time(NULL);
         if (time_diff > 0) {
             crm_info("Waiting up to %ld seconds for cluster actions to complete", time_diff);
         } else {
             print_pending_actions(out, data_set->actions);
             pe_free_working_set(data_set);
             return ETIME;
         }
         if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
             sleep(WAIT_SLEEP_S);
         }
 
         /* Get latest transition graph */
         pe_reset_working_set(data_set);
         rc = update_working_set_from_cib(out, data_set, cib);
         if (rc != pcmk_rc_ok) {
             pe_free_working_set(data_set);
             return rc;
         }
         pcmk__schedule_actions(data_set, data_set->input, NULL);
 
         if (!printed_version_warning) {
             /* If the DC has a different version than the local node, the two
              * could come to different conclusions about what actions need to be
              * done. Warn the user in this case.
              *
              * @TODO A possible long-term solution would be to reimplement the
              * wait as a new controller operation that would be forwarded to the
              * DC. However, that would have potential problems of its own.
              */
             const char *dc_version = g_hash_table_lookup(data_set->config_hash,
                                                          "dc-version");
 
             if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
                 out->info(out, "warning: wait option may not work properly in "
                           "mixed-version cluster");
                 printed_version_warning = TRUE;
             }
         }
 
     } while (actions_are_pending(data_set->actions));
 
     pe_free_working_set(data_set);
     return rc;
 }
 
 static const char *
 get_action(const char *rsc_action) {
     const char *action = NULL;
 
     if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
         action = "validate-all";
 
     } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
         action = "monitor";
 
     } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
                                     "force-demote", "force-promote", NULL)) {
         action = rsc_action+6;
     } else {
         action = rsc_action;
     }
 
     return action;
 }
 
 crm_exit_t
 cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
                                  const char *rsc_class, const char *rsc_prov,
                                  const char *rsc_type, const char *rsc_action,
                                  GHashTable *params, GHashTable *override_hash,
                                  int timeout_ms, int resource_verbose, gboolean force,
                                  int check_level)
 {
     const char *class = NULL;
     const char *action = NULL;
-    GHashTable *params_copy = NULL;
     crm_exit_t exit_code = CRM_EX_OK;
     svc_action_t *op = NULL;
 
     class = !pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei) ?
                 rsc_class : resources_find_service_class(rsc_type);
 
     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         out->err(out, "Sorry, the %s option doesn't support %s resources yet",
                  rsc_action, class);
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
     } else if (pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_SYSTEMD,
                 PCMK_RESOURCE_CLASS_UPSTART, PCMK_RESOURCE_CLASS_NAGIOS, NULL)) {
         out->err(out, "Sorry, the %s option doesn't support %s resources",
                  rsc_action, class);
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
     }
 
     action = get_action(rsc_action);
 
     /* If no timeout was provided, grab the default. */
     if (timeout_ms == 0) {
         timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
     }
 
     /* add meta_timeout env needed by some resource agents */
     g_hash_table_insert(params, strdup("CRM_meta_timeout"),
                         crm_strdup_printf("%d", timeout_ms));
 
     /* add crm_feature_set env needed by some resource agents */
     g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
 
     if (check_level >= 0) {
         char *level = crm_strdup_printf("%d", check_level);
         setenv("OCF_CHECK_LEVEL", level, 1);
         free(level);
     }
 
-    /* resources_action_create frees the params hash table it's passed, but we
-     * may need to reuse it in a second call to resources_action_create.  Thus
-     * we'll make a copy here so that gets freed and the original remains for
-     * reuse.
-     */
-    params_copy = pcmk__str_table_dup(params);
-
-    op = resources_action_create(rsc_name ? rsc_name : "test", rsc_class, rsc_prov,
-                                 rsc_type, action, 0, timeout_ms, params_copy, 0);
-    if (op == NULL) {
-        /* Re-run with stderr enabled so we can display a sane error message */
-        crm_enable_stderr(TRUE);
-        params_copy = pcmk__str_table_dup(params);
-        op = resources_action_create(rsc_name ? rsc_name : "test", rsc_class, rsc_prov,
-                                     rsc_type, action, 0, timeout_ms, params_copy, 0);
-
-        /* Callers of cli_resource_execute expect that the params hash table will
-         * be freed.  That function uses this one, so for that reason and for
-         * making the two act the same, we should free the hash table here too.
-         */
-        g_hash_table_destroy(params);
+    op = services__create_resource_action(rsc_name? rsc_name : "test",
+                                          rsc_class, rsc_prov, rsc_type, action,
+                                          0, timeout_ms, params, 0);
+    if ((op == NULL) || (op->rc != PCMK_OCF_OK)) {
+        const char *reason = NULL;
 
-        /* We know op will be NULL, but this makes static analysis happy */
-        services_action_free(op);
-        crm_exit(CRM_EX_DATAERR);
-        return exit_code; // Never reached, but helps static analysis
+        if (op == NULL) {
+            reason = strerror(ENOMEM);
+        } else {
+            reason = crm_exit_str(op->rc);
+        }
+        out->err(out, "Operation %s for %s (%s%s%s:%s) failed: %s",
+                 action, rsc_name, rsc_class, (rsc_prov? ":" : ""),
+                 (rsc_prov? rsc_prov : ""), rsc_type, reason);
+        crm_exit((op == NULL)? CRM_EX_OSERR : op->rc);
     }
 
     setenv("HA_debug", resource_verbose > 0 ? "1" : "0", 1);
     if(resource_verbose > 1) {
         setenv("OCF_TRACE_RA", "1", 1);
     }
 
     /* A resource agent using the standard ocf-shellfuncs library will not print
      * messages to stderr if it doesn't have a controlling terminal (e.g. if
      * crm_resource is called via script or ssh). This forces it to do so.
      */
     setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
 
     if (override_hash) {
         GHashTableIter iter;
         char *name = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, override_hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) & name, (gpointer *) & value)) {
             g_hash_table_replace(op->params, strdup(name), strdup(value));
         }
     }
 
     if (services_action_sync(op)) {
         exit_code = op->rc;
 
         /* Lookup exit code based on rc for LSB resources */
         if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei) &&
               pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
             exit_code = services_get_ocf_exitcode(action, exit_code);
         }
 
         out->message(out, "resource-agent-action", resource_verbose, rsc_class,
                      rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
                      exit_code, op->status, op->stdout_data, op->stderr_data);
     } else {
         exit_code = op->rc == 0 ? CRM_EX_ERROR : op->rc;
     }
 
     services_action_free(op);
     /* See comment above about why we free params here. */
     g_hash_table_destroy(params);
     return exit_code;
 }
 
 crm_exit_t
 cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
                      const char *rsc_action, GHashTable *override_hash,
                      int timeout_ms, cib_t * cib, pe_working_set_t *data_set,
                      int resource_verbose, gboolean force, int check_level)
 {
     pcmk__output_t *out = data_set->priv;
     crm_exit_t exit_code = CRM_EX_OK;
     const char *rid = NULL;
     const char *rtype = NULL;
     const char *rprov = NULL;
     const char *rclass = NULL;
     GHashTable *params = NULL;
 
     if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
                                     "force-promote", NULL)) {
         if(pe_rsc_is_clone(rsc)) {
             GList *nodes = cli_resource_search(rsc, requested_name, data_set);
             if(nodes != NULL && force == FALSE) {
                 out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
                          rsc_action, rsc->id);
                 out->err(out, "Try setting target-role=Stopped first or specifying "
                          "the force option");
                 return CRM_EX_UNSAFE;
             }
 
             g_list_free_full(nodes, free);
         }
     }
 
     if(pe_rsc_is_clone(rsc)) {
         /* Grab the first child resource in the hope it's not a group */
         rsc = rsc->children->data;
     }
 
     if(rsc->variant == pe_group) {
         out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
         return CRM_EX_UNIMPLEMENT_FEATURE;
     } else if (rsc->variant == pe_container || pe_rsc_is_bundled(rsc)) {
         out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
         return CRM_EX_UNIMPLEMENT_FEATURE;
     }
 
     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
     rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 
     params = generate_resource_params(rsc, NULL /* @TODO use local node */,
                                       data_set);
 
     if (timeout_ms == 0) {
         timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action), data_set);
     }
 
     rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;
 
     exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
                                                  params, override_hash, timeout_ms,
                                                  resource_verbose, force, check_level);
     return exit_code;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_move(pe_resource_t *rsc, const char *rsc_id, const char *host_name,
                   const char *move_lifetime, cib_t *cib, int cib_options,
                   pe_working_set_t *data_set, gboolean promoted_role_only,
                   gboolean force)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_ok;
     unsigned int count = 0;
     pe_node_t *current = NULL;
     pe_node_t *dest = pe_find_node(data_set->nodes, host_name);
     bool cur_is_dest = FALSE;
 
     if (dest == NULL) {
         return pcmk_rc_node_unknown;
     }
 
     if (promoted_role_only && !pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         pe_resource_t *p = uber_parent(rsc);
 
         if (pcmk_is_set(p->flags, pe_rsc_promotable)) {
             out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
             rsc_id = p->id;
             rsc = p;
 
         } else {
             out->info(out, "Ignoring master option: %s is not promotable", rsc_id);
             promoted_role_only = FALSE;
         }
     }
 
     current = pe__find_active_requires(rsc, &count);
 
     if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         GList *iter = NULL;
         unsigned int promoted_count = 0;
         pe_node_t *promoted_node = NULL;
 
         for(iter = rsc->children; iter; iter = iter->next) {
             pe_resource_t *child = (pe_resource_t *)iter->data;
             enum rsc_role_e child_role = child->fns->state(child, TRUE);
 
             if (child_role == RSC_ROLE_PROMOTED) {
                 rsc = child;
                 promoted_node = pe__current_node(child);
                 promoted_count++;
             }
         }
         if (promoted_role_only || (promoted_count != 0)) {
             count = promoted_count;
             current = promoted_node;
         }
 
     }
 
     if (count > 1) {
         if (pe_rsc_is_clone(rsc)) {
             current = NULL;
         } else {
             return pcmk_rc_multiple;
         }
     }
 
     if (current && (current->details == dest->details)) {
         cur_is_dest = TRUE;
         if (force) {
             crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
                      rsc_id, promoted_role_only?"promoted":"active", dest->details->uname);
         } else {
             return pcmk_rc_already;
         }
     }
 
     /* Clear any previous prefer constraints across all nodes. */
     cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, cib_options, FALSE, force);
 
     /* Clear any previous ban constraints on 'dest'. */
     cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib,
                        cib_options, TRUE, force);
 
     /* Record an explicit preference for 'dest' */
     rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
                              cib, cib_options, promoted_role_only);
 
     crm_trace("%s%s now prefers node %s%s",
               rsc->id, (promoted_role_only? " (promoted)" : ""),
               dest->details->uname, force?"(forced)":"");
 
     /* only ban the previous location if current location != destination location.
      * it is possible to use -M to enforce a location without regard of where the
      * resource is currently located */
     if(force && (cur_is_dest == FALSE)) {
         /* Ban the original location if possible */
         if(current) {
             (void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
                                    NULL, cib, cib_options, promoted_role_only);
 
         } else if(count > 1) {
             out->info(out, "Resource '%s' is currently %s in %d locations. "
                       "One may now move to %s",
                       rsc_id, (promoted_role_only? "promoted" : "active"),
                       count, dest->details->uname);
             out->info(out, "To prevent '%s' from being %s at a specific location, "
                       "specify a node.",
                       rsc_id, (promoted_role_only? "promoted" : "active"));
 
         } else {
             crm_trace("Not banning %s from its current location: not active", rsc_id);
         }
     }
 
     return rc;
 }