No OneTemporary
Actions

Size

731 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/configure.ac b/configure.ac
	index 4c2724ff02..80ba5b381c 100644
	--- a/configure.ac
	+++ b/configure.ac
	@@ -1,2204 +1,2205 @@
	dnl
	dnl autoconf for Pacemaker
	dnl
	dnl Copyright 2009-2022 the Pacemaker project contributors
	dnl
	dnl The version control history for this file may have further details.
	dnl
	dnl This source code is licensed under the GNU General Public License version 2
	dnl or later (GPLv2+) WITHOUT ANY WARRANTY.

	dnl ===============================================
	dnl Bootstrap
	dnl ===============================================
	AC_PREREQ(2.64)

	dnl AC_CONFIG_MACRO_DIR is deprecated as of autoconf 2.70 (2020-12-08).
	dnl Once we can require that version, we can simplify this, and no longer
	dnl need ACLOCAL_AMFLAGS in Makefile.am.
	m4_ifdef([AC_CONFIG_MACRO_DIRS],
	[AC_CONFIG_MACRO_DIRS([m4])],
	[AC_CONFIG_MACRO_DIR([m4])])

	AC_DEFUN([AC_DATAROOTDIR_CHECKED])

	dnl Suggested structure:
	dnl information on the package
	dnl checks for programs
	dnl checks for libraries
	dnl checks for header files
	dnl checks for types
	dnl checks for structures
	dnl checks for compiler characteristics
	dnl checks for library functions
	dnl checks for system services

	m4_include([m4/version.m4])
	AC_INIT([pacemaker], VERSION_NUMBER, [users@clusterlabs.org], [pacemaker],
	PCMK_URL)

	PCMK_FEATURES=""

	LT_CONFIG_LTDL_DIR([libltdl])
	AC_CONFIG_AUX_DIR([libltdl/config])
	AC_CANONICAL_HOST

	dnl Where #defines that autoconf makes (e.g. HAVE_whatever) go
	dnl
	dnl Internal header: include/config.h
	dnl - Contains ALL defines
	dnl - include/config.h.in is generated automatically by autoheader
	dnl - NOT to be included in any header files except crm_internal.h
	dnl (which is also not to be included in any other header files)
	dnl
	dnl External header: include/crm_config.h
	dnl - Contains a subset of defines checked here
	dnl - Manually edit include/crm_config.h.in to have configure include
	dnl new defines
	dnl - Should not include HAVE_* defines
	dnl - Safe to include anywhere
	AC_CONFIG_HEADERS([include/config.h include/crm_config.h])

	dnl 1.13: minimum automake version required
	dnl foreign: don't require GNU-standard top-level files
	dnl tar-ustar: use (older) POSIX variant of generated tar rather than v7
	dnl subdir-objects: keep .o's with their .c's (no-op in 2.0+)
	AM_INIT_AUTOMAKE([1.13 foreign tar-ustar subdir-objects])

	dnl Require minimum version of pkg-config
	PKG_PROG_PKG_CONFIG(0.27)
	AS_IF([test "x${PKG_CONFIG}" != x], [],
	[AC_MSG_FAILURE([Could not find required build tool pkg-config (0.27 or later)])])
	PKG_INSTALLDIR
	PKG_NOARCH_INSTALLDIR

	dnl Example 2.4. Silent Custom Rule to Generate a File
	dnl %-bar.pc: %.pc
	dnl $(AM_V_GEN)$(LN_S) $(notdir $^) $@

	dnl Versioned attributes implementation is not yet production-ready
	AC_DEFINE_UNQUOTED(ENABLE_VERSIONED_ATTRS, 0, [Enable versioned attributes])

	CC_IN_CONFIGURE=yes
	export CC_IN_CONFIGURE

	LDD=ldd

	dnl ========================================================================
	dnl Compiler characteristics
	dnl ========================================================================

	dnl A particular compiler can be forced by setting the CC environment variable
	AC_PROG_CC

	dnl Use at least C99 if possible. This will generate an "obsolete" warning
	dnl since autoconf 2.70, but is needed for older versions.
	AC_PROG_CC_STDC

	dnl C++ is not needed for build, just maintainer utilities
	AC_PROG_CXX

	dnl We use md5.c from gnulib, which has its own m4 macros. Per its docs:
	dnl "The macro gl_EARLY must be called as soon as possible after verifying that
	dnl the C compiler is working. ... The core part of the gnulib checks are done
	dnl by the macro gl_INIT." In addition, prevent gnulib from introducing OpenSSL
	dnl as a dependency.
	gl_EARLY
	gl_SET_CRYPTO_CHECK_DEFAULT([no])
	gl_INIT

	# --enable-new-dtags: Use RUNPATH instead of RPATH.
	# It is necessary to have this done before libtool does linker detection.
	# See also: https://github.com/kronosnet/kronosnet/issues/107
	AX_CHECK_LINK_FLAG([-Wl,--enable-new-dtags],
	[AM_LDFLAGS=-Wl,--enable-new-dtags],
	[AC_MSG_ERROR(["Linker support for --enable-new-dtags is required"])])
	AC_SUBST([AM_LDFLAGS])

	saved_LDFLAGS="$LDFLAGS"
	LDFLAGS="$AM_LDFLAGS $LDFLAGS"
	LT_INIT([dlopen])
	LDFLAGS="$saved_LDFLAGS"
	LTDL_INIT([convenience])

	AC_TYPE_SIZE_T
	AC_CHECK_SIZEOF(char)
	AC_CHECK_SIZEOF(short)
	AC_CHECK_SIZEOF(int)
	AC_CHECK_SIZEOF(long)
	AC_CHECK_SIZEOF(long long)

	dnl ===============================================
	dnl Helpers
	dnl ===============================================
	cc_supports_flag() {
	local CFLAGS="-Werror $@"
	AC_MSG_CHECKING([whether $CC supports $@])
	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ ]])],
	[RC=0; AC_MSG_RESULT([yes])],
	[RC=1; AC_MSG_RESULT([no])])
	return $RC
	}

	# Some tests need to use their own CFLAGS

	cc_temp_flags() {
	ac_save_CFLAGS="$CFLAGS"
	CFLAGS="$*"
	}

	cc_restore_flags() {
	CFLAGS=$ac_save_CFLAGS
	}

	# expand_path_option $path_variable_name $default
	expand_path_option() {
	# The first argument is the variable name (not value)
	ac_path_varname="$1"

	# Get the original value of the variable
	ac_path_value=$(eval echo "\${${ac_path_varname}}")

	# Expand any literal variable expressions in the value so that we don't
	# end up with something like '${prefix}' in #defines etc.
	#
	# Autoconf deliberately leaves values unexpanded to allow overriding
	# the configure script choices in make commands (for example,
	# "make exec_prefix=/foo install"). No longer being able to do this seems
	# like no great loss.
	eval ac_path_value=$(eval echo "${ac_path_value}")

	# Use (expanded) default if necessary
	AS_IF([test x"${ac_path_value}" = x""],
	[eval ac_path_value=$(eval echo "$2")])

	# Require a full path
	AS_CASE(["$ac_path_value"],
	[/*], [eval ${ac_path_varname}="$ac_path_value"],
	[*], [AC_MSG_ERROR([$ac_path_varname value "$ac_path_value" is not a full path])]
	)
	}

	# yes_no_try $user_response $default
	DISABLED=0
	REQUIRED=1
	OPTIONAL=2
	yes_no_try() {
	local value
	AS_IF([test x"$1" = x""], [value="$2"], [value="$1"])
	AS_CASE(["`echo "$value" \| tr '[A-Z]' '[a-z]'`"],
	[0\|no\|false\|disable], [return $DISABLED],
	[1\|yes\|true\|enable], [return $REQUIRED],
	[try\|check], [return $OPTIONAL]
	)
	AC_MSG_ERROR([Invalid option value "$value"])
	}

	check_systemdsystemunitdir() {
	AC_MSG_CHECKING([which system unit file directory to use])
	PKG_CHECK_VAR([systemdsystemunitdir], [systemd], [systemdsystemunitdir])
	AC_MSG_RESULT([${systemdsystemunitdir}])
	test x"$systemdsystemunitdir" != x""
	return $?
	}

	dnl ===============================================
	dnl Configure Options
	dnl ===============================================

	dnl Actual library checks come later, but pkg-config can be used here to grab
	dnl external values to use as defaults for configure options

	dnl Per the autoconf docs, --enable-/--disable- options should control
	dnl features inherent to Pacemaker, while --with-/--without- options should
	dnl control the use of external software. However, --enable-/--disable- may
	dnl implicitly require additional external dependencies, and
	dnl --with-/--without- may implicitly enable or disable features, so the
	dnl line is blurry.
	dnl
	dnl We also use --with-* options for custom file, directory, and path
	dnl locations, since autoconf does not provide an option type for those.

	dnl --enable-* options: build process

	AC_ARG_ENABLE([quiet],
	[AS_HELP_STRING([--enable-quiet],
	[suppress make output unless there is an error @<:@no@:>@])]
	)
	yes_no_try "$enable_quiet" "no"
	enable_quiet=$?

	AC_ARG_ENABLE([fatal-warnings],
	[AS_HELP_STRING([--enable-fatal-warnings],
	[enable pedantic and fatal warnings for gcc @<:@try@:>@])],
	)
	yes_no_try "$enable_fatal_warnings" "try"
	enable_fatal_warnings=$?

	AC_ARG_ENABLE([hardening],
	[AS_HELP_STRING([--enable-hardening],
	[harden the resulting executables/libraries @<:@try@:>@])]
	)
	yes_no_try "$enable_hardening" "try"
	enable_hardening=$?

	dnl --enable-* options: features

	AC_ARG_ENABLE([systemd],
	[AS_HELP_STRING([--enable-systemd],
	[enable support for managing resources via systemd @<:@try@:>@])]
	)
	yes_no_try "$enable_systemd" "try"
	enable_systemd=$?

	AC_ARG_ENABLE([upstart],
	[AS_HELP_STRING([--enable-upstart],
	[enable support for managing resources via Upstart (deprecated) @<:@try@:>@])]
	)
	yes_no_try "$enable_upstart" "try"
	enable_upstart=$?

	dnl --enable-* options: features inherent to Pacemaker

	AC_ARG_ENABLE([compat-2.0],
	[AS_HELP_STRING([--enable-compat-2.0], m4_normalize([
	preserve certain output as it was in 2.0; this option will be
	available only for the lifetime of the 2.1 series @<:@no@:>@]))]
	)
	yes_no_try "$enable_compat_2_0" "no"
	enable_compat_2_0=$?
	AS_IF([test $enable_compat_2_0 -ne $DISABLED],
	[
	AC_DEFINE_UNQUOTED([PCMK__COMPAT_2_0], [1],
	[Keep certain output compatible with 2.0 release series])
	PCMK_FEATURES="$PCMK_FEATURES compat-2.0"
	]
	)

	# Add an option to create symlinks at the pre-2.0.0 daemon name locations, so
	# that users and tools can continue to invoke those names directly (e.g., for
	# meta-data). This option will be removed in a future release.
	AC_ARG_ENABLE([legacy-links],
	[AS_HELP_STRING([--enable-legacy-links],
	[add symlinks for old daemon names (deprecated) @<:@no@:>@])]
	)
	yes_no_try "$enable_legacy_links" "no"
	enable_legacy_links=$?
	AM_CONDITIONAL([BUILD_LEGACY_LINKS], [test $enable_legacy_links -ne $DISABLED])

	# AM_GNU_GETTEXT calls AM_NLS which defines the nls option, but it defaults
	# to enabled. We override the definition of AM_NLS to flip the default and mark
	# it as experimental in the help text.
	AC_DEFUN([AM_NLS],
	[AC_MSG_CHECKING([whether NLS is requested])
	AC_ARG_ENABLE([nls],
	[AS_HELP_STRING([--enable-nls],
	[use Native Language Support (experimental)])],
	USE_NLS=$enableval, USE_NLS=no)
	AC_MSG_RESULT([$USE_NLS])
	AC_SUBST([USE_NLS])]
	)

	AM_GNU_GETTEXT([external])
	AM_GNU_GETTEXT_VERSION([0.18])

	AS_IF([test "$enable_nls" = "yes"], [PCMK_FEATURES="$PCMK_FEATURES nls"])

	dnl --with-* options: external software support, and custom locations

	dnl This argument is defined via an M4 macro so default can be a variable
	AC_DEFUN([VERSION_ARG],
	[AC_ARG_WITH([version],
	[AS_HELP_STRING([--with-version=VERSION],
	[override package version @<:@$1@:>@])],
	[ PACEMAKER_VERSION="$withval" ],
	[ PACEMAKER_VERSION="$PACKAGE_VERSION" ])]
	)
	VERSION_ARG(VERSION_NUMBER)

	# Redefine PACKAGE_VERSION and VERSION according to PACEMAKER_VERSION in case
	# the user used --with-version. Unfortunately, this can only affect the
	# substitution variables and later uses in this file, not the config.h
	# constants, so we have to be careful to use only PACEMAKER_VERSION in C code.
	PACKAGE_VERSION=$PACEMAKER_VERSION
	VERSION=$PACEMAKER_VERSION

	CRM_DAEMON_USER=""
	AC_ARG_WITH([daemon-user],
	[AS_HELP_STRING([--with-daemon-user=USER],
	[user to run unprivileged Pacemaker daemons as (advanced option: changing this may break other cluster components unless similarly configured) @<:@hacluster@:>@])],
	[ CRM_DAEMON_USER="$withval" ]
	)

	CRM_DAEMON_GROUP=""
	AC_ARG_WITH([daemon-group],
	[AS_HELP_STRING([--with-daemon-group=GROUP],
	[group to run unprivileged Pacemaker daemons as (advanced option: changing this may break other cluster components unless similarly configured) @<:@haclient@:>@])],
	[ CRM_DAEMON_GROUP="$withval" ]
	)

	BUG_URL=""
	AC_ARG_WITH([bug-url],
	[AS_HELP_STRING([--with-bug-url=DIR], m4_normalize([
	address where users should submit bug reports
	@<:@https://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker@:>@]))],
	[ BUG_URL="$withval" ]
	)

	dnl --with-* options: features

	AC_ARG_WITH([cibsecrets],
	[AS_HELP_STRING([--with-cibsecrets],
	[support separate file for CIB secrets @<:@no@:>@])]
	)
	yes_no_try "$with_cibsecrets" "no"
	with_cibsecrets=$?

	AC_ARG_WITH([gnutls],
	[AS_HELP_STRING([--with-gnutls],
	[support Pacemaker Remote and remote-tls-port using GnuTLS @<:@try@:>@])]
	)
	yes_no_try "$with_gnutls" "try"
	with_gnutls=$?

	PCMK_GNUTLS_PRIORITIES="NORMAL"
	AC_ARG_WITH([gnutls-priorities],
	[AS_HELP_STRING([--with-gnutls-priorities],
	[default GnuTLS cipher priorities @<:@NORMAL@:>@])],
	[ test x"$withval" = x"no" \|\| PCMK_GNUTLS_PRIORITIES="$withval" ]
	)

	AC_ARG_WITH([concurrent-fencing-default],
	[AS_HELP_STRING([--with-concurrent-fencing-default],
	[default value for concurrent-fencing cluster option @<:@false@:>@])],
	)
	AS_CASE([$with_concurrent_fencing_default],
	[""], [with_concurrent_fencing_default="false"],
	[false], [],
	[true], [PCMK_FEATURES="$PCMK_FEATURES default-concurrent-fencing"],
	[AC_MSG_ERROR([Invalid value "$with_concurrent_fencing_default" for --with-concurrent-fencing-default])]
	)
	AC_DEFINE_UNQUOTED([PCMK__CONCURRENT_FENCING_DEFAULT],
	["$with_concurrent_fencing_default"],
	[Default value for concurrent-fencing cluster option])

	AC_ARG_WITH([sbd-sync-default],
	[AS_HELP_STRING([--with-sbd-sync-default], m4_normalize([
	default value used by sbd if SBD_SYNC_RESOURCE_STARTUP
	environment variable is not set @<:@false@:>@]))],
	)
	AS_CASE([$with_sbd_sync_default],
	[""], [with_sbd_sync_default=false],
	[false], [],
	[true], [PCMK_FEATURES="$PCMK_FEATURES default-sbd-sync"],
	[AC_MSG_ERROR([Invalid value "$with_sbd_sync_default" for --with-sbd-sync-default])]
	)
	AC_DEFINE_UNQUOTED([PCMK__SBD_SYNC_DEFAULT],
	[$with_sbd_sync_default],
	[Default value for SBD_SYNC_RESOURCE_STARTUP environment variable])

	AC_ARG_WITH([resource-stickiness-default],
	[AS_HELP_STRING([--with-resource-stickiness-default],
	[If positive, value to add to new CIBs as explicit resource default for resource-stickiness @<:@0@:>@])],
	)
	errmsg="Invalid value \"$with_resource_stickiness_default\" for --with-resource-stickiness-default"
	AS_CASE([$with_resource_stickiness_default],
	[0\|""], [with_resource_stickiness_default="0"],
	[[[!0-9]]], [AC_MSG_ERROR([$errmsg])],
	[PCMK_FEATURES="$PCMK_FEATURES default-resource-stickiness"]
	)
	AC_DEFINE_UNQUOTED([PCMK__RESOURCE_STICKINESS_DEFAULT],
	[$with_resource_stickiness_default],
	[Default value for resource-stickiness resource meta-attribute])

	AC_ARG_WITH([corosync],
	[AS_HELP_STRING([--with-corosync],
	[support the Corosync messaging and membership layer @<:@try@:>@])]
	)
	yes_no_try "$with_corosync" "try"
	with_corosync=$?

	dnl @TODO This should be a corosync pkgconfig variable
	PCMK__COROSYNC_CONF=""
	AC_ARG_WITH([corosync-conf],
	[AS_HELP_STRING([--with-corosync-conf], m4_normalize([
	location of Corosync configuration file
	@<:@SYSCONFDIR/corosync/corosync.conf@:>@]))],
	[ PCMK__COROSYNC_CONF="$withval" ]
	)

	AC_ARG_WITH([nagios],
	[AS_HELP_STRING([--with-nagios], [support nagios resources])]
	)
	yes_no_try "$with_nagios" "try"
	with_nagios=$?

	dnl --with-* options: directory locations

	AC_ARG_WITH([nagios-plugin-dir],
	[AS_HELP_STRING([--with-nagios-plugin-dir=DIR],
	[directory for nagios plugins @<:@LIBEXECDIR/nagios/plugins@:>@])],
	[ NAGIOS_PLUGIN_DIR="$withval" ]
	)

	AC_ARG_WITH([nagios-metadata-dir],
	[AS_HELP_STRING([--with-nagios-metadata-dir=DIR],
	[directory for nagios plugins metadata @<:@DATADIR/nagios/plugins-metadata@:>@])],
	[ NAGIOS_METADATA_DIR="$withval" ]
	)

	INITDIR=""
	AC_ARG_WITH([initdir],
	[AS_HELP_STRING([--with-initdir=DIR],
	[directory for init (rc) scripts])],
	[ INITDIR="$withval" ]
	)

	systemdsystemunitdir="${systemdsystemunitdir-}"
	AC_ARG_WITH([systemdsystemunitdir],
	[AS_HELP_STRING([--with-systemdsystemunitdir=DIR],
	[directory for systemd unit files (advanced option: must match what systemd uses)])],
	[ systemdsystemunitdir="$withval" ]
	)

	CONFIGDIR=""
	AC_ARG_WITH([configdir],
	[AS_HELP_STRING([--with-configdir=DIR],
	[directory for Pacemaker configuration file @<:@SYSCONFDIR/sysconfig@:>@])],
	[ CONFIGDIR="$withval" ]
	)

	dnl --runstatedir is available as of autoconf 2.70 (2020-12-08). When users
	dnl have an older version, they can use our --with-runstatedir.
	pcmk_runstatedir=""
	AC_ARG_WITH([runstatedir],
	[AS_HELP_STRING([--with-runstatedir=DIR],
	[modifiable per-process data @<:@LOCALSTATEDIR/run@:>@ (ignored if --runstatedir is available)])],
	[ pcmk_runstatedir="$withval" ]
	)

	CRM_LOG_DIR=""
	AC_ARG_WITH([logdir],
	[AS_HELP_STRING([--with-logdir=DIR],
	[directory for Pacemaker log file @<:@LOCALSTATEDIR/log/pacemaker@:>@])],
	[ CRM_LOG_DIR="$withval" ]
	)

	CRM_BUNDLE_DIR=""
	AC_ARG_WITH([bundledir],
	[AS_HELP_STRING([--with-bundledir=DIR],
	[directory for Pacemaker bundle logs @<:@LOCALSTATEDIR/log/pacemaker/bundles@:>@])],
	[ CRM_BUNDLE_DIR="$withval" ]
	)

	dnl Get default from resource-agents if possible. Otherwise, the default uses
	dnl /usr/lib rather than libdir because it's determined by the OCF project and
	dnl not Pacemaker. Even if a user wants to install Pacemaker to /usr/local or
	dnl such, the OCF agents will be expected in their usual location. However, we
	dnl do give the user the option to override it.
	PKG_CHECK_VAR([OCF_ROOT_DIR], [resource-agents], [ocfrootdir], [],
	[OCF_ROOT_DIR="/usr/lib/ocf"])
	AC_ARG_WITH([ocfdir],
	[AS_HELP_STRING([--with-ocfdir=DIR], m4_normalize([
	OCF resource agent root directory (advanced option: changing this
	may break other cluster components unless similarly configured)
	@<:@value from resource-agents package if available otherwise
	/usr/lib/ocf@:>@]))],
	[ OCF_ROOT_DIR="$withval" ]
	)
	AC_SUBST(OCF_ROOT_DIR)
	AC_DEFINE_UNQUOTED([OCF_ROOT_DIR], ["$OCF_ROOT_DIR"],
	[OCF root directory for resource agents and libraries])

	PKG_CHECK_VAR([OCF_RA_PATH], [resource-agents], [ocfrapath], [],
	[OCF_RA_PATH="$OCF_ROOT_DIR/resource.d"])
	AC_ARG_WITH([ocfrapath],
	[AS_HELP_STRING([--with-ocfrapath=DIR], m4_normalize([
	OCF resource agent directories (colon-separated) to search
	@<:@value from resource-agents package if available otherwise
	OCFDIR/resource.d@:>@]))],
	[ OCF_RA_PATH="$withval" ]
	)
	AC_SUBST(OCF_RA_PATH)

	OCF_RA_INSTALL_DIR="$OCF_ROOT_DIR/resource.d"
	AC_ARG_WITH([ocfrainstalldir],
	[AS_HELP_STRING([--with-ocfrainstalldir=DIR], m4_normalize([
	OCF installation directory for Pacemakers resource agents
	@<:@OCFDIR/resource.d@:>@]))],
	[ OCF_RA_INSTALL_DIR="$withval" ]
	)
	AC_SUBST(OCF_RA_INSTALL_DIR)

	dnl Get default from fence-agents if available
	PKG_CHECK_VAR([FA_PREFIX], [fence-agents], [prefix],
	[PCMK__FENCE_BINDIR="${FA_PREFIX}/sbin"],
	[PCMK__FENCE_BINDIR="$sbindir"])
	AC_ARG_WITH([fence-bindir],
	[AS_HELP_STRING([--with-fence-bindir=DIR], m4_normalize([
	directory for executable fence agents @<:@value from fence-agents
	package if available otherwise SBINDIR@:>@]))],
	[ PCMK__FENCE_BINDIR="$withval" ]
	)
	AC_SUBST(PCMK__FENCE_BINDIR)

	dnl --with-* options: non-production testing

	AC_ARG_WITH([profiling],
	[AS_HELP_STRING([--with-profiling],
	[disable optimizations, for effective profiling @<:@no@:>@])]
	)
	yes_no_try "$with_profiling" "no"
	with_profiling=$?

	AC_ARG_WITH([coverage],
	[AS_HELP_STRING([--with-coverage],
	[disable optimizations, for effective profiling and coverage testing @<:@no@:>@])]
	)
	yes_no_try "$with_coverage" "no"
	with_coverage=$?

	AC_ARG_WITH([sanitizers],
	[AS_HELP_STRING([--with-sanitizers=...,...],
	[enable SANitizer build, do NOT use for production. Only ASAN/UBSAN/TSAN are currently supported])],
	[ SANITIZERS="$withval" ],
	[ SANITIZERS="" ])

	dnl Environment variable options

	AC_ARG_VAR([CFLAGS_HARDENED_LIB], [extra C compiler flags for hardened libraries])
	AC_ARG_VAR([LDFLAGS_HARDENED_LIB], [extra linker flags for hardened libraries])

	AC_ARG_VAR([CFLAGS_HARDENED_EXE], [extra C compiler flags for hardened executables])
	AC_ARG_VAR([LDFLAGS_HARDENED_EXE], [extra linker flags for hardened executables])


	dnl ===============================================
	dnl General Processing
	dnl ===============================================

	AC_DEFINE_UNQUOTED(PACEMAKER_VERSION, "$VERSION",
	[Version number of this Pacemaker build])

	PACKAGE_SERIES=`echo $VERSION \| awk -F. '{ print $1"."$2 }'`
	AC_SUBST(PACKAGE_SERIES)

	AC_PROG_LN_S
	AC_PROG_MKDIR_P

	# Check for fatal warning support
	AS_IF([test $enable_fatal_warnings -ne $DISABLED && test "$GCC" = "yes" && cc_supports_flag -Werror],
	[WERROR="-Werror"],
	[
	WERROR=""
	AS_CASE([$enable_fatal_warnings],
	[$REQUIRED], [AC_MSG_ERROR([Compiler does not support fatal warnings])],
	[$OPTIONAL], [
	AC_MSG_NOTICE([Compiler does not support fatal warnings])
	enable_fatal_warnings=$DISABLED
	])
	])

	AC_MSG_NOTICE([Sanitizing prefix: ${prefix}])
	AS_IF([test "$prefix" = "NONE"],
	[
	prefix=/usr
	dnl Fix default variables - "prefix" variable if not specified
	AS_IF([test "$localstatedir" = "\${prefix}/var"],
	[localstatedir="/var"])
	AS_IF([test "$sysconfdir" = "\${prefix}/etc"],
	[sysconfdir="/etc"])
	])

	AC_MSG_NOTICE([Sanitizing exec_prefix: ${exec_prefix}])
	case $exec_prefix in
	prefix\|NONE)
	exec_prefix=$prefix
	;;
	esac

	AC_MSG_NOTICE([Sanitizing INITDIR: ${INITDIR}])
	case $INITDIR in
	prefix) INITDIR=$prefix;;
	"")
	AC_MSG_CHECKING([which init (rc) directory to use])
	for initdir in /etc/init.d /etc/rc.d/init.d /sbin/init.d \
	/usr/local/etc/rc.d /etc/rc.d
	do
	if
	test -d $initdir
	then
	INITDIR=$initdir
	break
	fi
	done
	AC_MSG_RESULT([$INITDIR])
	;;
	esac
	AC_SUBST(INITDIR)

	AC_MSG_NOTICE([Sanitizing libdir: ${libdir}])
	case $libdir in
	prefix\|NONE)
	AC_MSG_CHECKING([which lib directory to use])
	for aDir in lib64 lib
	do
	trydir="${exec_prefix}/${aDir}"
	if
	test -d ${trydir}
	then
	libdir=${trydir}
	break
	fi
	done
	AC_MSG_RESULT([$libdir]);
	;;
	esac

	dnl Expand values of autoconf-provided directory options
	expand_path_option prefix
	expand_path_option exec_prefix
	expand_path_option bindir
	expand_path_option sbindir
	expand_path_option libexecdir
	expand_path_option datadir
	expand_path_option sysconfdir
	expand_path_option sharedstatedir
	expand_path_option localstatedir
	expand_path_option libdir
	expand_path_option includedir
	expand_path_option oldincludedir
	expand_path_option infodir
	expand_path_option mandir

	dnl Home-grown variables

	expand_path_option localedir "${datadir}/locale"
	AC_DEFINE_UNQUOTED([PCMK__LOCALE_DIR],["$localedir"], [Base directory for message catalogs])

	AS_IF([test x"${runstatedir}" = x""], [runstatedir="${pcmk_runstatedir}"])
	expand_path_option runstatedir "${localstatedir}/run"
	AC_DEFINE_UNQUOTED([PCMK_RUN_DIR], ["$runstatedir"],
	[Location for modifiable per-process data])
	AC_SUBST(runstatedir)

	expand_path_option INITDIR
	AC_DEFINE_UNQUOTED([PCMK__LSB_INIT_DIR], ["$INITDIR"],
	[Location for LSB init scripts])

	expand_path_option docdir "${datadir}/doc/${PACKAGE}-${VERSION}"
	AC_SUBST(docdir)

	expand_path_option CONFIGDIR "${sysconfdir}/sysconfig"
	AC_SUBST(CONFIGDIR)

	expand_path_option PCMK__COROSYNC_CONF "${sysconfdir}/corosync/corosync.conf"
	AC_SUBST(PCMK__COROSYNC_CONF)

	expand_path_option CRM_LOG_DIR "${localstatedir}/log/pacemaker"
	AC_DEFINE_UNQUOTED(CRM_LOG_DIR,"$CRM_LOG_DIR", Location for Pacemaker log file)
	AC_SUBST(CRM_LOG_DIR)

	expand_path_option CRM_BUNDLE_DIR "${localstatedir}/log/pacemaker/bundles"
	AC_DEFINE_UNQUOTED(CRM_BUNDLE_DIR,"$CRM_BUNDLE_DIR", Location for Pacemaker bundle logs)
	AC_SUBST(CRM_BUNDLE_DIR)

	expand_path_option PCMK__FENCE_BINDIR
	AC_DEFINE_UNQUOTED(PCMK__FENCE_BINDIR,"$PCMK__FENCE_BINDIR",
	[Location for executable fence agents])

	expand_path_option OCF_RA_PATH
	AC_DEFINE_UNQUOTED([OCF_RA_PATH], ["$OCF_RA_PATH"],
	[OCF directories to search for resource agents ])

	AS_IF([test x"${PCMK_GNUTLS_PRIORITIES}" != x""], [],
	[AC_MSG_ERROR([--with-gnutls-priorities value must not be empty])])
	AC_DEFINE_UNQUOTED([PCMK_GNUTLS_PRIORITIES], ["$PCMK_GNUTLS_PRIORITIES"],
	[GnuTLS cipher priorities])

	if test x"${BUG_URL}" = x""; then
	BUG_URL="https://bugs.clusterlabs.org/enter_bug.cgi?product=Pacemaker"
	fi
	AC_SUBST(BUG_URL)

	for j in prefix exec_prefix bindir sbindir libexecdir datadir sysconfdir \
	sharedstatedir localstatedir libdir includedir oldincludedir infodir \
	mandir INITDIR docdir CONFIGDIR localedir
	do
	dirname=`eval echo '${'${j}'}'`
	if
	test ! -d "$dirname"
	then
	AC_MSG_WARN([$j directory ($dirname) does not exist (yet)])
	fi
	done

	us_auth=
	AC_CHECK_HEADER([sys/socket.h], [
	AC_CHECK_DECL([SO_PEERCRED], [
	# Linux
	AC_CHECK_TYPE([struct ucred], [
	us_auth=peercred_ucred;
	AC_DEFINE([US_AUTH_PEERCRED_UCRED], [1],
	[Define if Unix socket auth method is
	getsockopt(s, SO_PEERCRED, &ucred, ...)])
	], [
	# OpenBSD
	AC_CHECK_TYPE([struct sockpeercred], [
	us_auth=localpeercred_sockepeercred;
	AC_DEFINE([US_AUTH_PEERCRED_SOCKPEERCRED], [1],
	[Define if Unix socket auth method is
	getsockopt(s, SO_PEERCRED, &sockpeercred, ...)])
	], [], [[#include <sys/socket.h>]])
	], [[#define _GNU_SOURCE
	#include <sys/socket.h>]])
	], [], [[#include <sys/socket.h>]])
	])

	AS_IF([test -z "${us_auth}"], [
	# FreeBSD
	AC_CHECK_DECL([getpeereid], [
	us_auth=getpeereid;
	AC_DEFINE([US_AUTH_GETPEEREID], [1],
	[Define if Unix socket auth method is
	getpeereid(s, &uid, &gid)])
	], [
	# Solaris/OpenIndiana
	AC_CHECK_DECL([getpeerucred], [
	us_auth=getpeerucred;
	AC_DEFINE([US_AUTH_GETPEERUCRED], [1],
	[Define if Unix socket auth method is
	getpeercred(s, &ucred)])
	], [
	AC_MSG_FAILURE([No way to authenticate a Unix socket peer])
	], [[#include <ucred.h>]])
	])
	])

	dnl OS-based decision-making is poor autotools practice; feature-based
	dnl mechanisms are strongly preferred. Keep this section to a bare minimum;
	dnl regard as a "necessary evil".
	INIT_EXT=""
	PROCFS=0
	case "$host_os" in
	dnl Solaris and some *BSD versions support procfs but not files we need
	bsd)
	INIT_EXT=".sh"
	;;
	linux)
	PROCFS=1
	;;
	darwin*)
	LIBS="$LIBS -L${prefix}/lib"
	CFLAGS="$CFLAGS -I${prefix}/include"
	;;
	esac

	AC_SUBST(INIT_EXT)
	AC_DEFINE_UNQUOTED([SUPPORT_PROCFS], [$PROCFS],
	[Define to 1 if procfs is supported])

	case "$host_cpu" in
	ppc64\|powerpc64)
	case $CFLAGS in
	powerpc64)
	;;
	*)
	if test "$GCC" = yes; then
	CFLAGS="$CFLAGS -m64"
	fi
	;;
	esac
	;;
	esac

	# C99 doesn't guarantee uint64_t type and related format specifiers, but
	# prerequisites, corosync + libqb, use that widely, so the target platforms
	# are already pre-constrained to those "64bit-clean" (doesn't imply native
	# bit width) and hence we deliberately refrain from artificial surrogates
	# (sans manipulation through cached values).
	AC_CACHE_VAL(
	[pcmk_cv_decl_inttypes],
	[
	AC_CHECK_DECLS(
	[PRIu64, PRIu32, PRIx32,
	SCNu64],
	[pcmk_cv_decl_inttypes="PRIu64 PRIu32 PRIx32 SCNu64"],
	[
	# test shall only react on "no" cached result & error out respectively
	if test "x$ac_cv_have_decl_PRIu64" = xno; then
	AC_MSG_ERROR([lack of inttypes.h based specifier serving uint64_t (PRIu64)])
	elif test "x$ac_cv_have_decl_PRIu32" = xno; then
	AC_MSG_ERROR([lack of inttypes.h based specifier serving uint32_t (PRIu32)])
	elif test "x$ac_cv_have_decl_PRIx32" = xno; then
	AC_MSG_ERROR([lack of inttypes.h based hexa specifier serving uint32_t (PRIx32)])
	elif test "x$ac_cv_have_decl_SCNu64" = xno; then
	AC_MSG_ERROR([lack of inttypes.h based specifier gathering uint64_t (SCNu64)])
	fi
	],
	[[#include <inttypes.h>]]
	)
	]
	)
	(
	set $pcmk_cv_decl_inttypes
	AC_DEFINE_UNQUOTED([U64T], [$1], [Correct format specifier for U64T])
	AC_DEFINE_UNQUOTED([U32T], [$2], [Correct format specifier for U32T])
	AC_DEFINE_UNQUOTED([X32T], [$3], [Correct format specifier for X32T])
	AC_DEFINE_UNQUOTED([U64TS], [$4], [Correct format specifier for U64TS])
	)

	dnl ===============================================
	dnl Program Paths
	dnl ===============================================

	PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin"
	export PATH

	dnl Replacing AC_PROG_LIBTOOL with AC_CHECK_PROG because LIBTOOL
	dnl was NOT being expanded all the time thus causing things to fail.
	AC_CHECK_PROGS(LIBTOOL, glibtool libtool libtool15 libtool13)
	AS_IF([test "x${LIBTOOL}" != "x"], [],
	[AC_MSG_FAILURE([Could not find required build tool libtool (or equivalent)])])

	dnl Pacemaker's executable python scripts will invoke the python specified by
	dnl configure's PYTHON variable. If not specified, AM_PATH_PYTHON will check a
	dnl built-in list with (unversioned) "python" having precedence. To configure
	dnl Pacemaker to use a specific python interpreter version, define PYTHON
	dnl when calling configure, for example: ./configure PYTHON=/usr/bin/python3.6

	dnl Ensure PYTHON is an absolute path
	AS_IF([test x"${PYTHON}" != x""], [AC_PATH_PROG([PYTHON], [$PYTHON])])

	dnl Require a minimum Python version
	AM_PATH_PYTHON([3.4])

	AC_PATH_PROGS([ASCIIDOC_CONV], [asciidoc asciidoctor])
	AC_PATH_PROG([HELP2MAN], [help2man])
	AC_PATH_PROG([SPHINX], [sphinx-build])
	AC_PATH_PROG([INKSCAPE], [inkscape])
	AC_PATH_PROG([XSLTPROC], [xsltproc])
	AC_PATH_PROG([XMLCATALOG], [xmlcatalog])

	dnl Bash is needed for building man pages and running regression tests.
	dnl BASH is already an environment variable, so use something else.
	AC_PATH_PROG([BASH_PATH], [bash])
	AS_IF([test "x${BASH_PATH}" != "x"], [],
	[AC_MSG_FAILURE([Could not find required build tool bash])])

	AC_PATH_PROGS(VALGRIND_BIN, valgrind, /usr/bin/valgrind)
	AC_DEFINE_UNQUOTED(VALGRIND_BIN, "$VALGRIND_BIN", Valgrind command)

	AM_CONDITIONAL(BUILD_HELP, test x"${HELP2MAN}" != x"")
	if test x"${HELP2MAN}" != x""; then
	PCMK_FEATURES="$PCMK_FEATURES generated-manpages"
	fi

	MANPAGE_XSLT=""
	if test x"${XSLTPROC}" != x""; then
	AC_MSG_CHECKING([for DocBook-to-manpage transform])
	# first try to figure out correct template using xmlcatalog query,
	# resort to extensive (semi-deterministic) file search if that fails
	DOCBOOK_XSL_URI='http://docbook.sourceforge.net/release/xsl/current'
	DOCBOOK_XSL_PATH='manpages/docbook.xsl'
	MANPAGE_XSLT=$(${XMLCATALOG} "" ${DOCBOOK_XSL_URI}/${DOCBOOK_XSL_PATH} \
	\| sed -n 's\|^file://\|\|p;q')
	if test x"${MANPAGE_XSLT}" = x""; then
	DIRS=$(find "${datadir}" -name $(basename $(dirname ${DOCBOOK_XSL_PATH})) \
	-type d 2>/dev/null \| LC_ALL=C sort)
	XSLT=$(basename ${DOCBOOK_XSL_PATH})
	for d in ${DIRS}
	do
	if test -f "${d}/${XSLT}"; then
	MANPAGE_XSLT="${d}/${XSLT}"
	break
	fi
	done
	fi
	fi
	AC_MSG_RESULT([$MANPAGE_XSLT])
	AC_SUBST(MANPAGE_XSLT)

	AM_CONDITIONAL(BUILD_XML_HELP, test x"${MANPAGE_XSLT}" != x"")
	if test x"${MANPAGE_XSLT}" != x""; then
	PCMK_FEATURES="$PCMK_FEATURES agent-manpages"
	fi

	AM_CONDITIONAL([IS_ASCIIDOC], [echo "${ASCIIDOC_CONV}" \| grep -Eq 'asciidoc$'])
	AM_CONDITIONAL([BUILD_ASCIIDOC], [test "x${ASCIIDOC_CONV}" != x])
	if test "x${ASCIIDOC_CONV}" != x; then
	PCMK_FEATURES="$PCMK_FEATURES ascii-docs"
	fi

	AM_CONDITIONAL([BUILD_SPHINX_DOCS],
	[test x"${SPHINX}" != x"" && test x"${INKSCAPE}" != x""])
	AM_COND_IF([BUILD_SPHINX_DOCS], [PCMK_FEATURES="$PCMK_FEATURES books"])

	dnl Pacemaker's shell scripts (and thus man page builders) rely on GNU getopt
	AC_MSG_CHECKING([for GNU-compatible getopt])
	IFS_orig=$IFS
	IFS=:
	for PATH_DIR in $PATH
	do
	IFS=$IFS_orig
	GETOPT_PATH="${PATH_DIR}/getopt"
	if test -f "$GETOPT_PATH" && test -x "$GETOPT_PATH" ; then
	$GETOPT_PATH -T >/dev/null 2>/dev/null
	if test $? -eq 4; then
	break
	fi
	fi
	GETOPT_PATH=""
	done
	IFS=$IFS_orig
	AS_IF([test -n "$GETOPT_PATH"], [AC_MSG_RESULT([$GETOPT_PATH])],
	[
	AC_MSG_RESULT([no])
	AC_MSG_ERROR([Could not find required build tool GNU-compatible getopt])
	])
	AC_SUBST([GETOPT_PATH])

	dnl ========================================================================
	dnl checks for library functions to replace them
	dnl
	dnl NoSuchFunctionName:
	dnl is a dummy function which no system supplies. It is here to make
	dnl the system compile semi-correctly on OpenBSD which doesn't know
	dnl how to create an empty archive
	dnl
	dnl scandir: Only on BSD.
	dnl System-V systems may have it, but hidden and/or deprecated.
	dnl A replacement function is supplied for it.
	dnl
	dnl strerror: returns a string that corresponds to an errno.
	dnl A replacement function is supplied for it.
	dnl
	dnl strnlen: is a gnu function similar to strlen, but safer.
	dnl We wrote a tolerably-fast replacement function for it.
	dnl
	dnl strndup: is a gnu function similar to strdup, but safer.
	dnl We wrote a tolerably-fast replacement function for it.

	AC_REPLACE_FUNCS(alphasort NoSuchFunctionName scandir strerror strchrnul strnlen strndup)

	dnl ===============================================
	dnl Libraries
	dnl ===============================================
	AC_CHECK_LIB(socket, socket) dnl -lsocket
	AC_CHECK_LIB(c, dlopen) dnl if dlopen is in libc...
	AC_CHECK_LIB(dl, dlopen) dnl -ldl (for Linux)
	AC_CHECK_LIB(rt, sched_getscheduler) dnl -lrt (for Tru64)
	AC_CHECK_LIB(gnugetopt, getopt_long) dnl -lgnugetopt ( if available )
	AC_CHECK_LIB(pam, pam_start) dnl -lpam (if available)

	PKG_CHECK_MODULES([UUID], [uuid],
	[CPPFLAGS="${CPPFLAGS} ${UUID_CFLAGS}"
	LIBS="${LIBS} ${UUID_LIBS}"])

	AC_CHECK_FUNCS([sched_setscheduler])
	if test "$ac_cv_func_sched_setscheduler" != yes; then
	PC_LIBS_RT=""
	else
	PC_LIBS_RT="-lrt"
	fi
	AC_SUBST(PC_LIBS_RT)

	# Require minimum glib version
	PKG_CHECK_MODULES([GLIB], [glib-2.0 >= 2.42.0],
	[CPPFLAGS="${CPPFLAGS} ${GLIB_CFLAGS}"
	LIBS="${LIBS} ${GLIB_LIBS}"])

	# Check whether high-resolution sleep function is available
	AC_CHECK_FUNCS([nanosleep usleep])

	#
	# Where is dlopen?
	#
	if test "$ac_cv_lib_c_dlopen" = yes; then
	LIBADD_DL=""
	elif test "$ac_cv_lib_dl_dlopen" = yes; then
	LIBADD_DL=-ldl
	else
	LIBADD_DL=${lt_cv_dlopen_libs}
	fi

	PKG_CHECK_MODULES(LIBXML2, [libxml-2.0],
	[CPPFLAGS="${CPPFLAGS} ${LIBXML2_CFLAGS}"
	LIBS="${LIBS} ${LIBXML2_LIBS}"])

	REQUIRE_LIB([xslt], [xsltApplyStylesheet])

	dnl ========================================================================
	dnl Headers
	dnl ========================================================================

	# Some distributions insert #warnings into deprecated headers. If we will
	# enable fatal warnings for the build, then enable them for the header checks
	# as well, otherwise the build could fail even though the header check
	# succeeds. (We should probably be doing this in more places.)
	cc_temp_flags "$CFLAGS $WERROR"

	# Optional headers (inclusion of these should be conditional in C code)
	AC_CHECK_HEADERS([getopt.h])
	AC_CHECK_HEADERS([linux/swab.h])
	AC_CHECK_HEADERS([stddef.h])
	AC_CHECK_HEADERS([sys/signalfd.h])
	AC_CHECK_HEADERS([uuid/uuid.h])
	AC_CHECK_HEADERS([security/pam_appl.h pam/pam_appl.h])

	# Required headers
	REQUIRE_HEADER([arpa/inet.h])
	REQUIRE_HEADER([ctype.h])
	REQUIRE_HEADER([dirent.h])
	REQUIRE_HEADER([errno.h])
	REQUIRE_HEADER([glib.h])
	REQUIRE_HEADER([grp.h])
	REQUIRE_HEADER([limits.h])
	REQUIRE_HEADER([netdb.h])
	REQUIRE_HEADER([netinet/in.h])
	REQUIRE_HEADER([netinet/ip.h], [
	#include <sys/types.h>
	#include <netinet/in.h>
	])
	REQUIRE_HEADER([pwd.h])
	REQUIRE_HEADER([signal.h])
	REQUIRE_HEADER([stdio.h])
	REQUIRE_HEADER([stdlib.h])
	REQUIRE_HEADER([string.h])
	REQUIRE_HEADER([strings.h])
	REQUIRE_HEADER([sys/ioctl.h])
	REQUIRE_HEADER([sys/param.h])
	REQUIRE_HEADER([sys/reboot.h])
	REQUIRE_HEADER([sys/resource.h])
	REQUIRE_HEADER([sys/socket.h])
	REQUIRE_HEADER([sys/stat.h])
	REQUIRE_HEADER([sys/time.h])
	REQUIRE_HEADER([sys/types.h])
	REQUIRE_HEADER([sys/utsname.h])
	REQUIRE_HEADER([sys/wait.h])
	REQUIRE_HEADER([time.h])
	REQUIRE_HEADER([unistd.h])
	REQUIRE_HEADER([libxml/xpath.h])
	REQUIRE_HEADER([libxslt/xslt.h])

	cc_restore_flags

	AC_CHECK_FUNCS([uuid_unparse], [],
	[AC_MSG_FAILURE([Could not find required C function uuid_unparse()])])

	AC_CACHE_CHECK([whether __progname and __progname_full are available],
	[pf_cv_var_progname], AC_LINK_IFELSE([
	AC_LANG_PROGRAM([[extern char __progname, __progname_full;]],
	[[__progname = "foo"; __progname_full = "foo bar";]],
	[pf_cv_var_progname="yes"],
	[pf_cv_var_progname="no"])
	]))
	AS_IF([test "$pf_cv_var_progname" = "yes"], [AC_DEFINE(HAVE___PROGNAME,1,[ ])])

	dnl ========================================================================
	dnl Generic declarations
	dnl ========================================================================

	AC_CHECK_DECLS([CLOCK_MONOTONIC], [PCMK_FEATURES="$PCMK_FEATURES monotonic"], [], [[
	#include <time.h>
	]])

	dnl ========================================================================
	dnl Unit test declarations
	dnl ========================================================================

	AC_CHECK_DECLS([assert_float_equal], [], [], [[
	#include <stdarg.h>
	#include <stddef.h>
	#include <setjmp.h>
	#include <cmocka.h>
	]])

	cc_temp_flags "$CFLAGS -Wl,--wrap=uname"

	WRAPPABLE_UNAME="no"

	AC_MSG_CHECKING([if uname() can be wrapped])
	AC_RUN_IFELSE([AC_LANG_SOURCE([[
	#include <sys/utsname.h>
	int __wrap_uname(struct utsname *buf) {
	return 100;
	}
	int main(int argc, char **argv) {
	struct utsname x;
	return uname(&x) == 100 ? 0 : 1;
	}
	]])],
	[ WRAPPABLE_UNAME="yes" ], [ WRAPPABLE_UNAME="no"])
	AC_MSG_RESULT([$WRAPPABLE_UNAME])
	AM_CONDITIONAL([WRAPPABLE_UNAME], [test "$WRAPPABLE_UNAME" = "yes"])

	cc_restore_flags

	dnl ========================================================================
	dnl Structures
	dnl ========================================================================

	AC_CHECK_MEMBERS([struct tm.tm_gmtoff],,,[[#include <time.h>]])
	AC_CHECK_MEMBER([struct dirent.d_type],
	AC_DEFINE(HAVE_STRUCT_DIRENT_D_TYPE,1,[Define this if struct dirent has d_type]),,
	[#include <dirent.h>])

	dnl ========================================================================
	dnl Functions
	dnl ========================================================================

	REQUIRE_FUNC([getopt])
	REQUIRE_FUNC([setenv])
	REQUIRE_FUNC([unsetenv])

	AC_CACHE_CHECK(whether sscanf supports %m,
	pf_cv_var_sscanf,
	AC_RUN_IFELSE([AC_LANG_SOURCE([[
	#include <stdio.h>
	const char *s = "some-command-line-arg";
	int main(int argc, char **argv) {
	char *name = NULL;
	int n = sscanf(s, "%ms", &name);
	return n == 1 ? 0 : 1;
	}
	]])],
	pf_cv_var_sscanf="yes", pf_cv_var_sscanf="no", pf_cv_var_sscanf="no"))

	AS_IF([test "$pf_cv_var_sscanf" = "yes"], [AC_DEFINE(SSCANF_HAS_M, 1, [ ])])

	dnl ========================================================================
	dnl bzip2
	dnl ========================================================================
	REQUIRE_HEADER([bzlib.h])
	REQUIRE_LIB([bz2], [BZ2_bzBuffToBuffCompress])

	dnl ========================================================================
	dnl sighandler_t is missing from Illumos, Solaris11 systems
	dnl ========================================================================

	AC_MSG_CHECKING([for sighandler_t])
	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <signal.h>]], [[sighandler_t *f;]])],
	[
	AC_MSG_RESULT([yes])
	AC_DEFINE([HAVE_SIGHANDLER_T], [1],
	[Define to 1 if sighandler_t is available])
	],
	[AC_MSG_RESULT([no])])

	dnl ========================================================================
	dnl ncurses
	dnl ========================================================================
	dnl
	dnl A few OSes (e.g. Linux) deliver a default "ncurses" alongside "curses".
	dnl Many non-Linux deliver "curses"; sites may add "ncurses".
	dnl
	dnl However, the source-code recommendation for both is to #include "curses.h"
	dnl (i.e. "ncurses" still wants the include to be simple, no-'n', "curses.h").
	dnl
	dnl ncurses takes precedence.
	dnl
	AC_CHECK_HEADERS([curses.h curses/curses.h ncurses.h ncurses/ncurses.h])

	dnl Although n-library is preferred, only look for it if the n-header was found.
	CURSESLIBS=''
	PC_NAME_CURSES=""
	PC_LIBS_CURSES=""
	AS_IF([test "$ac_cv_header_ncurses_h" = "yes"], [
	AC_CHECK_LIB(ncurses, printw,
	[AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
	CURSESLIBS=`$PKG_CONFIG --libs ncurses` \|\| CURSESLIBS='-lncurses'
	PC_NAME_CURSES="ncurses"
	])

	AS_IF([test "$ac_cv_header_ncurses_ncurses_h" = "yes"], [
	AC_CHECK_LIB(ncurses, printw,
	[AC_DEFINE(HAVE_LIBNCURSES,1, have ncurses library)])
	CURSESLIBS=`$PKG_CONFIG --libs ncurses` \|\| CURSESLIBS='-lncurses'
	PC_NAME_CURSES="ncurses"
	])

	dnl Only look for non-n-library if there was no n-library.
	AS_IF([test X"$CURSESLIBS" = X"" && test "$ac_cv_header_curses_h" = "yes"], [
	AC_CHECK_LIB(curses, printw,
	[CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
	PC_LIBS_CURSES="$CURSESLIBS"
	])

	dnl Only look for non-n-library if there was no n-library.
	AS_IF([test X"$CURSESLIBS" = X"" && test "$ac_cv_header_curses_curses_h" = "yes"], [
	AC_CHECK_LIB(curses, printw,
	[CURSESLIBS='-lcurses'; AC_DEFINE(HAVE_LIBCURSES,1, have curses library)])
	PC_LIBS_CURSES="$CURSESLIBS"
	])

	if test "x$CURSESLIBS" != "x"; then
	PCMK_FEATURES="$PCMK_FEATURES ncurses"
	fi

	dnl Check for printw() prototype compatibility
	AS_IF([test X"$CURSESLIBS" != X"" && cc_supports_flag -Wcast-qual], [
	ac_save_LIBS=$LIBS
	LIBS="$CURSESLIBS"
	cc_temp_flags "-Wcast-qual $WERROR"
	# avoid broken test because of hardened build environment in Fedora 23+
	# - https://fedoraproject.org/wiki/Changes/Harden_All_Packages
	# - https://bugzilla.redhat.com/1297985
	AS_IF([cc_supports_flag -fPIC], [CFLAGS="$CFLAGS -fPIC"])

	AC_MSG_CHECKING([whether curses library is compatible])
	AC_LINK_IFELSE(
	[AC_LANG_PROGRAM([
	#if defined(HAVE_NCURSES_H)
	# include <ncurses.h>
	#elif defined(HAVE_NCURSES_NCURSES_H)
	# include <ncurses/ncurses.h>
	#elif defined(HAVE_CURSES_H)
	# include <curses.h>
	#endif
	],
	[printw((const char *)"Test");]
	)],
	[AC_MSG_RESULT([yes])],
	[
	AC_MSG_RESULT([no])
	AC_MSG_WARN(m4_normalize([Disabling curses because the printw()
	function of your (n)curses library is old.
	If you wish to enable curses, update to a
	newer version (ncurses 5.4 or later is
	recommended, available from
	https://invisible-island.net/ncurses/)
	]))
	AC_DEFINE([HAVE_INCOMPATIBLE_PRINTW], [1],
	[Define to 1 if curses library has incompatible printw()])
	]
	)

	LIBS=$ac_save_LIBS
	cc_restore_flags
	])

	AC_SUBST(CURSESLIBS)
	AC_SUBST(PC_NAME_CURSES)
	AC_SUBST(PC_LIBS_CURSES)

	dnl ========================================================================
	dnl Profiling and GProf
	dnl ========================================================================

	CFLAGS_ORIG="$CFLAGS"
	AS_IF([test $with_coverage -ne $DISABLED],
	[
	with_profiling=$REQUIRED
	PCMK_FEATURES="$PCMK_FEATURES coverage"
	CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage"
	dnl During linking, make sure to specify -lgcov or -coverage
	]
	)

	AS_IF([test $with_profiling -ne $DISABLED],
	[
	with_profiling=$REQUIRED
	PCMK_FEATURES="$PCMK_FEATURES profile"

	dnl Disable various compiler optimizations
	CFLAGS="$CFLAGS -fno-omit-frame-pointer -fno-inline -fno-builtin"
	dnl CFLAGS="$CFLAGS -fno-inline-functions"
	dnl CFLAGS="$CFLAGS -fno-default-inline"
	dnl CFLAGS="$CFLAGS -fno-inline-functions-called-once"
	dnl CFLAGS="$CFLAGS -fno-optimize-sibling-calls"

	dnl Turn off optimization so tools can get accurate line numbers
	CFLAGS=`echo $CFLAGS \| sed \
	-e 's/-O.\ //g' \
	-e 's/-Wp,-D_FORTIFY_SOURCE=.\ //g' \
	-e 's/-D_FORTIFY_SOURCE=.\ //g'`
	CFLAGS="$CFLAGS -O0 -g3 -gdwarf-2"

	AC_MSG_NOTICE([CFLAGS before adding profiling options: $CFLAGS_ORIG])
	AC_MSG_NOTICE([CFLAGS after: $CFLAGS])
	]
	)
	AC_DEFINE_UNQUOTED([SUPPORT_PROFILING], [$with_profiling], [Support profiling])
	AM_CONDITIONAL([BUILD_PROFILING], [test "$with_profiling" = "$REQUIRED"])

	dnl ========================================================================
	dnl Cluster infrastructure - LibQB
	dnl ========================================================================

	PKG_CHECK_MODULES(libqb, libqb >= 0.17)
	CPPFLAGS="$libqb_CFLAGS $CPPFLAGS"
	LIBS="$libqb_LIBS $LIBS"

	dnl libqb libqb-2.0.3 + ipc-connect-async-API (2022-01)
	AC_CHECK_FUNCS([qb_ipcc_connect_async])

	dnl libqb 2.0.2+ (2020-10)
	AC_CHECK_FUNCS([qb_ipcc_auth_get])

	dnl libqb 2.0.0+ (2020-05)
	CHECK_ENUM_VALUE([qb/qblog.h],[qb_log_conf],[QB_LOG_CONF_MAX_LINE_LEN])
	CHECK_ENUM_VALUE([qb/qblog.h],[qb_log_conf],[QB_LOG_CONF_ELLIPSIS])

	dnl Support Linux-HA fence agents if available
	if test "$cross_compiling" != "yes"; then
	CPPFLAGS="$CPPFLAGS -I${prefix}/include/heartbeat"
	fi
	AC_CHECK_HEADERS([stonith/stonith.h],
	[
	AC_CHECK_LIB([pils], [PILLoadPlugin])
	AC_CHECK_LIB([plumb], [G_main_add_IPC_Channel])
	PCMK_FEATURES="$PCMK_FEATURES lha"
	])
	AM_CONDITIONAL([BUILD_LHA_SUPPORT], [test "$ac_cv_header_stonith_stonith_h" = "yes"])

	dnl ===============================================
	dnl Variables needed for substitution
	dnl ===============================================
	CRM_SCHEMA_DIRECTORY="${datadir}/pacemaker"
	AC_DEFINE_UNQUOTED(CRM_SCHEMA_DIRECTORY,"$CRM_SCHEMA_DIRECTORY", Location for the Pacemaker Relax-NG Schema)
	AC_SUBST(CRM_SCHEMA_DIRECTORY)

	CRM_CORE_DIR="${localstatedir}/lib/pacemaker/cores"
	AC_DEFINE_UNQUOTED([CRM_CORE_DIR], ["$CRM_CORE_DIR"],
	[Directory Pacemaker daemons should change to (without systemd, core files will go here)])
	AC_SUBST(CRM_CORE_DIR)

	if test x"${CRM_DAEMON_USER}" = x""; then
	CRM_DAEMON_USER="hacluster"
	fi
	AC_DEFINE_UNQUOTED(CRM_DAEMON_USER,"$CRM_DAEMON_USER", User to run Pacemaker daemons as)
	AC_SUBST(CRM_DAEMON_USER)

	if test x"${CRM_DAEMON_GROUP}" = x""; then
	CRM_DAEMON_GROUP="haclient"
	fi
	AC_DEFINE_UNQUOTED(CRM_DAEMON_GROUP,"$CRM_DAEMON_GROUP", Group to run Pacemaker daemons as)
	AC_SUBST(CRM_DAEMON_GROUP)

	CRM_PACEMAKER_DIR=${localstatedir}/lib/pacemaker
	AC_DEFINE_UNQUOTED(CRM_PACEMAKER_DIR,"$CRM_PACEMAKER_DIR", Location to store directory produced by Pacemaker daemons)
	AC_SUBST(CRM_PACEMAKER_DIR)

	CRM_BLACKBOX_DIR=${localstatedir}/lib/pacemaker/blackbox
	AC_DEFINE_UNQUOTED(CRM_BLACKBOX_DIR,"$CRM_BLACKBOX_DIR", Where to keep blackbox dumps)
	AC_SUBST(CRM_BLACKBOX_DIR)

	PE_STATE_DIR="${localstatedir}/lib/pacemaker/pengine"
	AC_DEFINE_UNQUOTED(PE_STATE_DIR,"$PE_STATE_DIR", Where to keep scheduler outputs)
	AC_SUBST(PE_STATE_DIR)

	CRM_CONFIG_DIR="${localstatedir}/lib/pacemaker/cib"
	AC_DEFINE_UNQUOTED(CRM_CONFIG_DIR,"$CRM_CONFIG_DIR", Where to keep configuration files)
	AC_SUBST(CRM_CONFIG_DIR)

	CRM_DAEMON_DIR="${libexecdir}/pacemaker"
	AC_DEFINE_UNQUOTED(CRM_DAEMON_DIR,"$CRM_DAEMON_DIR", Location for Pacemaker daemons)
	AC_SUBST(CRM_DAEMON_DIR)

	CRM_STATE_DIR="${runstatedir}/crm"
	AC_DEFINE_UNQUOTED([CRM_STATE_DIR], ["$CRM_STATE_DIR"],
	[Where to keep state files and sockets])
	AC_SUBST(CRM_STATE_DIR)

	CRM_RSCTMP_DIR="${runstatedir}/resource-agents"
	AC_DEFINE_UNQUOTED(CRM_RSCTMP_DIR,"$CRM_RSCTMP_DIR", Where resource agents should keep state files)
	AC_SUBST(CRM_RSCTMP_DIR)

	PACEMAKER_CONFIG_DIR="${sysconfdir}/pacemaker"
	AC_DEFINE_UNQUOTED(PACEMAKER_CONFIG_DIR,"$PACEMAKER_CONFIG_DIR", Where to keep configuration files like authkey)
	AC_SUBST(PACEMAKER_CONFIG_DIR)

	AC_DEFINE_UNQUOTED(SBIN_DIR,"$sbindir",[Location for system binaries])

	AC_PATH_PROGS(GIT, git false)

	AC_MSG_CHECKING([build version])
	BUILD_VERSION=$Format:%h$
	if test $BUILD_VERSION != ":%h$"; then
	AC_MSG_RESULT([$BUILD_VERSION (archive hash)])
	elif test -x $GIT && test -d .git; then
	BUILD_VERSION=`$GIT log --pretty="format:%h" -n 1`
	AC_MSG_RESULT([$BUILD_VERSION (git hash)])
	else
	# The current directory name make a reasonable default
	# Most generated archives will include the hash or tag
	BASE=`basename $PWD`
	BUILD_VERSION=`echo $BASE \| sed s:.*[[Pp]]acemaker-::`
	AC_MSG_RESULT([$BUILD_VERSION (directory name)])
	fi

	AC_DEFINE_UNQUOTED(BUILD_VERSION, "$BUILD_VERSION", Build version)
	AC_SUBST(BUILD_VERSION)

	HAVE_dbus=1
	PKG_CHECK_MODULES([DBUS], [dbus-1],
	[CPPFLAGS="${CPPFLAGS} ${DBUS_CFLAGS}"],
	[HAVE_dbus=0])
	AC_DEFINE_UNQUOTED(SUPPORT_DBUS, $HAVE_dbus, Support dbus)
	AM_CONDITIONAL(BUILD_DBUS, test $HAVE_dbus = 1)
	AC_CHECK_TYPES([DBusBasicValue],,,[[#include <dbus/dbus.h>]])
	if test $HAVE_dbus = 0; then
	PC_NAME_DBUS=""
	else
	PC_NAME_DBUS="dbus-1"
	fi
	AC_SUBST(PC_NAME_DBUS)

	AS_CASE([$enable_systemd],
	[$REQUIRED], [
	AS_IF([test $HAVE_dbus = 0],
	[AC_MSG_FAILURE([Cannot support systemd resources without DBus])])
	AS_IF([test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
	[AC_MSG_FAILURE([Cannot support systemd resources without monotonic clock])])
	AS_IF([check_systemdsystemunitdir], [],
	[AC_MSG_FAILURE([Cannot support systemd resources without systemdsystemunitdir])])
	],
	[$OPTIONAL], [
	AS_IF([test $HAVE_dbus = 0 \
	\|\| test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
	[enable_systemd=$DISABLED],
	[
	AC_MSG_CHECKING([for systemd version (using dbus-send)])
	ret=$({ dbus-send --system --print-reply \
	--dest=org.freedesktop.systemd1 \
	/org/freedesktop/systemd1 \
	org.freedesktop.DBus.Properties.Get \
	string:org.freedesktop.systemd1.Manager \
	string:Version 2>/dev/null \
	\|\| echo "version unavailable"; } \| tail -n1)
	# sanitize output a bit (interested just in value, not type),
	# ret is intentionally unenquoted so as to normalize whitespace
	ret=$(echo ${ret} \| cut -d' ' -f2-)
	AC_MSG_RESULT([${ret}])
	AS_IF([test "$ret" != "unavailable" \
	\|\| systemctl --version 2>/dev/null \| grep -q systemd],
	[
	AS_IF([check_systemdsystemunitdir],
	[enable_systemd=$REQUIRED],
	[enable_systemd=$DISABLED])
	],
	[enable_systemd=$DISABLED]
	)
	])
	],
	)
	AC_MSG_CHECKING([whether to enable support for managing resources via systemd])
	AS_IF([test $enable_systemd -eq $DISABLED], [AC_MSG_RESULT([no])],
	[
	AC_MSG_RESULT([yes])
	PCMK_FEATURES="$PCMK_FEATURES systemd"
	]
	)
	AC_SUBST([systemdsystemunitdir])
	AC_DEFINE_UNQUOTED([SUPPORT_SYSTEMD], [$enable_systemd],
	[Support systemd resources])
	AM_CONDITIONAL([BUILD_SYSTEMD], [test $enable_systemd = $REQUIRED])
	AC_SUBST(SUPPORT_SYSTEMD)

	AS_CASE([$enable_upstart],
	[$REQUIRED], [
	AS_IF([test $HAVE_dbus = 0],
	[AC_MSG_FAILURE([Cannot support Upstart resources without DBus])])
	],
	[$OPTIONAL], [
	AS_IF([test $HAVE_dbus = 0], [enable_upstart=$DISABLED],
	[
	AC_MSG_CHECKING([for Upstart version (using dbus-send)])
	ret=$({ dbus-send --system --print-reply \
	--dest=com.ubuntu.Upstart \
	/com/ubuntu/Upstart org.freedesktop.DBus.Properties.Get \
	string:com.ubuntu.Upstart0_6 string:version 2>/dev/null \
	\|\| echo "version unavailable"; } \| tail -n1)
	# sanitize output a bit (interested just in value, not type),
	# ret is intentionally unenquoted so as to normalize whitespace
	ret=$(echo ${ret} \| cut -d' ' -f2-)
	AC_MSG_RESULT([${ret}])
	AS_IF([test "$ret" != "unavailable" \
	\|\| initctl --version 2>/dev/null \| grep -q upstart],
	[enable_upstart=$REQUIRED],
	[enable_upstart=$DISABLED]
	)
	])
	],
	)
	AC_MSG_CHECKING([whether to enable support for managing resources via Upstart])
	AS_IF([test $enable_upstart -eq $DISABLED], [AC_MSG_RESULT([no])],
	[
	AC_MSG_RESULT([yes])
	PCMK_FEATURES="$PCMK_FEATURES upstart"
	]
	)
	AC_DEFINE_UNQUOTED([SUPPORT_UPSTART], [$enable_upstart],
	[Support Upstart resources])
	AM_CONDITIONAL([BUILD_UPSTART], [test $enable_upstart -eq $REQUIRED])
	AC_SUBST(SUPPORT_UPSTART)

	AS_CASE([$with_nagios],
	[$REQUIRED], [
	AS_IF([test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
	[AC_MSG_FAILURE([Cannot support nagios resources without monotonic clock])])
	],
	[$OPTIONAL], [
	AS_IF([test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "no"],
	[with_nagios=$DISABLED], [with_nagios=$REQUIRED])
	]
	)
	AS_IF([test $with_nagios -eq $REQUIRED], [PCMK_FEATURES="$PCMK_FEATURES nagios"])
	AC_DEFINE_UNQUOTED([SUPPORT_NAGIOS], [$with_nagios], [Support nagios plugins])
	AM_CONDITIONAL([BUILD_NAGIOS], [test $with_nagios -eq $REQUIRED])

	if test x"$NAGIOS_PLUGIN_DIR" = x""; then
	NAGIOS_PLUGIN_DIR="${libexecdir}/nagios/plugins"
	fi

	AC_DEFINE_UNQUOTED(NAGIOS_PLUGIN_DIR, "$NAGIOS_PLUGIN_DIR", Directory for nagios plugins)
	AC_SUBST(NAGIOS_PLUGIN_DIR)

	if test x"$NAGIOS_METADATA_DIR" = x""; then
	NAGIOS_METADATA_DIR="${datadir}/nagios/plugins-metadata"
	fi

	AC_DEFINE_UNQUOTED(NAGIOS_METADATA_DIR, "$NAGIOS_METADATA_DIR", Directory for nagios plugins metadata)
	AC_SUBST(NAGIOS_METADATA_DIR)

	STACKS=""
	CLUSTERLIBS=""
	PC_NAME_CLUSTER=""

	dnl ========================================================================
	dnl Cluster stack - Corosync
	dnl ========================================================================

	COROSYNC_LIBS=""

	AS_CASE([$with_corosync],
	[$REQUIRED], [
	# These will be fatal if unavailable
	PKG_CHECK_MODULES([cpg], [libcpg])
	PKG_CHECK_MODULES([cfg], [libcfg])
	PKG_CHECK_MODULES([cmap], [libcmap])
	PKG_CHECK_MODULES([quorum], [libquorum])
	PKG_CHECK_MODULES([libcorosync_common], [libcorosync_common])
	]
	[$OPTIONAL], [
	PKG_CHECK_MODULES([cpg], [libcpg], [], [with_corosync=$DISABLED])
	PKG_CHECK_MODULES([cfg], [libcfg], [], [with_corosync=$DISABLED])
	PKG_CHECK_MODULES([cmap], [libcmap], [], [with_corosync=$DISABLED])
	PKG_CHECK_MODULES([quorum], [libquorum], [], [with_corosync=$DISABLED])
	PKG_CHECK_MODULES([libcorosync_common], [libcorosync_common], [], [with_corosync=$DISABLED])
	AS_IF([test $with_corosync -ne $DISABLED], [with_corosync=$REQUIRED])
	]
	)
	AS_IF([test $with_corosync -ne $DISABLED],
	[
	AC_MSG_CHECKING([for Corosync 2 or later])
	AC_MSG_RESULT([yes])
	CFLAGS="$CFLAGS $libqb_CFLAGS $cpg_CFLAGS $cfg_CFLAGS $cmap_CFLAGS $quorum_CFLAGS $libcorosync_common_CFLAGS"
	CPPFLAGS="$CPPFLAGS `$PKG_CONFIG --cflags-only-I corosync`"
	COROSYNC_LIBS="$COROSYNC_LIBS $cpg_LIBS $cfg_LIBS $cmap_LIBS $quorum_LIBS $libcorosync_common_LIBS"
	CLUSTERLIBS="$CLUSTERLIBS $COROSYNC_LIBS"
	PC_NAME_CLUSTER="$PC_CLUSTER_NAME libcfg libcmap libcorosync_common libcpg libquorum"
	STACKS="$STACKS corosync-ge-2"

	dnl Shutdown tracking added (back) to corosync Jan 2021
	saved_LIBS="$LIBS"
	LIBS="$LIBS $COROSYNC_LIBS"
	AC_CHECK_FUNCS([corosync_cfg_trackstart])
	LIBS="$saved_LIBS"
	]
	)
	AC_DEFINE_UNQUOTED([SUPPORT_COROSYNC], [$with_corosync],
	[Support the Corosync messaging and membership layer])
	AM_CONDITIONAL([BUILD_CS_SUPPORT], [test $with_corosync -eq $REQUIRED])
	AC_SUBST([SUPPORT_COROSYNC])

	dnl
	dnl Cluster stack - Sanity
	dnl

	AS_IF([test "x$STACKS" != "x"], [AC_MSG_NOTICE([Supported stacks:${STACKS}])],
	[AC_MSG_FAILURE([At least one cluster stack must be supported])])

	PCMK_FEATURES="${PCMK_FEATURES}${STACKS}"

	AC_SUBST(CLUSTERLIBS)
	AC_SUBST(PC_NAME_CLUSTER)

	dnl ========================================================================
	dnl CIB secrets
	dnl ========================================================================

	AS_IF([test $with_cibsecrets -ne $DISABLED],
	[
	with_cibsecrets=$REQUIRED
	PCMK_FEATURES="$PCMK_FEATURES cibsecrets"
	LRM_CIBSECRETS_DIR="${localstatedir}/lib/pacemaker/lrm/secrets"
	AC_DEFINE_UNQUOTED([LRM_CIBSECRETS_DIR], ["$LRM_CIBSECRETS_DIR"],
	[Location for CIB secrets])
	AC_SUBST([LRM_CIBSECRETS_DIR])
	]
	)
	AC_DEFINE_UNQUOTED([SUPPORT_CIBSECRETS], [$with_cibsecrets], [Support CIB secrets])
	AM_CONDITIONAL([BUILD_CIBSECRETS], [test $with_cibsecrets -eq $REQUIRED])

	dnl ========================================================================
	dnl GnuTLS
	dnl ========================================================================

	dnl Require GnuTLS >=2.12.0 (2011-03) for Pacemaker Remote support
	PC_NAME_GNUTLS=""
	AS_CASE([$with_gnutls],
	[$REQUIRED], [
	REQUIRE_LIB([gnutls], [gnutls_sec_param_to_pk_bits])
	REQUIRE_HEADER([gnutls/gnutls.h])
	],
	[$OPTIONAL], [
	AC_CHECK_LIB([gnutls], [gnutls_sec_param_to_pk_bits],
	[], [with_gnutls=$DISABLED])
	AC_CHECK_HEADERS([gnutls/gnutls.h], [], [with_gnutls=$DISABLED])
	]
	)
	AS_IF([test $with_gnutls -ne $DISABLED],
	[
	PC_NAME_GNUTLS="gnutls"
	PCMK_FEATURES="$PCMK_FEATURES remote"
	]
	)
	AC_SUBST([PC_NAME_GNUTLS])
	AM_CONDITIONAL([BUILD_REMOTE], [test $with_gnutls -ne $DISABLED])

	dnl ========================================================================
	dnl System Health
	dnl ========================================================================

	dnl Check if servicelog development package is installed
	SERVICELOG=servicelog-1
	SERVICELOG_EXISTS="no"
	AC_MSG_CHECKING([for $SERVICELOG packages])
	if
	$PKG_CONFIG --exists $SERVICELOG
	then
	PKG_CHECK_MODULES([SERVICELOG], [servicelog-1])
	SERVICELOG_EXISTS="yes"
	PCMK_FEATURES="$PCMK_FEATURES servicelog"
	fi
	AC_MSG_RESULT([$SERVICELOG_EXISTS])
	AM_CONDITIONAL(BUILD_SERVICELOG, test "$SERVICELOG_EXISTS" = "yes")

	dnl Check if OpenIMPI packages and servicelog are installed
	OPENIPMI="OpenIPMI OpenIPMIposix"
	OPENIPMI_SERVICELOG_EXISTS="no"
	AC_MSG_CHECKING([for $SERVICELOG $OPENIPMI packages])
	if
	$PKG_CONFIG --exists $OPENIPMI $SERVICELOG
	then
	PKG_CHECK_MODULES([OPENIPMI_SERVICELOG],[OpenIPMI OpenIPMIposix])
	REQUIRE_HEADER([malloc.h])
	OPENIPMI_SERVICELOG_EXISTS="yes"
	PCMK_FEATURES="$PCMK_FEATURES ipmiservicelogd"
	fi
	AC_MSG_RESULT([$OPENIPMI_SERVICELOG_EXISTS])
	AM_CONDITIONAL(BUILD_OPENIPMI_SERVICELOG, test "$OPENIPMI_SERVICELOG_EXISTS" = "yes")

	# --- ASAN/UBSAN/TSAN (see man gcc) ---
	# when using SANitizers, we need to pass the -fsanitize..
	# to both CFLAGS and LDFLAGS. The CFLAGS/LDFLAGS must be
	# specified as first in the list or there will be runtime
	# issues (for example user has to LD_PRELOAD asan for it to work
	# properly).

	AS_IF([test -n "${SANITIZERS}"], [
	SANITIZERS=$(echo $SANITIZERS \| sed -e 's/,/ /g')
	for SANITIZER in $SANITIZERS
	do
	AS_CASE([$SANITIZER],
	[asan\|ASAN], [
	SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=address"
	SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=address -lasan"
	PCMK_FEATURES="$PCMK_FEATURES asan"
	REQUIRE_LIB([asan],[main])
	],
	[ubsan\|UBSAN], [
	SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=undefined"
	SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=undefined -lubsan"
	PCMK_FEATURES="$PCMK_FEATURES ubsan"
	REQUIRE_LIB([ubsan],[main])
	],
	[tsan\|TSAN], [
	SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=thread"
	SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=thread -ltsan"
	PCMK_FEATURES="$PCMK_FEATURES tsan"
	REQUIRE_LIB([tsan],[main])
	])
	done
	])


	dnl ========================================================================
	dnl Compiler flags
	dnl ========================================================================

	dnl Make sure that CFLAGS is not exported. If the user did
	dnl not have CFLAGS in their environment then this should have
	dnl no effect. However if CFLAGS was exported from the user's
	dnl environment, then the new CFLAGS will also be exported
	dnl to sub processes.
	if export \| fgrep " CFLAGS=" > /dev/null; then
	SAVED_CFLAGS="$CFLAGS"
	unset CFLAGS
	CFLAGS="$SAVED_CFLAGS"
	unset SAVED_CFLAGS
	fi

	CC_EXTRAS=""

	AS_IF([test "$GCC" != yes], [CFLAGS="$CFLAGS -g"], [
	CFLAGS="$CFLAGS -ggdb"

	dnl When we don't have diagnostic push / pull, we can't explicitly disable
	dnl checking for nonliteral formats in the places where they occur on purpose
	dnl thus we disable nonliteral format checking globally as we are aborting
	dnl on warnings.
	dnl what makes the things really ugly is that nonliteral format checking is
	dnl obviously available as an extra switch in very modern gcc but for older
	dnl gcc this is part of -Wformat=2
	dnl so if we have push/pull we can enable -Wformat=2 -Wformat-nonliteral
	dnl if we don't have push/pull but -Wformat-nonliteral we can enable -Wformat=2
	dnl otherwise none of both

	gcc_diagnostic_push_pull=no
	cc_temp_flags "$CFLAGS $WERROR"
	AC_MSG_CHECKING([for gcc diagnostic push / pull])
	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
	#pragma GCC diagnostic push
	#pragma GCC diagnostic pop
	]])],
	[
	AC_MSG_RESULT([yes])
	gcc_diagnostic_push_pull=yes
	], AC_MSG_RESULT([no]))
	cc_restore_flags

	AS_IF([cc_supports_flag "-Wformat-nonliteral"],
	[gcc_format_nonliteral=yes],
	[gcc_format_nonliteral=no])

	# We had to eliminate -Wnested-externs because of libtool changes
	# Make sure to order options so that the former stand for prerequisites
	# of the latter (e.g., -Wformat-nonliteral requires -Wformat).
	EXTRA_FLAGS="-fgnu89-inline"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wall"
	EXTRA_FLAGS="$EXTRA_FLAGS -Waggregate-return"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wbad-function-cast"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wcast-align"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wdeclaration-after-statement"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wendif-labels"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wfloat-equal"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wformat-security"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wmissing-prototypes"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wmissing-declarations"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wnested-externs"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wno-long-long"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wno-strict-aliasing"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wpointer-arith"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wstrict-prototypes"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wwrite-strings"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wunused-but-set-variable"
	EXTRA_FLAGS="$EXTRA_FLAGS -Wunsigned-char"

	AS_IF([test "x$gcc_diagnostic_push_pull" = "xyes"],
	[
	AC_DEFINE([GCC_FORMAT_NONLITERAL_CHECKING_ENABLED], [],
	[gcc can complain about nonliterals in format])
	EXTRA_FLAGS="$EXTRA_FLAGS -Wformat=2 -Wformat-nonliteral"
	],
	[test "x$gcc_format_nonliteral" = "xyes"],
	[EXTRA_FLAGS="$EXTRA_FLAGS -Wformat=2"])

	# Additional warnings it might be nice to enable one day
	# -Wshadow
	# -Wunreachable-code
	for j in $EXTRA_FLAGS
	do
	AS_IF([cc_supports_flag $CC_EXTRAS $j], [CC_EXTRAS="$CC_EXTRAS $j"])
	done

	AC_MSG_NOTICE([Using additional gcc flags: ${CC_EXTRAS}])
	])

	dnl
	dnl Hardening flags
	dnl
	dnl The prime control of whether to apply (targeted) hardening build flags and
	dnl which ones is --{enable,disable}-hardening option passed to ./configure:
	dnl
	dnl --enable-hardening=try (default):
	dnl depending on whether any of CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE,
	dnl CFLAGS_HARDENED_LIB or LDFLAGS_HARDENED_LIB environment variables
	dnl (see below) is set and non-null, all these custom flags (even if not
	dnl set) are used as are, otherwise the best effort is made to offer
	dnl reasonably strong hardening in several categories (RELRO, PIE,
	dnl "bind now", stack protector) according to what the selected toolchain
	dnl can offer
	dnl
	dnl --enable-hardening:
	dnl same effect as --enable-hardening=try when the environment variables
	dnl in question are suppressed
	dnl
	dnl --disable-hardening:
	dnl do not apply any targeted hardening measures at all
	dnl
	dnl The user-injected environment variables that regulate the hardening in
	dnl default case are as follows:
	dnl
	dnl * CFLAGS_HARDENED_EXE, LDFLAGS_HARDENED_EXE
	dnl compiler and linker flags (respectively) for daemon programs
	dnl (pacemakerd, pacemaker-attrd, pacemaker-controld, pacemaker-execd,
	dnl pacemaker-based, pacemaker-fenced, pacemaker-remoted,
	dnl pacemaker-schedulerd)
	dnl
	dnl * CFLAGS_HARDENED_LIB, LDFLAGS_HARDENED_LIB
	dnl compiler and linker flags (respectively) for libraries linked
	dnl with the daemon programs
	dnl
	dnl Note that these are purposedly targeted variables (addressing particular
	dnl targets all over the scattered Makefiles) and have no effect outside of
	dnl the predestined scope (e.g., CLI utilities). For a global reach,
	dnl use CFLAGS, LDFLAGS, etc. as usual.
	dnl
	dnl For guidance on the suitable flags consult, for instance:
	dnl https://fedoraproject.org/wiki/Changes/Harden_All_Packages#Detailed_Harden_Flags_Description
	dnl https://owasp.org/index.php/C-Based_Toolchain_Hardening#GCC.2FBinutils
	dnl

	AS_IF([test $enable_hardening -eq $OPTIONAL],
	[
	AS_IF([test "$(env \| grep -Ec '^(C\|LD)FLAGS_HARDENED_(EXE\|LIB)=.')" = 0],
	[enable_hardening=$REQUIRED],
	[AC_MSG_NOTICE([Hardening: using custom flags from environment])]
	)
	],
	[
	unset CFLAGS_HARDENED_EXE
	unset CFLAGS_HARDENED_LIB
	unset LDFLAGS_HARDENED_EXE
	unset LDFLAGS_HARDENED_LIB
	]
	)
	AS_CASE([$enable_hardening],
	[$DISABLED], [AC_MSG_NOTICE([Hardening: explicitly disabled])],
	[$REQUIRED], [
	CFLAGS_HARDENED_EXE=
	CFLAGS_HARDENED_LIB=
	LDFLAGS_HARDENED_EXE=
	LDFLAGS_HARDENED_LIB=
	relro=0
	pie=0
	bindnow=0
	stackprot="none"
	# daemons incl. libs: partial RELRO
	flag="-Wl,-z,relro"
	CC_CHECK_LDFLAGS(["${flag}"],
	[
	LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
	LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"
	relro=1
	])
	# daemons: PIE for both CFLAGS and LDFLAGS
	AS_IF([cc_supports_flag -fPIE],
	[
	flag="-pie"
	CC_CHECK_LDFLAGS(["${flag}"],
	[
	CFLAGS_HARDENED_EXE="${CFLAGS_HARDENED_EXE} -fPIE"
	LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
	pie=1
	])
	]
	)
	# daemons incl. libs: full RELRO if sensible + as-needed linking
	# so as to possibly mitigate startup performance
	# hit caused by excessive linking with unneeded
	# libraries
	AS_IF([test "${relro}" = 1 && test "${pie}" = 1],
	[
	flag="-Wl,-z,now"
	CC_CHECK_LDFLAGS(["${flag}"],
	[
	LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
	LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"
	bindnow=1
	])
	]
	)
	AS_IF([test "${bindnow}" = 1],
	[
	flag="-Wl,--as-needed"
	CC_CHECK_LDFLAGS(["${flag}"],
	[
	LDFLAGS_HARDENED_EXE="${LDFLAGS_HARDENED_EXE} ${flag}"
	LDFLAGS_HARDENED_LIB="${LDFLAGS_HARDENED_LIB} ${flag}"
	])
	])
	# universal: prefer strong > all > default stack protector if possible
	flag=
	AS_IF([cc_supports_flag -fstack-protector-strong],
	[
	flag="-fstack-protector-strong"
	stackprot="strong"
	],
	[cc_supports_flag -fstack-protector-all],
	[
	flag="-fstack-protector-all"
	stackprot="all"
	],
	[cc_supports_flag -fstack-protector],
	[
	flag="-fstack-protector"
	stackprot="default"
	]
	)
	AS_IF([test -n "${flag}"], [CC_EXTRAS="${CC_EXTRAS} ${flag}"])
	# universal: enable stack clash protection if possible
	AS_IF([cc_supports_flag -fstack-clash-protection],
	[
	CC_EXTRAS="${CC_EXTRAS} -fstack-clash-protection"
	AS_IF([test "${stackprot}" = "none"],
	[stackprot="clash-only"],
	[stackprot="${stackprot}+clash"]
	)
	]
	)
	# Log a summary
	AS_IF([test "${relro}" = 1 \|\| test "${pie}" = 1 \|\| test "${stackprot}" != "none"],
	[AC_MSG_NOTICE(m4_normalize([Hardening:
	relro=${relro}
	pie=${pie}
	bindnow=${bindnow}
	stackprot=${stackprot}]))
	],
	[AC_MSG_WARN([Hardening: no suitable features in the toolchain detected])]
	)
	],
	)

	CFLAGS="$SANITIZERS_CFLAGS $CFLAGS $CC_EXTRAS"
	LDFLAGS="$SANITIZERS_LDFLAGS $LDFLAGS"
	CFLAGS_HARDENED_EXE="$SANITIZERS_CFLAGS $CFLAGS_HARDENED_EXE"
	LDFLAGS_HARDENED_EXE="$SANITIZERS_LDFLAGS $LDFLAGS_HARDENED_EXE"

	NON_FATAL_CFLAGS="$CFLAGS"
	AC_SUBST(NON_FATAL_CFLAGS)

	dnl
	dnl We reset CFLAGS to include our warnings after all function
	dnl checking goes on, so that our warning flags don't keep the
	dnl AC_*FUNCS() calls above from working. In particular, -Werror will
	dnl always cause us troubles if we set it before here.
	dnl
	dnl
	AS_IF([test $enable_fatal_warnings -ne $DISABLED], [
	AC_MSG_NOTICE([Enabling fatal compiler warnings])
	CFLAGS="$CFLAGS $WERROR"
	])
	AC_SUBST(CFLAGS)

	dnl This is useful for use in Makefiles that need to remove one specific flag
	CFLAGS_COPY="$CFLAGS"
	AC_SUBST(CFLAGS_COPY)

	AC_SUBST(LIBADD_DL) dnl extra flags for dynamic linking libraries

	AC_SUBST(LOCALE)

	dnl Options for cleaning up the compiler output
	AS_IF([test $enable_quiet -ne $DISABLED],
	[
	AC_MSG_NOTICE([Suppressing make details])
	QUIET_LIBTOOL_OPTS="--silent"
	QUIET_MAKE_OPTS="-s" # POSIX compliant
	],
	[
	QUIET_LIBTOOL_OPTS=""
	QUIET_MAKE_OPTS=""
	]
	)

	dnl Put the above variables to use
	LIBTOOL="${LIBTOOL} --tag=CC \$(QUIET_LIBTOOL_OPTS)"
	MAKEFLAGS="${MAKEFLAGS} ${QUIET_MAKE_OPTS}"

	# Make features list available (sorted alphabetically, without leading space)
	PCMK_FEATURES=`echo "$PCMK_FEATURES" \| sed -e 's/^ //' -e 's/ /\n/g' \| sort \| xargs`
	AC_DEFINE_UNQUOTED(CRM_FEATURES, "$PCMK_FEATURES", Set of enabled features)
	AC_SUBST(PCMK_FEATURES)

	AC_SUBST(CC)
	AC_SUBST(MAKEFLAGS)
	AC_SUBST(LIBTOOL)
	AC_SUBST(QUIET_LIBTOOL_OPTS)

	dnl Files we output that need to be executable
	CONFIG_FILES_EXEC([cts/cts-cli],
	[cts/cts-coverage],
	[cts/cts-exec],
	[cts/cts-fencing],
	[cts/cts-regression],
	[cts/cts-scheduler],
	[cts/lxc_autogen.sh],
	[cts/benchmark/clubench],
	[cts/lab/CTSlab.py],
	[cts/lab/OCFIPraTest.py],
	[cts/lab/cluster_test],
	[cts/lab/cts],
	[cts/lab/cts-log-watcher],
	[cts/support/LSBDummy],
	[cts/support/cts-support],
	[cts/support/fence_dummy],
	[cts/support/pacemaker-cts-dummyd],
	[daemons/fenced/fence_legacy],
	[daemons/fenced/fence_watchdog],
	[doc/abi-check],
	[extra/resources/ClusterMon],
	[extra/resources/HealthSMART],
	[extra/resources/SysInfo],
	[extra/resources/ifspeed],
	[extra/resources/o2cb],
	[maint/bumplibs],
	[tools/crm_failcount],
	[tools/crm_master],
	[tools/crm_report],
	[tools/crm_standby],
	[tools/cibsecret],
	[tools/pcmk_simtimes])

	dnl Other files we output
	AC_CONFIG_FILES(Makefile \
	cts/Makefile \
	cts/benchmark/Makefile \
	cts/lab/Makefile \
	cts/lab/CTSvars.py \
	cts/scheduler/Makefile \
	cts/scheduler/dot/Makefile \
	cts/scheduler/exp/Makefile \
	cts/scheduler/scores/Makefile \
	cts/scheduler/stderr/Makefile \
	cts/scheduler/summary/Makefile \
	cts/scheduler/xml/Makefile \
	cts/support/Makefile \
	cts/support/pacemaker-cts-dummyd@.service \
	daemons/Makefile \
	daemons/attrd/Makefile \
	daemons/based/Makefile \
	daemons/controld/Makefile \
	daemons/execd/Makefile \
	daemons/execd/pacemaker_remote \
	daemons/execd/pacemaker_remote.service \
	daemons/fenced/Makefile \
	daemons/pacemakerd/Makefile \
	daemons/pacemakerd/pacemaker.service \
	daemons/pacemakerd/pacemaker.upstart \
	daemons/pacemakerd/pacemaker.combined.upstart \
	daemons/schedulerd/Makefile \
	devel/Makefile \
	doc/Doxyfile \
	doc/Makefile \
	doc/sphinx/Makefile \
	etc/Makefile \
	etc/init.d/pacemaker \
	etc/logrotate.d/pacemaker \
	extra/Makefile \
	extra/alerts/Makefile \
	extra/resources/Makefile \
	include/Makefile \
	include/crm/Makefile \
	include/crm/cib/Makefile \
	include/crm/common/Makefile \
	include/crm/cluster/Makefile \
	include/crm/fencing/Makefile \
	include/crm/pengine/Makefile \
	include/pcmki/Makefile \
	replace/Makefile \
	lib/Makefile \
	lib/libpacemaker.pc \
	lib/pacemaker.pc \
	lib/pacemaker-cib.pc \
	lib/pacemaker-lrmd.pc \
	lib/pacemaker-service.pc \
	lib/pacemaker-pe_rules.pc \
	lib/pacemaker-pe_status.pc \
	lib/pacemaker-fencing.pc \
	lib/pacemaker-cluster.pc \
	lib/common/Makefile \
	lib/common/tests/Makefile \
	- lib/common/tests/acl/Makefile \
	+ lib/common/tests/acl/Makefile \
	lib/common/tests/agents/Makefile \
	lib/common/tests/cmdline/Makefile \
	lib/common/tests/flags/Makefile \
	lib/common/tests/io/Makefile \
	lib/common/tests/iso8601/Makefile \
	lib/common/tests/lists/Makefile \
	lib/common/tests/nvpair/Makefile \
	lib/common/tests/operations/Makefile \
	lib/common/tests/results/Makefile \
	+ lib/common/tests/scores/Makefile \
	lib/common/tests/strings/Makefile \
	lib/common/tests/utils/Makefile \
	lib/common/tests/xpath/Makefile \
	lib/cluster/Makefile \
	lib/cib/Makefile \
	lib/gnu/Makefile \
	lib/pacemaker/Makefile \
	lib/pengine/Makefile \
	lib/pengine/tests/Makefile \
	lib/pengine/tests/rules/Makefile \
	lib/fencing/Makefile \
	lib/lrmd/Makefile \
	lib/services/Makefile \
	maint/Makefile \
	rpm/Makefile \
	tests/Makefile \
	tools/Makefile \
	tools/report.collector \
	tools/report.common \
	tools/crm_mon.service \
	tools/crm_mon.upstart \
	xml/Makefile \
	xml/pacemaker-schemas.pc \
	po/Makefile.in \
	)

	dnl Now process the entire list of files added by previous
	dnl calls to AC_CONFIG_FILES()
	AC_OUTPUT()

	dnl *****************
	dnl Configure summary
	dnl *****************

	AC_MSG_NOTICE([])
	AC_MSG_NOTICE([$PACKAGE configuration:])
	AC_MSG_NOTICE([ Version = ${VERSION} (Build: $BUILD_VERSION)])
	AC_MSG_NOTICE([ Features = ${PCMK_FEATURES}])
	AC_MSG_NOTICE([])
	AC_MSG_NOTICE([ Prefix = ${prefix}])
	AC_MSG_NOTICE([ Executables = ${sbindir}])
	AC_MSG_NOTICE([ Man pages = ${mandir}])
	AC_MSG_NOTICE([ Libraries = ${libdir}])
	AC_MSG_NOTICE([ Header files = ${includedir}])
	AC_MSG_NOTICE([ Arch-independent files = ${datadir}])
	AC_MSG_NOTICE([ State information = ${localstatedir}])
	AC_MSG_NOTICE([ System configuration = ${sysconfdir}])
	AC_MSG_NOTICE([ OCF agents = ${OCF_ROOT_DIR}])
	AC_MSG_NOTICE([])
	AC_MSG_NOTICE([ HA group name = ${CRM_DAEMON_GROUP}])
	AC_MSG_NOTICE([ HA user name = ${CRM_DAEMON_USER}])
	AC_MSG_NOTICE([])
	AC_MSG_NOTICE([ CFLAGS = ${CFLAGS}])
	AC_MSG_NOTICE([ CFLAGS_HARDENED_EXE = ${CFLAGS_HARDENED_EXE}])
	AC_MSG_NOTICE([ CFLAGS_HARDENED_LIB = ${CFLAGS_HARDENED_LIB}])
	AC_MSG_NOTICE([ LDFLAGS_HARDENED_EXE = ${LDFLAGS_HARDENED_EXE}])
	AC_MSG_NOTICE([ LDFLAGS_HARDENED_LIB = ${LDFLAGS_HARDENED_LIB}])
	AC_MSG_NOTICE([ Libraries = ${LIBS}])
	AC_MSG_NOTICE([ Stack Libraries = ${CLUSTERLIBS}])
	AC_MSG_NOTICE([ Unix socket auth method = ${us_auth}])
	diff --git a/cts/scheduler/xml/systemhealthm1.xml b/cts/scheduler/xml/systemhealthm1.xml
	index 6c4a4d60fe..a5102d007c 100644
	--- a/cts/scheduler/xml/systemhealthm1.xml
	+++ b/cts/scheduler/xml/systemhealthm1.xml
	@@ -1,51 +1,51 @@
	<cib validate-with="pacemaker-3.0" crm_feature_set="3.0.1" have-quorum="1" admin_epoch="0" epoch="13924" num_updates="0" cib-last-written="Fri Jul 13 13:51:16 2012" dc-uuid="c97a3ee5-02d8-4fad-a9fb-a79ae2b35549">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.0.3-b133b3f19797c00f9189f4b66b513963f9d25db9"/>
	- <nvpair id="cib-bootstrap-options-pe-node-health-strategy" name="pe-node-health-strategy" value="migrate-on-red"/>
	+ <nvpair id="cib-bootstrap-options-node-health-strategy" name="node-health-strategy" value="migrate-on-red"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="c97a3ee5-02d8-4fad-a9fb-a79ae2b35549" uname="hs21c" type="member"/>
	<node id="737318c6-0f92-4592-9754-45967d45aff7" uname="hs21d" type="member"/>
	</nodes>
	<resources>
	<primitive id="stonith-1" class="stonith" type="dummy"/>
	<primitive class="ocf" id="apache_1" provider="heartbeat" type="apache">
	<meta_attributes id="apache_1-meta_attributes">
	<nvpair id="nvpair-e97ad6f0-db94-47af-8f2c-056a80a21a4d" name="target-role" value="Started"/>
	</meta_attributes>
	<operations id="apache_1-operations">
	<op id="apache_1-op-monitor-10" interval="10" name="monitor" start-delay="1m" timeout="20"/>
	</operations>
	<instance_attributes id="apache_1-instance_attributes">
	<nvpair id="nvpair-a915b958-b215-4fa1-b465-23ab60aa06ca" name="configfile"/>
	</instance_attributes>
	</primitive>
	<primitive class="ocf" id="nfs_1" provider="heartbeat" type="Filesystem">
	<meta_attributes id="nfs_1-meta_attributes">
	<nvpair id="nvpair-c6f3f9e7-ec22-4c24-b3f6-3828228529dd" name="target-role" value="Started"/>
	</meta_attributes>
	<operations id="nfs_1-operations">
	<op id="nfs_1-op-monitor-20" interval="20" name="monitor" start-delay="10" timeout="40"/>
	</operations>
	<instance_attributes id="nfs_1-instance_attributes">
	<nvpair id="nvpair-601b9377-3c22-4094-b8c1-8e7c12b3b00a" name="device" value="hs21d:/nfs"/>
	<nvpair id="nvpair-1ea657a3-eaaf-48f3-912e-ce3a3431c220" name="directory" value="/mnt/tmp"/>
	<nvpair id="nvpair-08b520ca-8679-4157-a0e3-d70985948290" name="fstype" value="nfs"/>
	<nvpair id="nvpair-b5699b8c-bc64-4efa-91d1-5d0618ec7382" name="options" value="ro"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints>
	<rsc_location id="preferred_location_apache_1" node="hs21c" rsc="apache_1" score="100"/>
	<rsc_location id="preferred_location_nfs_1" node="hs21c" rsc="nfs_1" score="100"/>
	</constraints>
	<op_defaults>
	<meta_attributes id="op_defaults-options"/>
	</op_defaults>
	<rsc_defaults/>
	</configuration>
	<status/>
	</cib>
	diff --git a/cts/scheduler/xml/systemhealthn1.xml b/cts/scheduler/xml/systemhealthn1.xml
	index fef0ebb9ea..9b30696c6c 100644
	--- a/cts/scheduler/xml/systemhealthn1.xml
	+++ b/cts/scheduler/xml/systemhealthn1.xml
	@@ -1,51 +1,51 @@
	<cib validate-with="pacemaker-3.0" crm_feature_set="3.0.1" have-quorum="1" admin_epoch="0" epoch="13924" num_updates="0" cib-last-written="Fri Jul 13 13:51:16 2012" dc-uuid="c97a3ee5-02d8-4fad-a9fb-a79ae2b35549">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.0.3-b133b3f19797c00f9189f4b66b513963f9d25db9"/>
	- <nvpair id="cib-bootstrap-options-pe-node-health-strategy" name="pe-node-health-strategy" value="none"/>
	+ <nvpair id="cib-bootstrap-options-node-health-strategy" name="node-health-strategy" value="none"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="c97a3ee5-02d8-4fad-a9fb-a79ae2b35549" uname="hs21c" type="member"/>
	<node id="737318c6-0f92-4592-9754-45967d45aff7" uname="hs21d" type="member"/>
	</nodes>
	<resources>
	<primitive id="stonith-1" class="stonith" type="dummy"/>
	<primitive class="ocf" id="apache_1" provider="heartbeat" type="apache">
	<meta_attributes id="apache_1-meta_attributes">
	<nvpair id="nvpair-e97ad6f0-db94-47af-8f2c-056a80a21a4d" name="target-role" value="Started"/>
	</meta_attributes>
	<operations id="apache_1-operations">
	<op id="apache_1-op-monitor-10" interval="10" name="monitor" start-delay="1m" timeout="20"/>
	</operations>
	<instance_attributes id="apache_1-instance_attributes">
	<nvpair id="nvpair-a915b958-b215-4fa1-b465-23ab60aa06ca" name="configfile"/>
	</instance_attributes>
	</primitive>
	<primitive class="ocf" id="nfs_1" provider="heartbeat" type="Filesystem">
	<meta_attributes id="nfs_1-meta_attributes">
	<nvpair id="nvpair-c6f3f9e7-ec22-4c24-b3f6-3828228529dd" name="target-role" value="Started"/>
	</meta_attributes>
	<operations id="nfs_1-operations">
	<op id="nfs_1-op-monitor-20" interval="20" name="monitor" start-delay="10" timeout="40"/>
	</operations>
	<instance_attributes id="nfs_1-instance_attributes">
	<nvpair id="nvpair-601b9377-3c22-4094-b8c1-8e7c12b3b00a" name="device" value="hs21d:/nfs"/>
	<nvpair id="nvpair-1ea657a3-eaaf-48f3-912e-ce3a3431c220" name="directory" value="/mnt/tmp"/>
	<nvpair id="nvpair-08b520ca-8679-4157-a0e3-d70985948290" name="fstype" value="nfs"/>
	<nvpair id="nvpair-b5699b8c-bc64-4efa-91d1-5d0618ec7382" name="options" value="ro"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints>
	<rsc_location id="preferred_location_apache_1" node="hs21c" rsc="apache_1" score="100"/>
	<rsc_location id="preferred_location_nfs_1" node="hs21c" rsc="nfs_1" score="100"/>
	</constraints>
	<op_defaults>
	<meta_attributes id="op_defaults-options"/>
	</op_defaults>
	<rsc_defaults/>
	</configuration>
	<status/>
	</cib>
	diff --git a/cts/scheduler/xml/systemhealtho1.xml b/cts/scheduler/xml/systemhealtho1.xml
	index da6fb2d6a8..56ee19d23e 100644
	--- a/cts/scheduler/xml/systemhealtho1.xml
	+++ b/cts/scheduler/xml/systemhealtho1.xml
	@@ -1,51 +1,51 @@
	<cib validate-with="pacemaker-3.0" crm_feature_set="3.0.1" have-quorum="1" admin_epoch="0" epoch="13924" num_updates="0" cib-last-written="Fri Jul 13 13:51:16 2012" dc-uuid="c97a3ee5-02d8-4fad-a9fb-a79ae2b35549">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.0.3-b133b3f19797c00f9189f4b66b513963f9d25db9"/>
	- <nvpair id="cib-bootstrap-options-pe-node-health-strategy" name="pe-node-health-strategy" value="only-green"/>
	+ <nvpair id="cib-bootstrap-options-node-health-strategy" name="node-health-strategy" value="only-green"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="c97a3ee5-02d8-4fad-a9fb-a79ae2b35549" uname="hs21c" type="member"/>
	<node id="737318c6-0f92-4592-9754-45967d45aff7" uname="hs21d" type="member"/>
	</nodes>
	<resources>
	<primitive id="stonith-1" class="stonith" type="dummy"/>
	<primitive class="ocf" id="apache_1" provider="heartbeat" type="apache">
	<meta_attributes id="apache_1-meta_attributes">
	<nvpair id="nvpair-e97ad6f0-db94-47af-8f2c-056a80a21a4d" name="target-role" value="Started"/>
	</meta_attributes>
	<operations id="apache_1-operations">
	<op id="apache_1-op-monitor-10" interval="10" name="monitor" start-delay="1m" timeout="20"/>
	</operations>
	<instance_attributes id="apache_1-instance_attributes">
	<nvpair id="nvpair-a915b958-b215-4fa1-b465-23ab60aa06ca" name="configfile"/>
	</instance_attributes>
	</primitive>
	<primitive class="ocf" id="nfs_1" provider="heartbeat" type="Filesystem">
	<meta_attributes id="nfs_1-meta_attributes">
	<nvpair id="nvpair-c6f3f9e7-ec22-4c24-b3f6-3828228529dd" name="target-role" value="Started"/>
	</meta_attributes>
	<operations id="nfs_1-operations">
	<op id="nfs_1-op-monitor-20" interval="20" name="monitor" start-delay="10" timeout="40"/>
	</operations>
	<instance_attributes id="nfs_1-instance_attributes">
	<nvpair id="nvpair-601b9377-3c22-4094-b8c1-8e7c12b3b00a" name="device" value="hs21d:/nfs"/>
	<nvpair id="nvpair-1ea657a3-eaaf-48f3-912e-ce3a3431c220" name="directory" value="/mnt/tmp"/>
	<nvpair id="nvpair-08b520ca-8679-4157-a0e3-d70985948290" name="fstype" value="nfs"/>
	<nvpair id="nvpair-b5699b8c-bc64-4efa-91d1-5d0618ec7382" name="options" value="ro"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints>
	<rsc_location id="preferred_location_apache_1" node="hs21c" rsc="apache_1" score="100"/>
	<rsc_location id="preferred_location_nfs_1" node="hs21c" rsc="nfs_1" score="100"/>
	</constraints>
	<op_defaults>
	<meta_attributes id="op_defaults-options"/>
	</op_defaults>
	<rsc_defaults/>
	</configuration>
	<status/>
	</cib>
	diff --git a/cts/scheduler/xml/systemhealthp1.xml b/cts/scheduler/xml/systemhealthp1.xml
	index 4c2279c84a..2e68c9daf8 100644
	--- a/cts/scheduler/xml/systemhealthp1.xml
	+++ b/cts/scheduler/xml/systemhealthp1.xml
	@@ -1,51 +1,51 @@
	<cib validate-with="pacemaker-3.0" crm_feature_set="3.0.1" have-quorum="1" admin_epoch="0" epoch="13924" num_updates="0" cib-last-written="Fri Jul 13 13:51:16 2012" dc-uuid="c97a3ee5-02d8-4fad-a9fb-a79ae2b35549">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.0.3-b133b3f19797c00f9189f4b66b513963f9d25db9"/>
	- <nvpair id="cib-bootstrap-options-pe-node-health-strategy" name="pe-node-health-strategy" value="progressive"/>
	+ <nvpair id="cib-bootstrap-options-node-health-strategy" name="node-health-strategy" value="progressive"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="c97a3ee5-02d8-4fad-a9fb-a79ae2b35549" uname="hs21c" type="member"/>
	<node id="737318c6-0f92-4592-9754-45967d45aff7" uname="hs21d" type="member"/>
	</nodes>
	<resources>
	<primitive id="stonith-1" class="stonith" type="dummy"/>
	<primitive class="ocf" id="apache_1" provider="heartbeat" type="apache">
	<meta_attributes id="apache_1-meta_attributes">
	<nvpair id="nvpair-e97ad6f0-db94-47af-8f2c-056a80a21a4d" name="target-role" value="Started"/>
	</meta_attributes>
	<operations id="apache_1-operations">
	<op id="apache_1-op-monitor-10" interval="10" name="monitor" start-delay="1m" timeout="20"/>
	</operations>
	<instance_attributes id="apache_1-instance_attributes">
	<nvpair id="nvpair-a915b958-b215-4fa1-b465-23ab60aa06ca" name="configfile"/>
	</instance_attributes>
	</primitive>
	<primitive class="ocf" id="nfs_1" provider="heartbeat" type="Filesystem">
	<meta_attributes id="nfs_1-meta_attributes">
	<nvpair id="nvpair-c6f3f9e7-ec22-4c24-b3f6-3828228529dd" name="target-role" value="Started"/>
	</meta_attributes>
	<operations id="nfs_1-operations">
	<op id="nfs_1-op-monitor-20" interval="20" name="monitor" start-delay="10" timeout="40"/>
	</operations>
	<instance_attributes id="nfs_1-instance_attributes">
	<nvpair id="nvpair-601b9377-3c22-4094-b8c1-8e7c12b3b00a" name="device" value="hs21d:/nfs"/>
	<nvpair id="nvpair-1ea657a3-eaaf-48f3-912e-ce3a3431c220" name="directory" value="/mnt/tmp"/>
	<nvpair id="nvpair-08b520ca-8679-4157-a0e3-d70985948290" name="fstype" value="nfs"/>
	<nvpair id="nvpair-b5699b8c-bc64-4efa-91d1-5d0618ec7382" name="options" value="ro"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints>
	<rsc_location id="preferred_location_apache_1" node="hs21c" rsc="apache_1" score="100"/>
	<rsc_location id="preferred_location_nfs_1" node="hs21c" rsc="nfs_1" score="100"/>
	</constraints>
	<op_defaults>
	<meta_attributes id="op_defaults-options"/>
	</op_defaults>
	<rsc_defaults/>
	</configuration>
	<status/>
	</cib>
	diff --git a/include/crm/common/util.h b/include/crm/common/util.h
	index f6c91e6993..cb2461713f 100644
	--- a/include/crm/common/util.h
	+++ b/include/crm/common/util.h
	@@ -1,151 +1,154 @@
	/*
	- * Copyright 2004-2021 the Pacemaker project contributors
	+ * Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_UTIL__H
	# define PCMK__CRM_COMMON_UTIL__H

	# include <sys/types.h> // gid_t, mode_t, size_t, time_t, uid_t
	# include <stdlib.h>
	# include <stdbool.h>
	# include <stdint.h> // uint32_t
	# include <limits.h>
	# include <signal.h>
	# include <glib.h>

	# include <libxml/tree.h>

	# include <crm/lrmd.h>
	# include <crm/common/acl.h>
	# include <crm/common/agents.h>
	# include <crm/common/results.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Utility functions
	* \ingroup core
	*/


	# define ONLINESTATUS "online" // Status of an online client
	# define OFFLINESTATUS "offline" // Status of an offline client

	/* public node attribute functions (from attrd_client.c) */
	char pcmk_promotion_score_name(const char rsc_id);

	/* public Pacemaker Remote functions (from remote.c) */
	int crm_default_remote_port(void);

	+/* public score-related functions (from scores.c) */
	+int char2score(const char *score);
	+char *score2char(int score);
	+char score2char_stack(int score, char buf, size_t len);
	+int pcmk__add_scores(int score1, int score2);
	+
	/* public string functions (from strings.c) */
	gboolean crm_is_true(const char *s);
	int crm_str_to_boolean(const char s, int ret);
	long long crm_get_msec(const char *input);
	char * crm_strip_trailing_newline(char *str);
	char crm_strdup_printf(char const format, ...) G_GNUC_PRINTF(1, 2);

	guint crm_parse_interval_spec(const char *input);
	-int char2score(const char *score);
	-char *score2char(int score);
	-char score2char_stack(int score, char buf, size_t len);

	/* public operation functions (from operations.c) */
	gboolean parse_op_key(const char key, char rsc_id, char *op_type,
	guint *interval_ms);
	gboolean decode_transition_key(const char key, char uuid, int transition_id,
	int action_id, int target_rc);
	gboolean decode_transition_magic(const char magic, char *uuid,
	int transition_id, int action_id,
	int op_status, int op_rc, int *target_rc);
	int rsc_op_expected_rc(lrmd_event_data_t *event);
	gboolean did_rsc_op_fail(lrmd_event_data_t *event, int target_rc);
	bool crm_op_needs_metadata(const char rsc_class, const char op);
	xmlNode crm_create_op_xml(xmlNode parent, const char *prefix,
	const char task, const char interval_spec,
	const char *timeout);
	#define CRM_DEFAULT_OP_TIMEOUT_S "20s"

	bool pcmk_is_probe(const char *task, guint interval);
	bool pcmk_xe_is_probe(xmlNode *xml_op);
	bool pcmk_xe_mask_probe_failure(xmlNode *xml_op);

	int compare_version(const char version1, const char version2);

	/* coverity[+kill] */
	void crm_abort(const char file, const char function, int line,
	const char *condition, gboolean do_core, gboolean do_fork);

	/*!
	* \brief Check whether any of specified flags are set in a flag group
	*
	* \param[in] flag_group The flag group being examined
	* \param[in] flags_to_check Which flags in flag_group should be checked
	*
	* \return true if \p flags_to_check is nonzero and any of its flags are set in
	* \p flag_group, or false otherwise
	*/
	static inline bool
	pcmk_any_flags_set(uint64_t flag_group, uint64_t flags_to_check)
	{
	return (flag_group & flags_to_check) != 0;
	}

	/*!
	* \brief Check whether all of specified flags are set in a flag group
	*
	* \param[in] flag_group The flag group being examined
	* \param[in] flags_to_check Which flags in flag_group should be checked
	*
	* \return true if \p flags_to_check is zero or all of its flags are set in
	* \p flag_group, or false otherwise
	*/
	static inline bool
	pcmk_all_flags_set(uint64_t flag_group, uint64_t flags_to_check)
	{
	return (flag_group & flags_to_check) == flags_to_check;
	}

	/*!
	* \brief Convenience alias for pcmk_all_flags_set(), to check single flag
	*/
	#define pcmk_is_set(g, f) pcmk_all_flags_set((g), (f))

	char crm_meta_name(const char field);
	const char crm_meta_value(GHashTable hash, const char *field);

	char crm_md5sum(const char buffer);

	char *crm_generate_uuid(void);

	// This belongs in ipc.h but is here for backward compatibility
	bool crm_is_daemon_name(const char *name);

	int crm_user_lookup(const char name, uid_t uid, gid_t * gid);
	int pcmk_daemon_user(uid_t uid, gid_t gid);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	void crm_gnutls_global_init(void);
	#endif

	char *pcmk_hostname(void);

	bool pcmk_str_is_infinity(const char *s);
	bool pcmk_str_is_minus_infinity(const char *s);

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	#include <crm/common/util_compat.h>
	#endif

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
	index 3ac3d329e5..530815012a 100644
	--- a/include/crm/pengine/internal.h
	+++ b/include/crm/pengine/internal.h
	@@ -1,582 +1,581 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PE_INTERNAL__H
	# define PE_INTERNAL__H
	# include <stdint.h>
	# include <string.h>
	# include <crm/pengine/status.h>
	# include <crm/pengine/remote_internal.h>
	# include <crm/common/internal.h>
	# include <crm/common/output_internal.h>

	# define pe_rsc_info(rsc, fmt, args...) crm_log_tag(LOG_INFO, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_debug(rsc, fmt, args...) crm_log_tag(LOG_DEBUG, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_trace(rsc, fmt, args...) crm_log_tag(LOG_TRACE, rsc ? rsc->id : "<NULL>", fmt, ##args)

	# define pe_err(fmt...) do { \
	was_processing_error = TRUE; \
	pcmk__config_err(fmt); \
	} while (0)

	# define pe_warn(fmt...) do { \
	was_processing_warning = TRUE; \
	pcmk__config_warn(fmt); \
	} while (0)

	# define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
	# define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }

	#define pe__set_working_set_flags(working_set, flags_to_set) do { \
	(working_set)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_working_set_flags(working_set, flags_to_clear) do { \
	(working_set)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_resource_flags(resource, flags_to_set) do { \
	(resource)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_resource_flags(resource, flags_to_clear) do { \
	(resource)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_action_flags(action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags(action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_raw_action_flags(action_flags, action_name, flags_to_set) do { \
	action_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action", action_name, \
	(action_flags), \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_raw_action_flags(action_flags, action_name, flags_to_clear) do { \
	action_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", action_name, \
	(action_flags), \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_action_flags_as(function, line, action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags_as(function, line, action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_order_flags(order_flags, flags_to_set) do { \
	order_flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_order_flags(order_flags, flags_to_clear) do { \
	order_flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_graph_flags(graph_flags, gr_action, flags_to_set) do { \
	graph_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Graph", \
	(gr_action)->uuid, graph_flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_graph_flags(graph_flags, gr_action, flags_to_clear) do { \
	graph_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Graph", \
	(gr_action)->uuid, graph_flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	// Some warnings we don't want to print every transition

	enum pe_warn_once_e {
	pe_wo_blind = (1 << 0),
	pe_wo_restart_type = (1 << 1),
	pe_wo_role_after = (1 << 2),
	pe_wo_poweroff = (1 << 3),
	pe_wo_require_all = (1 << 4),
	pe_wo_order_score = (1 << 5),
	pe_wo_neg_threshold = (1 << 6),
	pe_wo_remove_after = (1 << 7),
	};

	extern uint32_t pe_wo;

	#define pe_warn_once(pe_wo_bit, fmt...) do { \
	if (!pcmk_is_set(pe_wo, pe_wo_bit)) { \
	if (pe_wo_bit == pe_wo_blind) { \
	crm_warn(fmt); \
	} else { \
	pe_warn(fmt); \
	} \
	pe_wo = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Warn-once", "logging", pe_wo, \
	(pe_wo_bit), #pe_wo_bit); \
	} \
	} while (0);


	typedef struct pe__location_constraint_s {
	char *id; // Constraint XML ID
	pe_resource_t *rsc_lh; // Resource being located
	enum rsc_role_e role_filter; // Role to locate
	enum pe_discover_e discover_mode; // Resource discovery
	GList node_list_rh; // List of pe_node_t
	} pe__location_t;

	typedef struct pe__order_constraint_s {
	int id;
	enum pe_ordering type;

	void *lh_opaque;
	pe_resource_t *lh_rsc;
	pe_action_t *lh_action;
	char *lh_action_task;

	void *rh_opaque;
	pe_resource_t *rh_rsc;
	pe_action_t *rh_action;
	char *rh_action_task;
	} pe__ordering_t;

	typedef struct notify_data_s {
	GSList *keys; // Environment variable name/value pairs

	const char *action;

	pe_action_t *pre;
	pe_action_t *post;
	pe_action_t *pre_done;
	pe_action_t *post_done;

	GList active; / notify_entry_t* */
	GList inactive; / notify_entry_t* */
	GList start; / notify_entry_t* */
	GList stop; / notify_entry_t* */
	GList demote; / notify_entry_t* */
	GList promote; / notify_entry_t* */
	GList promoted; / notify_entry_t* */
	GList unpromoted; / notify_entry_t* */
	GHashTable *allowed_nodes;

	} notify_data_t;

	bool pe_can_fence(pe_working_set_t data_set, pe_node_t node);

	-int pe__add_scores(int score1, int score2);
	void add_hash_param(GHashTable * hash, const char name, const char value);

	char native_parameter(pe_resource_t rsc, pe_node_t * node, gboolean create, const char *name,
	pe_working_set_t * data_set);
	pe_node_t native_location(const pe_resource_t rsc, GList **list, int current);

	void pe_metadata(pcmk__output_t *out);
	void verify_pe_options(GHashTable * options);

	void common_update_score(pe_resource_t * rsc, const char *id, int score);
	void native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set, gboolean failed);

	gboolean native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean group_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean pe__unpack_bundle(pe_resource_t rsc, pe_working_set_t data_set);

	pe_resource_t native_find_rsc(pe_resource_t rsc, const char id, const pe_node_t node,
	int flags);

	gboolean native_active(pe_resource_t * rsc, gboolean all);
	gboolean group_active(pe_resource_t * rsc, gboolean all);
	gboolean clone_active(pe_resource_t * rsc, gboolean all);
	gboolean pe__bundle_active(pe_resource_t *rsc, gboolean all);

	void native_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void group_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void clone_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void pe__print_bundle(pe_resource_t rsc, const char pre_text, long options,
	void *print_data);

	gchar * pcmk__native_output_string(pe_resource_t rsc, const char name, pe_node_t *node,
	uint32_t show_opts, const char *target_role, bool show_nodes);

	int pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...);
	char pe__node_display_name(pe_node_t node, bool print_detail);

	static inline const char *
	pe__rsc_bool_str(pe_resource_t *rsc, uint64_t rsc_flag)
	{
	return pcmk__btoa(pcmk_is_set(rsc->flags, rsc_flag));
	}

	int pe__clone_xml(pcmk__output_t *out, va_list args);
	int pe__clone_default(pcmk__output_t *out, va_list args);
	int pe__group_xml(pcmk__output_t *out, va_list args);
	int pe__group_default(pcmk__output_t *out, va_list args);
	int pe__bundle_xml(pcmk__output_t *out, va_list args);
	int pe__bundle_html(pcmk__output_t *out, va_list args);
	int pe__bundle_text(pcmk__output_t *out, va_list args);
	int pe__node_html(pcmk__output_t *out, va_list args);
	int pe__node_text(pcmk__output_t *out, va_list args);
	int pe__node_xml(pcmk__output_t *out, va_list args);
	int pe__resource_xml(pcmk__output_t *out, va_list args);
	int pe__resource_html(pcmk__output_t *out, va_list args);
	int pe__resource_text(pcmk__output_t *out, va_list args);

	void native_free(pe_resource_t * rsc);
	void group_free(pe_resource_t * rsc);
	void clone_free(pe_resource_t * rsc);
	void pe__free_bundle(pe_resource_t *rsc);

	enum rsc_role_e native_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e group_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e clone_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e pe__bundle_resource_state(const pe_resource_t *rsc,
	gboolean current);

	void pe__count_common(pe_resource_t *rsc);
	void pe__count_bundle(pe_resource_t *rsc);

	gboolean common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc, pe_resource_t * parent,
	pe_working_set_t * data_set);
	void common_free(pe_resource_t * rsc);

	pe_node_t pe__copy_node(const pe_node_t this_node);
	extern time_t get_effective_time(pe_working_set_t * data_set);

	/* Failure handling utilities (from failcounts.c) */

	// bit flags for fail count handling options
	enum pe_fc_flags_e {
	pe_fc_default = (1 << 0),
	pe_fc_effective = (1 << 1), // don't count expired failures
	pe_fc_fillers = (1 << 2), // if container, include filler failures in count
	};

	int pe_get_failcount(pe_node_t node, pe_resource_t rsc, time_t *last_failure,
	uint32_t flags, xmlNode *xml_op,
	pe_working_set_t *data_set);

	pe_action_t pe__clear_failcount(pe_resource_t rsc, pe_node_t *node,
	const char *reason,
	pe_working_set_t *data_set);

	/* Functions for finding/counting a resource's active nodes */

	pe_node_t pe__find_active_on(const pe_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);
	pe_node_t pe__find_active_requires(const pe_resource_t rsc,
	unsigned int *count);

	static inline pe_node_t *
	pe__current_node(const pe_resource_t *rsc)
	{
	return pe__find_active_on(rsc, NULL, NULL);
	}


	/* Binary like operators for lists of nodes */
	extern void node_list_exclude(GHashTable * list, GList *list2, gboolean merge_scores);

	GHashTable pe__node_list2table(GList list);

	static inline gpointer
	pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
	{
	if (hash) {
	return g_hash_table_lookup(hash, key);
	}
	return NULL;
	}

	extern pe_action_t get_pseudo_op(const char name, pe_working_set_t * data_set);
	extern gboolean order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order);

	/* Printing functions for debug */
	extern void print_str_str(gpointer key, gpointer value, gpointer user_data);
	extern void pe__output_node(pe_node_t * node, gboolean details, pcmk__output_t *out);

	void pe__show_node_weights_as(const char file, const char function,
	int line, bool to_log, pe_resource_t *rsc,
	const char comment, GHashTable nodes,
	pe_working_set_t *data_set);

	#define pe__show_node_weights(level, rsc, text, nodes, data_set) \
	pe__show_node_weights_as(__FILE__, __func__, __LINE__, \
	(level), (rsc), (text), (nodes), (data_set))

	/* Sorting functions */
	extern gint sort_rsc_priority(gconstpointer a, gconstpointer b);
	extern gint sort_rsc_index(gconstpointer a, gconstpointer b);

	extern xmlNode find_rsc_op_entry(pe_resource_t rsc, const char *key);

	extern pe_action_t custom_action(pe_resource_t rsc, char key, const char task, pe_node_t * on_node,
	gboolean optional, gboolean foo, pe_working_set_t * data_set);

	# define delete_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DELETE, 0)
	# define delete_action(rsc, node, optional) custom_action( \
	rsc, delete_key(rsc), CRMD_ACTION_DELETE, node, \
	optional, TRUE, data_set);

	# define stopped_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
	# define stopped_action(rsc, node, optional) custom_action( \
	rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node, \
	optional, TRUE, data_set);

	# define stop_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOP, 0)
	# define stop_action(rsc, node, optional) custom_action( \
	rsc, stop_key(rsc), CRMD_ACTION_STOP, node, \
	optional, TRUE, data_set);

	# define reload_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_RELOAD_AGENT, 0)
	# define start_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_START, 0)
	# define start_action(rsc, node, optional) custom_action( \
	rsc, start_key(rsc), CRMD_ACTION_START, node, \
	optional, TRUE, data_set)

	# define started_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STARTED, 0)
	# define started_action(rsc, node, optional) custom_action( \
	rsc, started_key(rsc), CRMD_ACTION_STARTED, node, \
	optional, TRUE, data_set)

	# define promote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
	# define promote_action(rsc, node, optional) custom_action( \
	rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node, \
	optional, TRUE, data_set)

	# define promoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
	# define promoted_action(rsc, node, optional) custom_action( \
	rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node, \
	optional, TRUE, data_set)

	# define demote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
	# define demote_action(rsc, node, optional) custom_action( \
	rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node, \
	optional, TRUE, data_set)

	# define demoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
	# define demoted_action(rsc, node, optional) custom_action( \
	rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node, \
	optional, TRUE, data_set)

	extern int pe_get_configured_timeout(pe_resource_t rsc, const char action,
	pe_working_set_t *data_set);

	extern pe_action_t find_first_action(GList input, const char uuid, const char task,
	pe_node_t * on_node);
	extern enum action_tasks get_complex_task(pe_resource_t * rsc, const char *name,
	gboolean allow_non_atomic);

	extern GList find_actions(GList input, const char key, const pe_node_t on_node);
	GList find_actions_exact(GList input, const char *key,
	const pe_node_t *on_node);
	extern GList find_recurring_actions(GList input, pe_node_t * not_on_node);
	GList pe__resource_actions(const pe_resource_t rsc, const pe_node_t *node,
	const char *task, bool require_node);

	extern void pe_free_action(pe_action_t * action);

	extern void resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
	pe_working_set_t * data_set);

	extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
	extern gboolean get_target_role(pe_resource_t * rsc, enum rsc_role_e *role);
	void pe__set_next_role(pe_resource_t *rsc, enum rsc_role_e role,
	const char *why);

	extern pe_resource_t find_clone_instance(pe_resource_t rsc, const char *sub_id,
	pe_working_set_t * data_set);

	extern void destroy_ticket(gpointer data);
	extern pe_ticket_t ticket_new(const char ticket_id, pe_working_set_t * data_set);

	// Resources for manipulating resource names
	const char pe_base_name_end(const char id);
	char clone_strip(const char last_rsc_id);
	char clone_zero(const char last_rsc_id);

	static inline bool
	pe_base_name_eq(pe_resource_t rsc, const char id)
	{
	if (id && rsc && rsc->id) {
	// Number of characters in rsc->id before any clone suffix
	size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;

	return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
	}
	return FALSE;
	}

	int pe__target_rc_from_xml(xmlNode *xml_op);

	gint sort_node_uname(gconstpointer a, gconstpointer b);
	bool is_set_recursive(pe_resource_t * rsc, long long flag, bool any);

	enum rsc_digest_cmp_val {
	/! Digests are the same /
	RSC_DIGEST_MATCH = 0,
	/! Params that require a restart changed /
	RSC_DIGEST_RESTART,
	/! Some parameter changed. /
	RSC_DIGEST_ALL,
	/*! rsc op didn't have a digest associated with it, so
	* it is unknown if parameters changed or not. */
	RSC_DIGEST_UNKNOWN,
	};

	typedef struct op_digest_cache_s {
	enum rsc_digest_cmp_val rc;
	xmlNode *params_all;
	xmlNode *params_secure;
	xmlNode *params_restart;
	char *digest_all_calc;
	char *digest_secure_calc;
	char *digest_restart_calc;
	} op_digest_cache_t;

	op_digest_cache_t pe__calculate_digests(pe_resource_t rsc, const char *task,
	guint interval_ms, pe_node_t node,
	xmlNode xml_op, GHashTable overrides,
	bool calc_secure,
	pe_working_set_t *data_set);

	void pe__free_digests(gpointer ptr);

	op_digest_cache_t rsc_action_digest_cmp(pe_resource_t rsc, xmlNode * xml_op, pe_node_t * node,
	pe_working_set_t * data_set);

	pe_action_t pe_fence_op(pe_node_t node, const char op, bool optional, const char reason, bool priority_delay, pe_working_set_t * data_set);
	void trigger_unfencing(
	pe_resource_t * rsc, pe_node_t node, const char reason, pe_action_t dependency, pe_working_set_t data_set);

	char pe__action2reason(pe_action_t action, enum pe_action_flags flag);
	void pe_action_set_reason(pe_action_t action, const char reason, bool overwrite);

	void pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_on_all(pe_working_set_t *data_set, uint64_t flag);

	gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref);

	void print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void * print_data, gboolean print_all);
	int pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, unsigned int options);
	void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay);

	pe_node_t pe_create_node(const char id, const char uname, const char type,
	const char score, pe_working_set_t data_set);
	void common_print(pe_resource_t * rsc, const char pre_text, const char name, pe_node_t node, long options, void print_data);
	int pe__common_output_text(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, unsigned int options);
	int pe__common_output_html(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, unsigned int options);
	pe_resource_t pe__find_bundle_replica(const pe_resource_t bundle,
	const pe_node_t *node);
	bool pe__bundle_needs_remote_name(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	const char pe__add_bundle_remote_name(pe_resource_t rsc,
	pe_working_set_t *data_set,
	xmlNode xml, const char field);
	const char pe_node_attribute_calculated(const pe_node_t node,
	const char *name,
	const pe_resource_t *rsc);
	const char pe_node_attribute_raw(pe_node_t node, const char *name);
	bool pe__is_universal_clone(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	void pe__add_param_check(xmlNode rsc_op, pe_resource_t rsc, pe_node_t *node,
	enum pe_check_parameters, pe_working_set_t *data_set);
	void pe__foreach_param_check(pe_working_set_t *data_set,
	void (cb)(pe_resource_t, pe_node_t, xmlNode,
	enum pe_check_parameters,
	pe_working_set_t*));
	void pe__free_param_checks(pe_working_set_t *data_set);

	bool pe__shutdown_requested(pe_node_t *node);
	void pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set);

	/*!
	* \internal
	* \brief Register xml formatting message functions.
	*/
	void pe__register_messages(pcmk__output_t *out);

	void pe__unpack_dataset_nvpairs(xmlNode xml_obj, const char set_name,
	pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	pe_working_set_t *data_set);

	bool pe__resource_is_disabled(pe_resource_t *rsc);
	pe_action_t pe__clear_resource_history(pe_resource_t rsc, pe_node_t *node,
	pe_working_set_t *data_set);

	GList pe__rscs_with_tag(pe_working_set_t data_set, const char *tag_name);
	GList pe__unames_with_tag(pe_working_set_t data_set, const char *tag_name);
	bool pe__rsc_has_tag(pe_working_set_t data_set, const char rsc, const char *tag);
	bool pe__uname_has_tag(pe_working_set_t data_set, const char node, const char *tag);

	bool pe__rsc_running_on_any(pe_resource_t rsc, GList node_list);
	GList pe__filter_rsc_list(GList rscs, GList *filter);
	GList * pe__build_node_name_list(pe_working_set_t data_set, const char s);
	GList * pe__build_rsc_list(pe_working_set_t data_set, const char s);

	bool pcmk__rsc_filtered_by_node(pe_resource_t rsc, GList only_node);

	gboolean pe__bundle_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__clone_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__group_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__native_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);

	xmlNode pe__failed_probe_for_rsc(pe_resource_t rsc, const char *name);

	const char pe__clone_child_id(pe_resource_t rsc);

	#endif
	diff --git a/include/pcmki/pcmki_sched_utils.h b/include/pcmki/pcmki_sched_utils.h
	index 1ce1fe8ff5..76fd10f891 100644
	--- a/include/pcmki/pcmki_sched_utils.h
	+++ b/include/pcmki/pcmki_sched_utils.h
	@@ -1,54 +1,48 @@
	/*
	- * Copyright 2004-2021 the Pacemaker project contributors
	+ * Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__PCMKI_PCMKI_SCHED_UTILS__H
	# define PCMK__PCMKI_PCMKI_SCHED_UTILS__H

	#include <stdbool.h> // bool
	#include <glib.h> // GList, GHashTable, gboolean, guint
	#include <crm/lrmd.h> // lrmd_event_data_t
	#include <crm/cib.h> // cib_t
	#include <crm/pengine/pe_types.h>
	#include <crm/common/xml_internal.h>
	#include <crm/pengine/internal.h>
	#include <pcmki/pcmki_scheduler.h>
	#include <pcmki/pcmki_transition.h>
	#include <pacemaker.h>

	/* Constraint helper functions */
	pcmk__colocation_t invert_constraint(pcmk__colocation_t constraint);

	pe__location_t copy_constraint(pe__location_t constraint);

	-GHashTable pcmk__copy_node_table(GHashTable nodes);
	GList pcmk__copy_node_list(const GList list, bool reset);
	-GList pcmk__sort_nodes(GList nodes, pe_node_t *active_node,
	- pe_working_set_t *data_set);
	-
	-bool pcmk__node_available(const pe_node_t *node);
	-bool pcmk__any_node_available(GHashTable *nodes);

	pe_resource_t find_compatible_child(pe_resource_t local_child,
	pe_resource_t *rsc, enum rsc_role_e filter,
	gboolean current,
	pe_working_set_t *data_set);
	pe_resource_t find_compatible_child_by_node(pe_resource_t local_child, pe_node_t * local_node, pe_resource_t * rsc,
	enum rsc_role_e filter, gboolean current);
	gboolean is_child_compatible(pe_resource_t child_rsc, pe_node_t local_node, enum rsc_role_e filter, gboolean current);
	enum pe_action_flags summary_action_flags(pe_action_t * action, GList children, pe_node_t node);
	enum action_tasks clone_child_action(pe_action_t * action);
	int copies_per_node(pe_resource_t * rsc);

	xmlNode pcmk__create_history_xml(xmlNode parent, lrmd_event_data_t *event,
	const char *caller_version, int target_rc,
	const char node, const char origin);

	# define LOAD_STOPPED "load_stopped"

	#endif
	diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am
	index 3014393518..b052fa4562 100644
	--- a/lib/common/Makefile.am
	+++ b/lib/common/Makefile.am
	@@ -1,116 +1,117 @@
	#
	# Copyright 2004-2022 the Pacemaker project contributors
	#
	# The version control history for this file may have further details.
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#
	include $(top_srcdir)/mk/common.mk

	AM_CPPFLAGS += -I$(top_builddir)/lib/gnu -I$(top_srcdir)/lib/gnu

	MOSTLYCLEANFILES = md5.c

	## libraries
	lib_LTLIBRARIES = libcrmcommon.la
	check_LTLIBRARIES = libcrmcommon_test.la

	# Disable -Wcast-qual if used, because we do some hacky casting,
	# and because libxml2 has some signatures that should be const but aren't
	# for backward compatibility reasons.

	# s390 needs -fPIC
	# s390-suse-linux/bin/ld: .libs/ipc.o: relocation R_390_PC32DBL against `__stack_chk_fail@@GLIBC_2.4' can not be used when making a shared object; recompile with -fPIC

	CFLAGS = $(CFLAGS_COPY:-Wcast-qual=) -fPIC

	# Without "." here, check-recursive will run through the subdirectories first
	# and then run "make check" here. This will fail, because there's things in
	# the subdirectories that need check_LTLIBRARIES built first. Adding "." here
	# changes the order so the subdirectories are processed afterwards.
	SUBDIRS = . tests

	noinst_HEADERS = crmcommon_private.h mock_private.h

	libcrmcommon_la_LDFLAGS = -version-info 42:0:8

	libcrmcommon_la_CFLAGS = $(CFLAGS_HARDENED_LIB)
	libcrmcommon_la_LDFLAGS += $(LDFLAGS_HARDENED_LIB)

	libcrmcommon_la_LIBADD = @LIBADD_DL@

	# If configured with --with-profiling or --with-coverage, BUILD_PROFILING will
	# be set and -fno-builtin will be added to the CFLAGS. However, libcrmcommon
	# uses the fabs() function which is normally supplied by gcc as one of its
	# builtins. Therefore we need to explicitly link against libm here or the
	# tests won't link.
	if BUILD_PROFILING
	libcrmcommon_la_LIBADD += -lm
	endif

	# Use += rather than backlashed continuation lines for parsing by bumplibs
	libcrmcommon_la_SOURCES =
	libcrmcommon_la_SOURCES += acl.c
	libcrmcommon_la_SOURCES += agents.c
	libcrmcommon_la_SOURCES += alerts.c
	libcrmcommon_la_SOURCES += attrd_client.c
	libcrmcommon_la_SOURCES += cib.c
	if BUILD_CIBSECRETS
	libcrmcommon_la_SOURCES += cib_secrets.c
	endif
	libcrmcommon_la_SOURCES += cmdline.c
	libcrmcommon_la_SOURCES += digest.c
	libcrmcommon_la_SOURCES += io.c
	libcrmcommon_la_SOURCES += ipc_client.c
	libcrmcommon_la_SOURCES += ipc_common.c
	libcrmcommon_la_SOURCES += ipc_controld.c
	libcrmcommon_la_SOURCES += ipc_pacemakerd.c
	libcrmcommon_la_SOURCES += ipc_schedulerd.c
	libcrmcommon_la_SOURCES += ipc_server.c
	libcrmcommon_la_SOURCES += iso8601.c
	libcrmcommon_la_SOURCES += lists.c
	libcrmcommon_la_SOURCES += logging.c
	libcrmcommon_la_SOURCES += mainloop.c
	libcrmcommon_la_SOURCES += messages.c
	libcrmcommon_la_SOURCES += nvpair.c
	libcrmcommon_la_SOURCES += operations.c
	libcrmcommon_la_SOURCES += options.c
	libcrmcommon_la_SOURCES += output.c
	libcrmcommon_la_SOURCES += output_html.c
	libcrmcommon_la_SOURCES += output_log.c
	libcrmcommon_la_SOURCES += output_none.c
	libcrmcommon_la_SOURCES += output_text.c
	libcrmcommon_la_SOURCES += output_xml.c
	libcrmcommon_la_SOURCES += patchset.c
	libcrmcommon_la_SOURCES += pid.c
	libcrmcommon_la_SOURCES += procfs.c
	libcrmcommon_la_SOURCES += remote.c
	libcrmcommon_la_SOURCES += results.c
	libcrmcommon_la_SOURCES += schemas.c
	+libcrmcommon_la_SOURCES += scores.c
	libcrmcommon_la_SOURCES += strings.c
	libcrmcommon_la_SOURCES += utils.c
	libcrmcommon_la_SOURCES += watchdog.c
	libcrmcommon_la_SOURCES += xml.c
	libcrmcommon_la_SOURCES += xpath.c

	# It's possible to build the library adding ../gnu/md5.c directly to SOURCES,
	# but distclean chokes on that because it tries to include the source's .Plo
	# file, which may have already been cleaned.
	nodist_libcrmcommon_la_SOURCES = md5.c

	WRAPPED = calloc getenv getpwnam_r uname
	WRAPPED_FLAGS = $(foreach fn,$(WRAPPED),-Wl,--wrap=$(fn))

	libcrmcommon_test_la_SOURCES = $(libcrmcommon_la_SOURCES)
	libcrmcommon_test_la_SOURCES += mock.c
	libcrmcommon_test_la_LDFLAGS = $(LDFLAGS_HARDENED_LIB) $(WRAPPED_FLAGS)
	libcrmcommon_test_la_CFLAGS = $(libcrmcommon_la_CFLAGS)
	libcrmcommon_test_la_LIBADD = $(libcrmcommon_la_LIBADD)
	nodist_libcrmcommon_test_la_SOURCES = $(nodist_libcrmcommon_la_SOURCES)

	md5.c: ../gnu/md5.c
	cp "$<" "$@"

	clean-generic:
	rm -f .log .debug .xml ~
	diff --git a/lib/common/scores.c b/lib/common/scores.c
	new file mode 100644
	index 0000000000..18e53c4c62
	--- /dev/null
	+++ b/lib/common/scores.c
	@@ -0,0 +1,200 @@
	+/*
	+ * Copyright 2004-2022 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU Lesser General Public License
	+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#include <crm_internal.h>
	+
	+#ifndef _GNU_SOURCE
	+# define _GNU_SOURCE
	+#endif
	+
	+#include <stdio.h> // snprintf(), NULL
	+#include <string.h> // strcpy(), strdup()
	+#include <sys/types.h> // size_t
	+
	+int pcmk__score_red = 0;
	+int pcmk__score_green = 0;
	+int pcmk__score_yellow = 0;
	+
	+/*!
	+ * \brief Get the integer value of a score string
	+ *
	+ * Given a string representation of a score, return the integer equivalent.
	+ * This accepts infinity strings as well as red, yellow, and green, and
	+ * bounds the result to +/-INFINITY.
	+ *
	+ * \param[in] score Score as string
	+ *
	+ * \return Integer value corresponding to \p score
	+ */
	+int
	+char2score(const char *score)
	+{
	+ if (score == NULL) {
	+ return 0;
	+
	+ } else if (pcmk_str_is_minus_infinity(score)) {
	+ return -CRM_SCORE_INFINITY;
	+
	+ } else if (pcmk_str_is_infinity(score)) {
	+ return CRM_SCORE_INFINITY;
	+
	+ } else if (pcmk__str_eq(score, "red", pcmk__str_casei)) {
	+ return pcmk__score_red;
	+
	+ } else if (pcmk__str_eq(score, "yellow", pcmk__str_casei)) {
	+ return pcmk__score_yellow;
	+
	+ } else if (pcmk__str_eq(score, "green", pcmk__str_casei)) {
	+ return pcmk__score_green;
	+
	+ } else {
	+ long long score_ll;
	+
	+ pcmk__scan_ll(score, &score_ll, 0LL);
	+ if (score_ll > CRM_SCORE_INFINITY) {
	+ return CRM_SCORE_INFINITY;
	+
	+ } else if (score_ll < -CRM_SCORE_INFINITY) {
	+ return -CRM_SCORE_INFINITY;
	+
	+ } else {
	+ return (int) score_ll;
	+ }
	+ }
	+}
	+
	+/*!
	+ * \brief Convert an integer score to a string, using a provided buffer
	+ *
	+ * Store the string equivalent of a given integer score in a given string
	+ * buffer, using "INFINITY" and "-INFINITY" when appropriate.
	+ *
	+ * \param[in] score Integer score to convert
	+ * \param[out] buf Where to store string representation of \p score
	+ * \param[in] len Size of \p buf (in bytes)
	+ *
	+ * \return \p buf (or NULL if \p len is too small)
	+ */
	+char *
	+score2char_stack(int score, char *buf, size_t len)
	+{
	+ CRM_CHECK((buf != NULL) && (len >= sizeof(CRM_MINUS_INFINITY_S)),
	+ return NULL);
	+
	+ if (score >= CRM_SCORE_INFINITY) {
	+ strcpy(buf, CRM_INFINITY_S);
	+ } else if (score <= -CRM_SCORE_INFINITY) {
	+ strcpy(buf, CRM_MINUS_INFINITY_S);
	+ } else {
	+ snprintf(buf, len, "%d", score);
	+ }
	+ return buf;
	+}
	+
	+/*!
	+ * \brief Return the string equivalent of an integer score
	+ *
	+ * Return the string equivalent of a given integer score, using "INFINITY" and
	+ * "-INFINITY" when appropriate.
	+ *
	+ * \param[in] score Integer score to convert
	+ *
	+ * \return Newly allocated string equivalent of \p score
	+ * \note The caller is responsible for freeing the return value. This function
	+ * asserts on memory errors, so the return value can be assumed to be
	+ * non-NULL.
	+ */
	+char *
	+score2char(int score)
	+{
	+ char *result = NULL;
	+
	+ if (score >= CRM_SCORE_INFINITY) {
	+ result = strdup(CRM_INFINITY_S);
	+ CRM_ASSERT(result != NULL);
	+
	+ } else if (score <= -CRM_SCORE_INFINITY) {
	+ result = strdup(CRM_MINUS_INFINITY_S);
	+ CRM_ASSERT(result != NULL);
	+
	+ } else {
	+ result = pcmk__itoa(score);
	+ }
	+ return result;
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Add two scores, bounding to +/-INFINITY
	+ *
	+ * \param[in] score1 First score to add
	+ * \param[in] score2 Second score to add
	+ */
	+int
	+pcmk__add_scores(int score1, int score2)
	+{
	+ int result = score1 + score2;
	+
	+ // First handle the cases where one or both is infinite
	+
	+ if (score1 <= -CRM_SCORE_INFINITY) {
	+
	+ if (score2 <= -CRM_SCORE_INFINITY) {
	+ crm_trace("-INFINITY + -INFINITY = -INFINITY");
	+ } else if (score2 >= CRM_SCORE_INFINITY) {
	+ crm_trace("-INFINITY + +INFINITY = -INFINITY");
	+ } else {
	+ crm_trace("-INFINITY + %d = -INFINITY", score2);
	+ }
	+
	+ return -CRM_SCORE_INFINITY;
	+
	+ } else if (score2 <= -CRM_SCORE_INFINITY) {
	+
	+ if (score1 >= CRM_SCORE_INFINITY) {
	+ crm_trace("+INFINITY + -INFINITY = -INFINITY");
	+ } else {
	+ crm_trace("%d + -INFINITY = -INFINITY", score1);
	+ }
	+
	+ return -CRM_SCORE_INFINITY;
	+
	+ } else if (score1 >= CRM_SCORE_INFINITY) {
	+
	+ if (score2 >= CRM_SCORE_INFINITY) {
	+ crm_trace("+INFINITY + +INFINITY = +INFINITY");
	+ } else {
	+ crm_trace("+INFINITY + %d = +INFINITY", score2);
	+ }
	+
	+ return CRM_SCORE_INFINITY;
	+
	+ } else if (score2 >= CRM_SCORE_INFINITY) {
	+ crm_trace("%d + +INFINITY = +INFINITY", score1);
	+ return CRM_SCORE_INFINITY;
	+ }
	+
	+ /* As long as CRM_SCORE_INFINITY is less than half of the maximum integer,
	+ * we can ignore the possibility of integer overflow
	+ */
	+
	+ // Bound result to infinity
	+
	+ if (result >= CRM_SCORE_INFINITY) {
	+ crm_trace("%d + %d = +INFINITY", score1, score2);
	+ return CRM_SCORE_INFINITY;
	+
	+ } else if (result <= -CRM_SCORE_INFINITY) {
	+ crm_trace("%d + %d = -INFINITY", score1, score2);
	+ return -CRM_SCORE_INFINITY;
	+ }
	+
	+ crm_trace("%d + %d = %d", score1, score2, result);
	+ return result;
	+}
	diff --git a/lib/common/tests/Makefile.am b/lib/common/tests/Makefile.am
	index e205825bcb..a9d622737c 100644
	--- a/lib/common/tests/Makefile.am
	+++ b/lib/common/tests/Makefile.am
	@@ -1,23 +1,24 @@
	#
	# Copyright 2020-2022 the Pacemaker project contributors
	#
	# The version control history for this file may have further details.
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#

	SUBDIRS = \
	- acl \
	+ acl \
	agents \
	cmdline \
	flags \
	- io \
	+ io \
	iso8601 \
	lists \
	nvpair \
	operations \
	results \
	+ scores \
	strings \
	utils \
	xpath
	diff --git a/lib/common/tests/scores/Makefile.am b/lib/common/tests/scores/Makefile.am
	new file mode 100644
	index 0000000000..395415ffee
	--- /dev/null
	+++ b/lib/common/tests/scores/Makefile.am
	@@ -0,0 +1,22 @@
	+#
	+# Copyright 2020-2022 the Pacemaker project contributors
	+#
	+# The version control history for this file may have further details.
	+#
	+# This source code is licensed under the GNU General Public License version 2
	+# or later (GPLv2+) WITHOUT ANY WARRANTY.
	+#
	+
	+AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/lib/common
	+LDADD = $(top_builddir)/lib/common/libcrmcommon.la -lcmocka
	+
	+include $(top_srcdir)/mk/tap.mk
	+
	+# Add "_test" to the end of all test program names to simplify .gitignore.
	+check_PROGRAMS = \
	+ char2score_test \
	+ pcmk__add_scores_test \
	+ score2char_stack_test \
	+ score2char_test
	+
	+TESTS = $(check_PROGRAMS)
	diff --git a/lib/common/tests/utils/char2score_test.c b/lib/common/tests/scores/char2score_test.c
	similarity index 100%
	rename from lib/common/tests/utils/char2score_test.c
	rename to lib/common/tests/scores/char2score_test.c
	diff --git a/lib/common/tests/scores/pcmk__add_scores_test.c b/lib/common/tests/scores/pcmk__add_scores_test.c
	new file mode 100644
	index 0000000000..bf7f0cf956
	--- /dev/null
	+++ b/lib/common/tests/scores/pcmk__add_scores_test.c
	@@ -0,0 +1,85 @@
	+/*
	+ * Copyright 2022 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU Lesser General Public License
	+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#include <crm_internal.h>
	+
	+#include <stdarg.h>
	+#include <stddef.h>
	+#include <stdint.h>
	+#include <setjmp.h>
	+#include <cmocka.h>
	+
	+static void
	+score1_minus_inf(void **state)
	+{
	+ assert_int_equal(pcmk__add_scores(-CRM_SCORE_INFINITY, -CRM_SCORE_INFINITY), -CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(-CRM_SCORE_INFINITY, -1), -CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(-CRM_SCORE_INFINITY, 0), -CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(-CRM_SCORE_INFINITY, 1), -CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(-CRM_SCORE_INFINITY, CRM_SCORE_INFINITY), -CRM_SCORE_INFINITY);
	+}
	+
	+static void
	+score2_minus_inf(void **state)
	+{
	+ assert_int_equal(pcmk__add_scores(-1, -CRM_SCORE_INFINITY), -CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(0, -CRM_SCORE_INFINITY), -CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(1, -CRM_SCORE_INFINITY), -CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(CRM_SCORE_INFINITY, -CRM_SCORE_INFINITY), -CRM_SCORE_INFINITY);
	+}
	+
	+static void
	+score1_pos_inf(void **state)
	+{
	+ assert_int_equal(pcmk__add_scores(CRM_SCORE_INFINITY, CRM_SCORE_INFINITY), CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(CRM_SCORE_INFINITY, -1), CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(CRM_SCORE_INFINITY, 0), CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(CRM_SCORE_INFINITY, 1), CRM_SCORE_INFINITY);
	+}
	+
	+static void
	+score2_pos_inf(void **state)
	+{
	+ assert_int_equal(pcmk__add_scores(-1, CRM_SCORE_INFINITY), CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(0, CRM_SCORE_INFINITY), CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(1, CRM_SCORE_INFINITY), CRM_SCORE_INFINITY);
	+}
	+
	+static void
	+result_infinite(void **state)
	+{
	+ assert_int_equal(pcmk__add_scores(INT_MAX, INT_MAX), CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(INT_MIN, INT_MIN), -CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(2000000, 50), CRM_SCORE_INFINITY);
	+ assert_int_equal(pcmk__add_scores(-4000000, 50), -CRM_SCORE_INFINITY);
	+}
	+
	+static void
	+result_finite(void **state)
	+{
	+ assert_int_equal(pcmk__add_scores(0, 0), 0);
	+ assert_int_equal(pcmk__add_scores(0, 100), 100);
	+ assert_int_equal(pcmk__add_scores(200, 0), 200);
	+ assert_int_equal(pcmk__add_scores(200, -50), 150);
	+}
	+
	+int main(int argc, char **argv)
	+{
	+ const struct CMUnitTest tests[] = {
	+ cmocka_unit_test(score1_minus_inf),
	+ cmocka_unit_test(score2_minus_inf),
	+ cmocka_unit_test(score1_pos_inf),
	+ cmocka_unit_test(score2_pos_inf),
	+ cmocka_unit_test(result_infinite),
	+ cmocka_unit_test(result_finite),
	+ };
	+
	+ cmocka_set_message_output(CM_OUTPUT_TAP);
	+ return cmocka_run_group_tests(tests, NULL, NULL);
	+}
	diff --git a/lib/common/tests/utils/score2char_stack_test.c b/lib/common/tests/scores/score2char_stack_test.c
	similarity index 100%
	rename from lib/common/tests/utils/score2char_stack_test.c
	rename to lib/common/tests/scores/score2char_stack_test.c
	diff --git a/lib/common/tests/utils/score2char_test.c b/lib/common/tests/scores/score2char_test.c
	similarity index 100%
	rename from lib/common/tests/utils/score2char_test.c
	rename to lib/common/tests/scores/score2char_test.c
	diff --git a/lib/common/tests/utils/Makefile.am b/lib/common/tests/utils/Makefile.am
	index fa927490f6..54cea48ba0 100644
	--- a/lib/common/tests/utils/Makefile.am
	+++ b/lib/common/tests/utils/Makefile.am
	@@ -1,41 +1,38 @@
	#
	# Copyright 2020-2022 the Pacemaker project contributors
	#
	# The version control history for this file may have further details.
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#

	AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/lib/common
	LDADD = $(top_builddir)/lib/common/libcrmcommon.la -lcmocka

	include $(top_srcdir)/mk/tap.mk

	crm_user_lookup_test_LDADD = $(top_builddir)/lib/common/libcrmcommon_test.la -lcmocka
	crm_user_lookup_test_LDFLAGS = -Wl,--wrap=calloc -Wl,--wrap=getpwnam_r

	pcmk_daemon_user_test_LDADD = $(top_builddir)/lib/common/libcrmcommon_test.la -lcmocka
	pcmk_daemon_user_test_LDFLAGS = -Wl,--wrap=getpwnam_r

	pcmk_hostname_test_LDADD = $(top_builddir)/lib/common/libcrmcommon_test.la -lcmocka
	pcmk_hostname_test_LDFLAGS = -Wl,--wrap=uname

	# Add "_test" to the end of all test program names to simplify .gitignore.
	-check_PROGRAMS = \
	- char2score_test \
	- compare_version_test \
	- crm_meta_name_test \
	- crm_meta_value_test \
	- crm_user_lookup_test \
	- pcmk_daemon_user_test \
	+check_PROGRAMS = \
	+ compare_version_test \
	+ crm_meta_name_test \
	+ crm_meta_value_test \
	+ crm_user_lookup_test \
	+ pcmk_daemon_user_test \
	pcmk_str_is_infinity_test \
	- pcmk_str_is_minus_infinity_test \
	- score2char_stack_test \
	- score2char_test
	+ pcmk_str_is_minus_infinity_test

	if WRAPPABLE_UNAME
	check_PROGRAMS += pcmk_hostname_test
	endif

	TESTS = $(check_PROGRAMS)
	diff --git a/lib/common/utils.c b/lib/common/utils.c
	index 2a1d389ab6..2dfbef2789 100644
	--- a/lib/common/utils.c
	+++ b/lib/common/utils.c
	@@ -1,636 +1,564 @@
	/*
	- * Copyright 2004-2021 the Pacemaker project contributors
	+ * Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <sys/utsname.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <limits.h>
	#include <pwd.h>
	#include <time.h>
	#include <libgen.h>
	#include <signal.h>

	#include <qb/qbdefs.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/cib/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/common/ipc.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/mainloop.h>
	#include <libxml2/libxml/relaxng.h>

	#include "crmcommon_private.h"

	#ifndef PW_BUFFER_LEN
	# define PW_BUFFER_LEN 500
	#endif

	CRM_TRACE_INIT_DATA(common);

	gboolean crm_config_error = FALSE;
	gboolean crm_config_warning = FALSE;
	char *crm_system_name = NULL;

	-int pcmk__score_red = 0;
	-int pcmk__score_green = 0;
	-int pcmk__score_yellow = 0;
	-
	-int
	-char2score(const char *score)
	-{
	- int score_f = 0;
	-
	- if (score == NULL) {
	-
	- } else if (pcmk_str_is_minus_infinity(score)) {
	- score_f = -CRM_SCORE_INFINITY;
	-
	- } else if (pcmk_str_is_infinity(score)) {
	- score_f = CRM_SCORE_INFINITY;
	-
	- } else if (pcmk__str_eq(score, "red", pcmk__str_casei)) {
	- score_f = pcmk__score_red;
	-
	- } else if (pcmk__str_eq(score, "yellow", pcmk__str_casei)) {
	- score_f = pcmk__score_yellow;
	-
	- } else if (pcmk__str_eq(score, "green", pcmk__str_casei)) {
	- score_f = pcmk__score_green;
	-
	- } else {
	- long long score_ll;
	-
	- pcmk__scan_ll(score, &score_ll, 0LL);
	- if (score_ll > CRM_SCORE_INFINITY) {
	- score_f = CRM_SCORE_INFINITY;
	-
	- } else if (score_ll < -CRM_SCORE_INFINITY) {
	- score_f = -CRM_SCORE_INFINITY;
	-
	- } else {
	- score_f = (int) score_ll;
	- }
	- }
	-
	- return score_f;
	-}
	-
	-char *
	-score2char_stack(int score, char *buf, size_t len)
	-{
	- CRM_CHECK((buf != NULL) && (len >= sizeof(CRM_MINUS_INFINITY_S)),
	- return NULL);
	-
	- if (score >= CRM_SCORE_INFINITY) {
	- strncpy(buf, CRM_INFINITY_S, 9);
	- } else if (score <= -CRM_SCORE_INFINITY) {
	- strncpy(buf, CRM_MINUS_INFINITY_S , 10);
	- } else {
	- snprintf(buf, len, "%d", score);
	- }
	- return buf;
	-}
	-
	-char *
	-score2char(int score)
	-{
	- if (score >= CRM_SCORE_INFINITY) {
	- return strdup(CRM_INFINITY_S);
	-
	- } else if (score <= -CRM_SCORE_INFINITY) {
	- return strdup(CRM_MINUS_INFINITY_S);
	- }
	- return pcmk__itoa(score);
	-}
	-
	int
	crm_user_lookup(const char name, uid_t uid, gid_t * gid)
	{
	int rc = pcmk_ok;
	char *buffer = NULL;
	struct passwd pwd;
	struct passwd *pwentry = NULL;

	buffer = calloc(1, PW_BUFFER_LEN);
	if (buffer == NULL) {
	return -ENOMEM;
	}

	rc = getpwnam_r(name, &pwd, buffer, PW_BUFFER_LEN, &pwentry);
	if (pwentry) {
	if (uid) {
	*uid = pwentry->pw_uid;
	}
	if (gid) {
	*gid = pwentry->pw_gid;
	}
	crm_trace("User %s has uid=%d gid=%d", name, pwentry->pw_uid, pwentry->pw_gid);

	} else {
	rc = rc? -rc : -EINVAL;
	crm_info("User %s lookup: %s", name, pcmk_strerror(rc));
	}

	free(buffer);
	return rc;
	}

	/*!
	* \brief Get user and group IDs of pacemaker daemon user
	*
	* \param[out] uid If non-NULL, where to store daemon user ID
	* \param[out] gid If non-NULL, where to store daemon group ID
	*
	* \return pcmk_ok on success, -errno otherwise
	*/
	int
	pcmk_daemon_user(uid_t uid, gid_t gid)
	{
	static uid_t daemon_uid;
	static gid_t daemon_gid;
	static bool found = false;
	int rc = pcmk_ok;

	if (!found) {
	rc = crm_user_lookup(CRM_DAEMON_USER, &daemon_uid, &daemon_gid);
	if (rc == pcmk_ok) {
	found = true;
	}
	}
	if (found) {
	if (uid) {
	*uid = daemon_uid;
	}
	if (gid) {
	*gid = daemon_gid;
	}
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Return the integer equivalent of a portion of a string
	*
	* \param[in] text Pointer to beginning of string portion
	* \param[out] end_text This will point to next character after integer
	*/
	static int
	version_helper(const char text, const char *end_text)
	{
	int atoi_result = -1;

	CRM_ASSERT(end_text != NULL);

	errno = 0;

	if (text != NULL && text[0] != 0) {
	/* seemingly sacrificing const-correctness -- because while strtol
	doesn't modify the input, it doesn't want to artificially taint the
	"end_text" pointer-to-pointer-to-first-char-in-string with constness
	in case the input wasn't actually constant -- by semantic definition
	not a single character will get modified so it shall be perfectly
	safe to make compiler happy with dropping "const" qualifier here */
	atoi_result = (int) strtol(text, (char **) end_text, 10);

	if (errno == EINVAL) {
	crm_err("Conversion of '%s' %c failed", text, text[0]);
	atoi_result = -1;
	}
	}
	return atoi_result;
	}

	/*
	* version1 < version2 : -1
	* version1 = version2 : 0
	* version1 > version2 : 1
	*/
	int
	compare_version(const char version1, const char version2)
	{
	int rc = 0;
	int lpc = 0;
	const char ver1_iter, ver2_iter;

	if (version1 == NULL && version2 == NULL) {
	return 0;
	} else if (version1 == NULL) {
	return -1;
	} else if (version2 == NULL) {
	return 1;
	}

	ver1_iter = version1;
	ver2_iter = version2;

	while (1) {
	int digit1 = 0;
	int digit2 = 0;

	lpc++;

	if (ver1_iter == ver2_iter) {
	break;
	}

	if (ver1_iter != NULL) {
	digit1 = version_helper(ver1_iter, &ver1_iter);
	}

	if (ver2_iter != NULL) {
	digit2 = version_helper(ver2_iter, &ver2_iter);
	}

	if (digit1 < digit2) {
	rc = -1;
	break;

	} else if (digit1 > digit2) {
	rc = 1;
	break;
	}

	if (ver1_iter != NULL && *ver1_iter == '.') {
	ver1_iter++;
	}
	if (ver1_iter != NULL && *ver1_iter == '\0') {
	ver1_iter = NULL;
	}

	if (ver2_iter != NULL && *ver2_iter == '.') {
	ver2_iter++;
	}
	if (ver2_iter != NULL && *ver2_iter == 0) {
	ver2_iter = NULL;
	}
	}

	if (rc == 0) {
	crm_trace("%s == %s (%d)", version1, version2, lpc);
	} else if (rc < 0) {
	crm_trace("%s < %s (%d)", version1, version2, lpc);
	} else if (rc > 0) {
	crm_trace("%s > %s (%d)", version1, version2, lpc);
	}

	return rc;
	}

	/*!
	* \brief Parse milliseconds from a Pacemaker interval specification
	*
	* \param[in] input Pacemaker time interval specification (a bare number of
	* seconds, a number with a unit optionally with whitespace
	* before and/or after the number, or an ISO 8601 duration)
	*
	* \return Milliseconds equivalent of given specification on success (limited
	* to the range of an unsigned integer), 0 if input is NULL,
	* or 0 (and set errno to EINVAL) on error
	*/
	guint
	crm_parse_interval_spec(const char *input)
	{
	long long msec = -1;

	errno = 0;
	if (input == NULL) {
	return 0;

	} else if (input[0] == 'P') {
	crm_time_t *period_s = crm_time_parse_duration(input);

	if (period_s) {
	msec = 1000 * crm_time_get_seconds(period_s);
	crm_time_free(period_s);
	}

	} else {
	msec = crm_get_msec(input);
	}

	if (msec < 0) {
	crm_warn("Using 0 instead of '%s'", input);
	errno = EINVAL;
	return 0;
	}
	return (msec >= G_MAXUINT)? G_MAXUINT : (guint) msec;
	}

	/*!
	* \internal
	* \brief Log a failed assertion
	*
	* \param[in] file File making the assertion
	* \param[in] function Function making the assertion
	* \param[in] line Line of file making the assertion
	* \param[in] assert_condition String representation of assertion
	*/
	static void
	log_assertion_as(const char file, const char function, int line,
	const char *assert_condition)
	{
	if (!pcmk__is_daemon) {
	crm_enable_stderr(TRUE); // Make sure command-line user sees message
	}
	crm_err("%s: Triggered fatal assertion at %s:%d : %s",
	function, file, line, assert_condition);
	}

	/* coverity[+kill] */
	/*!
	* \internal
	* \brief Log a failed assertion and abort
	*
	* \param[in] file File making the assertion
	* \param[in] function Function making the assertion
	* \param[in] line Line of file making the assertion
	* \param[in] assert_condition String representation of assertion
	*
	* \note This does not return
	*/
	static _Noreturn void
	abort_as(const char file, const char function, int line,
	const char *assert_condition)
	{
	log_assertion_as(file, function, line, assert_condition);
	abort();
	}

	/* coverity[+kill] */
	/*!
	* \internal
	* \brief Handle a failed assertion
	*
	* When called by a daemon, fork a child that aborts (to dump core), otherwise
	* abort the current process.
	*
	* \param[in] file File making the assertion
	* \param[in] function Function making the assertion
	* \param[in] line Line of file making the assertion
	* \param[in] assert_condition String representation of assertion
	*/
	static void
	fail_assert_as(const char file, const char function, int line,
	const char *assert_condition)
	{
	int status = 0;
	pid_t pid = 0;

	if (!pcmk__is_daemon) {
	abort_as(file, function, line, assert_condition); // does not return
	}

	pid = fork();
	switch (pid) {
	case -1: // Fork failed
	crm_warn("%s: Cannot dump core for non-fatal assertion at %s:%d "
	": %s", function, file, line, assert_condition);
	break;

	case 0: // Child process: just abort to dump core
	abort();
	break;

	default: // Parent process: wait for child
	crm_err("%s: Forked child [%d] to record non-fatal assertion at "
	"%s:%d : %s", function, pid, file, line, assert_condition);
	crm_write_blackbox(SIGTRAP, NULL);
	do {
	if (waitpid(pid, &status, 0) == pid) {
	return; // Child finished dumping core
	}
	} while (errno == EINTR);
	if (errno == ECHILD) {
	// crm_mon ignores SIGCHLD
	crm_trace("Cannot wait on forked child [%d] "
	"(SIGCHLD is probably ignored)", pid);
	} else {
	crm_err("Cannot wait on forked child [%d]: %s",
	pid, pcmk_rc_str(errno));
	}
	break;
	}
	}

	/* coverity[+kill] */
	void
	crm_abort(const char file, const char function, int line,
	const char *assert_condition, gboolean do_core, gboolean do_fork)
	{
	if (!do_fork) {
	abort_as(file, function, line, assert_condition);
	} else if (do_core) {
	fail_assert_as(file, function, line, assert_condition);
	} else {
	log_assertion_as(file, function, line, assert_condition);
	}
	}

	/*!
	* \internal
	* \brief Convert the current process to a daemon process
	*
	* Fork a child process, exit the parent, create a PID file with the current
	* process ID, and close the standard input/output/error file descriptors.
	* Exit instead if a daemon is already running and using the PID file.
	*
	* \param[in] name Daemon executable name
	* \param[in] pidfile File name to use as PID file
	*/
	void
	pcmk__daemonize(const char name, const char pidfile)
	{
	int rc;
	pid_t pid;

	/* Check before we even try... */
	rc = pcmk__pidfile_matches(pidfile, 1, name, &pid);
	if ((rc != pcmk_rc_ok) && (rc != ENOENT)) {
	crm_err("%s: already running [pid %lld in %s]",
	name, (long long) pid, pidfile);
	printf("%s: already running [pid %lld in %s]\n",
	name, (long long) pid, pidfile);
	crm_exit(CRM_EX_ERROR);
	}

	pid = fork();
	if (pid < 0) {
	fprintf(stderr, "%s: could not start daemon\n", name);
	crm_perror(LOG_ERR, "fork");
	crm_exit(CRM_EX_OSERR);

	} else if (pid > 0) {
	crm_exit(CRM_EX_OK);
	}

	rc = pcmk__lock_pidfile(pidfile, name);
	if (rc != pcmk_rc_ok) {
	crm_err("Could not lock '%s' for %s: %s " CRM_XS " rc=%d",
	pidfile, name, pcmk_rc_str(rc), rc);
	printf("Could not lock '%s' for %s: %s (%d)\n",
	pidfile, name, pcmk_rc_str(rc), rc);
	crm_exit(CRM_EX_ERROR);
	}

	umask(S_IWGRP \| S_IWOTH \| S_IROTH);

	close(STDIN_FILENO);
	pcmk__open_devnull(O_RDONLY); // stdin (fd 0)

	close(STDOUT_FILENO);
	pcmk__open_devnull(O_WRONLY); // stdout (fd 1)

	close(STDERR_FILENO);
	pcmk__open_devnull(O_WRONLY); // stderr (fd 2)
	}

	char *
	crm_meta_name(const char *field)
	{
	int lpc = 0;
	int max = 0;
	char *crm_name = NULL;

	CRM_CHECK(field != NULL, return NULL);
	crm_name = crm_strdup_printf(CRM_META "_%s", field);

	/* Massage the names so they can be used as shell variables */
	max = strlen(crm_name);
	for (; lpc < max; lpc++) {
	switch (crm_name[lpc]) {
	case '-':
	crm_name[lpc] = '_';
	break;
	}
	}
	return crm_name;
	}

	const char *
	crm_meta_value(GHashTable * hash, const char *field)
	{
	char *key = NULL;
	const char *value = NULL;

	key = crm_meta_name(field);
	if (key) {
	value = g_hash_table_lookup(hash, key);
	free(key);
	}

	return value;
	}

	#ifdef HAVE_UUID_UUID_H
	# include <uuid/uuid.h>
	#endif

	char *
	crm_generate_uuid(void)
	{
	unsigned char uuid[16];
	char buffer = malloc(37); / Including NUL byte */

	uuid_generate(uuid);
	uuid_unparse(uuid, buffer);
	return buffer;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	void
	crm_gnutls_global_init(void)
	{
	signal(SIGPIPE, SIG_IGN);
	gnutls_global_init();
	}
	#endif

	/*!
	* \brief Get the local hostname
	*
	* \return Newly allocated string with name, or NULL (and set errno) on error
	*/
	char *
	pcmk_hostname()
	{
	struct utsname hostinfo;

	return (uname(&hostinfo) < 0)? NULL : strdup(hostinfo.nodename);
	}

	bool
	pcmk_str_is_infinity(const char *s) {
	return pcmk__str_any_of(s, CRM_INFINITY_S, CRM_PLUS_INFINITY_S, NULL);
	}

	bool
	pcmk_str_is_minus_infinity(const char *s) {
	return pcmk__str_eq(s, CRM_MINUS_INFINITY_S, pcmk__str_none);
	}

	/*!
	* \internal
	* \brief Sleep for given milliseconds
	*
	* \param[in] ms Time to sleep
	*
	* \note The full time might not be slept if a signal is received.
	*/
	void
	pcmk__sleep_ms(unsigned int ms)
	{
	// @TODO Impose a sane maximum sleep to avoid hanging a process for long
	//CRM_CHECK(ms <= MAX_SLEEP, ms = MAX_SLEEP);

	// Use sleep() for any whole seconds
	if (ms >= 1000) {
	sleep(ms / 1000);
	ms -= ms / 1000;
	}

	if (ms == 0) {
	return;
	}

	#if defined(HAVE_NANOSLEEP)
	// nanosleep() is POSIX-2008, so prefer that
	{
	struct timespec req = { .tv_sec = 0, .tv_nsec = (long) (ms * 1000000) };

	nanosleep(&req, NULL);
	}
	#elif defined(HAVE_USLEEP)
	// usleep() is widely available, though considered obsolete
	usleep((useconds_t) ms);
	#else
	// Otherwise use a trick with select() timeout
	{
	struct timeval tv = { .tv_sec = 0, .tv_usec = (suseconds_t) ms };

	select(0, NULL, NULL, NULL, &tv);
	}
	#endif
	}
	diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h
	index 70392ad87a..f222f6eacc 100644
	--- a/lib/pacemaker/libpacemaker_private.h
	+++ b/lib/pacemaker/libpacemaker_private.h
	@@ -1,367 +1,386 @@
	/*
	* Copyright 2021-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__LIBPACEMAKER_PRIVATE__H
	# define PCMK__LIBPACEMAKER_PRIVATE__H

	/* This header is for the sole use of libpacemaker, so that functions can be
	* declared with G_GNUC_INTERNAL for efficiency.
	*/

	#include <crm/pengine/pe_types.h> // pe_action_t, pe_node_t, pe_working_set_t

	// Actions (pcmk_sched_actions.c)

	G_GNUC_INTERNAL
	void pcmk__update_action_for_orderings(pe_action_t *action,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__log_action(const char pre_text, pe_action_t action, bool details);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_rsc_pseudo_action(pe_resource_t rsc, const char *task,
	bool optional, bool runnable);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_cancel_action(pe_resource_t rsc, const char *name,
	guint interval_ms, pe_node_t *node);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_shutdown_action(pe_node_t node,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__action_locks_rsc_to_node(const pe_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__deduplicate_action_inputs(pe_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__output_actions(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__check_action_config(pe_resource_t rsc, pe_node_t node,
	xmlNode *xml_op);

	G_GNUC_INTERNAL
	void pcmk__handle_rsc_config_changes(pe_working_set_t *data_set);

	// Producing transition graphs (pcmk_graph_producer.c)

	G_GNUC_INTERNAL
	bool pcmk__graph_has_loop(pe_action_t init_action, pe_action_t action,
	pe_action_wrapper_t *input);

	G_GNUC_INTERNAL
	void pcmk__add_action_to_graph(pe_action_t action, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__create_graph(pe_working_set_t *data_set);


	// Fencing (pcmk_sched_fencing.c)

	G_GNUC_INTERNAL
	void pcmk__order_vs_fence(pe_action_t stonith_op, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__order_vs_unfence(pe_resource_t rsc, pe_node_t node,
	pe_action_t *action, enum pe_ordering order,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__fence_guest(pe_node_t node, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	bool pcmk__node_unfenced(pe_node_t *node);

	G_GNUC_INTERNAL
	bool pcmk__is_unfence_device(const pe_resource_t *rsc,
	const pe_working_set_t *data_set);


	// Injected scheduler inputs (pcmk_sched_injections.c)

	void pcmk__inject_scheduler_input(pe_working_set_t data_set, cib_t cib,
	pcmk_injections_t *injections);


	// Constraints of any type (pcmk_sched_constraints.c)

	G_GNUC_INTERNAL
	pe_resource_t pcmk__find_constraint_resource(GList rsc_list, const char *id);

	G_GNUC_INTERNAL
	xmlNode pcmk__expand_tags_in_sets(xmlNode xml_obj,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__valid_resource_or_tag(pe_working_set_t data_set, const char id,
	pe_resource_t rsc, pe_tag_t tag);

	G_GNUC_INTERNAL
	bool pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	bool convert_rsc, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__create_internal_constraints(pe_working_set_t *data_set);


	// Location constraints

	G_GNUC_INTERNAL
	void pcmk__unpack_location(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	pe__location_t pcmk__new_location(const char id, pe_resource_t *rsc,
	int node_weight, const char *discover_mode,
	pe_node_t *foo_node,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_locations(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_location(pe__location_t constraint, pe_resource_t rsc);


	// Colocation constraints

	enum pcmk__coloc_affects {
	pcmk__coloc_affects_nothing = 0,
	pcmk__coloc_affects_location,
	pcmk__coloc_affects_role,
	};

	G_GNUC_INTERNAL
	enum pcmk__coloc_affects pcmk__colocation_affects(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint,
	bool preview);

	G_GNUC_INTERNAL
	void pcmk__apply_coloc_to_weights(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint);

	G_GNUC_INTERNAL
	void pcmk__apply_coloc_to_priority(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint);

	G_GNUC_INTERNAL
	void pcmk__unpack_colocation(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__new_colocation(const char id, const char node_attr, int score,
	pe_resource_t dependent, pe_resource_t primary,
	const char dependent_role, const char primary_role,
	bool influence, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__block_colocated_starts(pe_action_t *action,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__new_ordering(pe_resource_t lh_rsc, char lh_task,
	pe_action_t lh_action, pe_resource_t rh_rsc,
	char rh_task, pe_action_t rh_action,
	enum pe_ordering type, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__unpack_ordering(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__disable_invalid_orderings(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__order_stops_before_shutdown(pe_node_t *node,
	pe_action_t *shutdown_op,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_orderings(pe_working_set_t *data_set);

	/*!
	* \internal
	* \brief Create a new ordering between two resource actions
	*
	* \param[in] lh_rsc Resource for 'first' action
	* \param[in] rh_rsc Resource for 'then' action
	* \param[in] lh_task Action key for 'first' action
	* \param[in] rh_task Action key for 'then' action
	* \param[in] flags Bitmask of enum pe_ordering flags
	* \param[in] data_set Cluster working set to add ordering to
	*/
	#define pcmk__order_resource_actions(lh_rsc, lh_task, rh_rsc, rh_task, \
	flags, data_set) \
	pcmk__new_ordering((lh_rsc), pcmk__op_key((lh_rsc)->id, (lh_task), 0), \
	NULL, \
	(rh_rsc), pcmk__op_key((rh_rsc)->id, (rh_task), 0), \
	NULL, (flags), (data_set))

	#define pcmk__order_starts(rsc1, rsc2, type, data_set) \
	pcmk__order_resource_actions((rsc1), CRMD_ACTION_START, \
	(rsc2), CRMD_ACTION_START, (type), (data_set))

	#define pcmk__order_stops(rsc1, rsc2, type, data_set) \
	pcmk__order_resource_actions((rsc1), CRMD_ACTION_STOP, \
	(rsc2), CRMD_ACTION_STOP, (type), (data_set))

	G_GNUC_INTERNAL
	void pcmk__unpack_rsc_ticket(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	bool pcmk__is_failed_remote_node(pe_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__order_remote_connection_actions(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__rsc_corresponds_to_guest(pe_resource_t rsc, pe_node_t node);

	G_GNUC_INTERNAL
	pe_node_t pcmk__connection_host_for_action(pe_action_t action);

	G_GNUC_INTERNAL
	void pcmk__substitute_remote_addr(pe_resource_t rsc, GHashTable params,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__add_bundle_meta_to_xml(xmlNode args_xml, pe_action_t action);


	// Groups (pcmk_sched_group.c)

	G_GNUC_INTERNAL
	GList pcmk__group_colocated_resources(pe_resource_t rsc,
	pe_resource_t *orig_rsc,
	GList *colocated_rscs);


	// Bundles (pcmk_sched_bundle.c)

	G_GNUC_INTERNAL
	void pcmk__output_bundle_actions(pe_resource_t *rsc);


	// Injections (pcmk_injections.c)

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_node(cib_t cib_conn, const char node, const char uuid);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_node_state_change(cib_t cib_conn, const char *node,
	bool up);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_resource_history(pcmk__output_t out, xmlNode *cib_node,
	const char *resource,
	const char *lrm_name,
	const char *rclass,
	const char *rtype,
	const char *rprovider);

	G_GNUC_INTERNAL
	void pcmk__inject_failcount(pcmk__output_t out, xmlNode cib_node,
	const char resource, const char task,
	guint interval_ms, int rc);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_action_result(xmlNode cib_resource,
	lrmd_event_data_t *op, int target_rc);


	+// Nodes (pcmk_sched_nodes.c)
	+
	+G_GNUC_INTERNAL
	+bool pcmk__node_available(const pe_node_t *node);
	+
	+G_GNUC_INTERNAL
	+bool pcmk__any_node_available(GHashTable *nodes);
	+
	+G_GNUC_INTERNAL
	+GHashTable pcmk__copy_node_table(GHashTable nodes);
	+
	+G_GNUC_INTERNAL
	+GList pcmk__sort_nodes(GList nodes, pe_node_t *active_node,
	+ pe_working_set_t *data_set);
	+
	+G_GNUC_INTERNAL
	+void pcmk__apply_node_health(pe_working_set_t *data_set);
	+
	+
	// Clone notifictions (pcmk_sched_notif.c)

	G_GNUC_INTERNAL
	void pcmk__create_notifications(pe_resource_t rsc, notify_data_t n_data);

	G_GNUC_INTERNAL
	notify_data_t pcmk__clone_notif_pseudo_ops(pe_resource_t rsc,
	const char *task,
	pe_action_t *action,
	pe_action_t *complete);

	G_GNUC_INTERNAL
	void pcmk__free_notification_data(notify_data_t *n_data);

	G_GNUC_INTERNAL
	void pcmk__order_notifs_after_fencing(pe_action_t action, pe_resource_t rsc,
	pe_action_t *stonith_op);


	// Functions applying to more than one variant (pcmk_sched_resource.c)

	G_GNUC_INTERNAL
	void pcmk__set_allocation_methods(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__rsc_agent_changed(pe_resource_t rsc, pe_node_t node,
	const xmlNode *rsc_entry, bool active_on_node);

	G_GNUC_INTERNAL
	GList pcmk__rscs_matching_id(const char id, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	GList pcmk__colocated_resources(pe_resource_t rsc, pe_resource_t *orig_rsc,
	GList *colocated_rscs);

	G_GNUC_INTERNAL
	void pcmk__output_resource_actions(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__assign_primitive(pe_resource_t rsc, pe_node_t chosen, bool force);

	G_GNUC_INTERNAL
	bool pcmk__assign_resource(pe_resource_t rsc, pe_node_t node, bool force);

	G_GNUC_INTERNAL
	void pcmk__unassign_resource(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__threshold_reached(pe_resource_t rsc, pe_node_t node,
	pe_resource_t **failed);

	G_GNUC_INTERNAL
	void pcmk__sort_resources(pe_working_set_t *data_set);


	// Functions related to probes (pcmk_sched_probes.c)

	G_GNUC_INTERNAL
	void pcmk__order_probes(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__schedule_probes(pe_working_set_t *data_set);


	// Functions related to node utilization (pcmk_sched_utilization.c)

	G_GNUC_INTERNAL
	int pcmk__compare_node_capacities(const pe_node_t *node1,
	const pe_node_t *node2);

	G_GNUC_INTERNAL
	void pcmk__consume_node_capacity(GHashTable *current_utilization,
	pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__release_node_capacity(GHashTable *current_utilization,
	pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__ban_insufficient_capacity(pe_resource_t rsc, pe_node_t *prefer,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL void pcmk__create_utilization_constraints(pe_resource_t *rsc,
	GList *allowed_nodes);

	#endif // PCMK__LIBPACEMAKER_PRIVATE__H
	diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c
	index 81e0ebea7a..71315e8c63 100644
	--- a/lib/pacemaker/pcmk_sched_allocate.c
	+++ b/lib/pacemaker/pcmk_sched_allocate.c
	@@ -1,746 +1,640 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	CRM_TRACE_INIT_DATA(pacemaker);

	extern bool pcmk__is_daemon;

	/*!
	* \internal
	* \brief Do deferred action checks after allocation
	*
	* \param[in] data_set Working set for cluster
	*/
	static void
	check_params(pe_resource_t rsc, pe_node_t node, xmlNode *rsc_op,
	enum pe_check_parameters check, pe_working_set_t *data_set)
	{
	const char *reason = NULL;
	op_digest_cache_t *digest_data = NULL;

	switch (check) {
	case pe_check_active:
	if (pcmk__check_action_config(rsc, node, rsc_op)
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {

	reason = "action definition changed";
	}
	break;

	case pe_check_last_failure:
	digest_data = rsc_action_digest_cmp(rsc, rsc_op, node, data_set);
	switch (digest_data->rc) {
	case RSC_DIGEST_UNKNOWN:
	crm_trace("Resource %s history entry %s on %s has no digest to compare",
	rsc->id, ID(rsc_op), node->details->id);
	break;
	case RSC_DIGEST_MATCH:
	break;
	default:
	reason = "resource parameters have changed";
	break;
	}
	break;
	}

	if (reason) {
	pe__clear_failcount(rsc, node, reason, data_set);
	}
	}

	static gboolean
	failcount_clear_action_exists(pe_node_t * node, pe_resource_t * rsc)
	{
	gboolean rc = FALSE;
	GList *list = pe__resource_actions(rsc, node, CRM_OP_CLEAR_FAILCOUNT, TRUE);

	if (list) {
	rc = TRUE;
	}
	g_list_free(list);
	return rc;
	}

	static void
	common_apply_stickiness(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	common_apply_stickiness(child_rsc, node, data_set);
	}
	return;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)
	&& rsc->stickiness != 0 && pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
	pe_node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (current == NULL) {

	} else if ((match != NULL)
	\|\| pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	pe_resource_t *sticky_rsc = rsc;

	resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
	pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
	" (node=%s, weight=%d)", sticky_rsc->id,
	node->details->uname, rsc->stickiness);
	} else {
	GHashTableIter iter;
	pe_node_t *nIter = NULL;

	pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
	" and node %s is not explicitly allowed", rsc->id, node->details->uname);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
	crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
	}
	}
	}

	/* Check the migration threshold only if a failcount clear action
	* has not already been placed for this resource on the node.
	* There is no sense in potentially forcing the resource from this
	* node if the failcount is being reset anyway.
	*
	* @TODO A clear_failcount operation can be scheduled in
	* pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
	* stage5() via check_params(). This runs in stage2(), so it cannot detect
	* those, meaning we might check the migration threshold when we shouldn't
	* -- worst case, we stop or move the resource, then move it back next
	* transition.
	*/
	if (failcount_clear_action_exists(node, rsc) == FALSE) {
	pe_resource_t *failed = NULL;

	if (pcmk__threshold_reached(rsc, node, &failed)) {
	resource_location(failed, node, -INFINITY, "__fail_limit__",
	data_set);
	}
	}
	}

	-static void
	-calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
	-{
	- const char key = (const char )gKey;
	- const char value = (const char )gValue;
	- int system_health = (int )user_data;
	-
	- if (!gKey \|\| !gValue \|\| !user_data) {
	- return;
	- }
	-
	- if (pcmk__starts_with(key, "#health")) {
	- int score;
	-
	- /* Convert the value into an integer */
	- score = char2score(value);
	-
	- /* Add it to the running total */
	- system_health = pe__add_scores(score, system_health);
	- }
	-}
	-
	-static gboolean
	-apply_system_health(pe_working_set_t * data_set)
	-{
	- GList *gIter = NULL;
	- const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
	- int base_health = 0;
	-
	- if (pcmk__str_eq(health_strategy, "none", pcmk__str_null_matches \| pcmk__str_casei)) {
	- /* Prevent any accidental health -> score translation */
	- pcmk__score_red = 0;
	- pcmk__score_yellow = 0;
	- pcmk__score_green = 0;
	- return TRUE;
	-
	- } else if (pcmk__str_eq(health_strategy, "migrate-on-red", pcmk__str_casei)) {
	-
	- /* Resources on nodes which have health values of red are
	- * weighted away from that node.
	- */
	- pcmk__score_red = -INFINITY;
	- pcmk__score_yellow = 0;
	- pcmk__score_green = 0;
	-
	- } else if (pcmk__str_eq(health_strategy, "only-green", pcmk__str_casei)) {
	-
	- /* Resources on nodes which have health values of red or yellow
	- * are forced away from that node.
	- */
	- pcmk__score_red = -INFINITY;
	- pcmk__score_yellow = -INFINITY;
	- pcmk__score_green = 0;
	-
	- } else if (pcmk__str_eq(health_strategy, "progressive", pcmk__str_casei)) {
	- /* Same as the above, but use the r/y/g scores provided by the user
	- * Defaults are provided by the pe_prefs table
	- * Also, custom health "base score" can be used
	- */
	- base_health = char2score(pe_pref(data_set->config_hash,
	- "node-health-base"));
	-
	- } else if (pcmk__str_eq(health_strategy, "custom", pcmk__str_casei)) {
	-
	- /* Requires the admin to configure the rsc_location constaints for
	- * processing the stored health scores
	- */
	- /* TODO: Check for the existence of appropriate node health constraints */
	- return TRUE;
	-
	- } else {
	- crm_err("Unknown node health strategy: %s", health_strategy);
	- return FALSE;
	- }
	-
	- crm_info("Applying automated node health strategy: %s", health_strategy);
	-
	- for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	- int system_health = base_health;
	- pe_node_t node = (pe_node_t ) gIter->data;
	-
	- /* Search through the node hash table for system health entries. */
	- g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);
	-
	- crm_info(" Node %s has an combined system health of %d",
	- node->details->uname, system_health);
	-
	- /* If the health is non-zero, then create a new location constraint so
	- * that the weight will be added later on.
	- */
	- if (system_health != 0) {
	-
	- GList *gIter2 = data_set->resources;
	-
	- for (; gIter2 != NULL; gIter2 = gIter2->next) {
	- pe_resource_t rsc = (pe_resource_t ) gIter2->data;
	-
	- pcmk__new_location(health_strategy, rsc, system_health, NULL,
	- node, data_set);
	- }
	- }
	- }
	-
	- return TRUE;
	-}
	-
	gboolean
	stage0(pe_working_set_t * data_set)
	{
	if (data_set->input == NULL) {
	return FALSE;
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_have_status)) {
	crm_trace("Calculating status");
	cluster_status(data_set);
	}

	pcmk__set_allocation_methods(data_set);
	- apply_system_health(data_set);
	+ pcmk__apply_node_health(data_set);
	pcmk__unpack_constraints(data_set);

	return TRUE;
	}

	static void
	rsc_discover_filter(pe_resource_t rsc, pe_node_t node)
	{
	pe_resource_t *top = uber_parent(rsc);
	pe_node_t *match;

	if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
	return;
	}

	g_list_foreach(rsc->children, (GFunc) rsc_discover_filter, node);

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match && match->rsc_discover_mode != pe_discover_exclusive) {
	match->weight = -INFINITY;
	}
	}

	static time_t
	shutdown_time(pe_node_t node, pe_working_set_t data_set)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
	time_t result = 0;

	if (shutdown) {
	long long result_ll;

	if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
	result = (time_t) result_ll;
	}
	}
	return result? result : get_effective_time(data_set);
	}

	static void
	apply_shutdown_lock(pe_resource_t rsc, pe_working_set_t data_set)
	{
	const char *class;

	// Only primitives and (uncloned) groups may be locked
	if (rsc->variant == pe_group) {
	g_list_foreach(rsc->children, (GFunc) apply_shutdown_lock, data_set);
	} else if (rsc->variant != pe_native) {
	return;
	}

	// Fence devices and remote connections can't be locked
	class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
	\|\| pe__resource_is_remote_conn(rsc, data_set)) {
	return;
	}

	if (rsc->lock_node != NULL) {
	// The lock was obtained from resource history

	if (rsc->running_on != NULL) {
	/* The resource was started elsewhere even though it is now
	* considered locked. This shouldn't be possible, but as a
	* failsafe, we don't want to disturb the resource now.
	*/
	pe_rsc_info(rsc,
	"Cancelling shutdown lock because %s is already active",
	rsc->id);
	pe__clear_resource_history(rsc, rsc->lock_node, data_set);
	rsc->lock_node = NULL;
	rsc->lock_time = 0;
	}

	// Only a resource active on exactly one node can be locked
	} else if (pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *node = rsc->running_on->data;

	if (node->details->shutdown) {
	if (node->details->unclean) {
	pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
	rsc->id, node->details->uname);
	} else {
	rsc->lock_node = node;
	rsc->lock_time = shutdown_time(node, data_set);
	}
	}
	}

	if (rsc->lock_node == NULL) {
	// No lock needed
	return;
	}

	if (data_set->shutdown_lock > 0) {
	time_t lock_expiration = rsc->lock_time + data_set->shutdown_lock;

	pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
	rsc->id, rsc->lock_node->details->uname,
	(long long) lock_expiration);
	pe__update_recheck_time(++lock_expiration, data_set);
	} else {
	pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
	rsc->id, rsc->lock_node->details->uname);
	}

	// If resource is locked to one node, ban it from all other nodes
	for (GList *item = data_set->nodes; item != NULL; item = item->next) {
	pe_node_t *node = item->data;

	if (strcmp(node->details->uname, rsc->lock_node->details->uname)) {
	resource_location(rsc, node, -CRM_SCORE_INFINITY,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK, data_set);
	}
	}
	}

	/*
	* \internal
	* \brief Stage 2 of cluster status: apply node-specific criteria
	*
	* Count known nodes, and apply location constraints, stickiness, and exclusive
	* resource discovery.
	*/
	gboolean
	stage2(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
	g_list_foreach(data_set->resources, (GFunc) apply_shutdown_lock, data_set);
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_no_compat)) {
	// @COMPAT API backward compatibility
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (node && (node->weight >= 0) && node->details->online
	&& (node->details->type != node_ping)) {
	data_set->max_valid_nodes++;
	}
	}
	}

	pcmk__apply_locations(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	GList *gIter2 = NULL;
	pe_node_t node = (pe_node_t ) gIter->data;

	gIter2 = data_set->resources;
	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	common_apply_stickiness(rsc, node, data_set);
	rsc_discover_filter(rsc, node);
	}
	}

	return TRUE;
	}

	static void
	allocate_resources(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	if (pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	/* Allocate remote connection resources first (which will also allocate
	* any colocation dependencies). If the connection is migrating, always
	* prefer the partial migration target.
	*/
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;
	if (rsc->is_remote_node == FALSE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating remote connection resource '%s'",
	rsc->id);
	rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
	}
	}

	/* now do the rest of the resources */
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;
	if (rsc->is_remote_node == TRUE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating %s resource '%s'",
	crm_element_name(rsc->xml), rsc->id);
	rsc->cmds->allocate(rsc, NULL, data_set);
	}
	}

	// Clear fail counts for orphaned rsc on all online nodes
	static void
	cleanup_orphans(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (node->details->online
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {

	pe_action_t *clear_op = NULL;

	clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
	data_set);

	/* We can't use order_action_then_stop() here because its
	* pe_order_preserve breaks things
	*/
	pcmk__new_ordering(clear_op->rsc, NULL, clear_op,
	rsc, stop_key(rsc), NULL,
	pe_order_optional, data_set);
	}
	}
	}

	gboolean
	stage5(pe_working_set_t * data_set)
	{
	pcmk__output_t *out = data_set->priv;
	GList *gIter = NULL;

	if (!pcmk__str_eq(data_set->placement_strategy, "default", pcmk__str_casei)) {
	pcmk__sort_resources(data_set);
	}

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	out->message(out, "node-capacity", node, "Original");
	}
	}

	crm_trace("Allocating services");
	/* Take (next) highest resource, assign it and create its actions */

	allocate_resources(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	out->message(out, "node-capacity", node, "Remaining");
	}
	}

	// Process deferred action checks
	pe__foreach_param_check(data_set, check_params);
	pe__free_param_checks(data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
	crm_trace("Calculating needed probes");
	pcmk__schedule_probes(data_set);
	}

	crm_trace("Handle orphans");
	if (pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* There's no need to recurse into rsc->children because those
	* should just be unallocated clone instances.
	*/
	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	cleanup_orphans(rsc, data_set);
	}
	}
	}

	crm_trace("Creating actions");

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	rsc->cmds->create_actions(rsc, data_set);
	}

	crm_trace("Creating done");
	return TRUE;
	}

	static gboolean
	is_managed(const pe_resource_t * rsc)
	{
	GList *gIter = rsc->children;

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	return TRUE;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (is_managed(child_rsc)) {
	return TRUE;
	}
	}

	return FALSE;
	}

	static gboolean
	any_managed_resources(pe_working_set_t * data_set)
	{

	GList *gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	if (is_managed(rsc)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	/*
	* Create dependencies for stonith and shutdown operations
	*/
	gboolean
	stage6(pe_working_set_t * data_set)
	{
	pe_action_t *dc_down = NULL;
	pe_action_t *stonith_op = NULL;
	gboolean integrity_lost = FALSE;
	gboolean need_stonith = TRUE;
	GList *gIter;
	GList *stonith_ops = NULL;
	GList *shutdown_ops = NULL;

	/* Remote ordering constraints need to happen prior to calculating fencing
	* because it is one more place we can mark nodes as needing fencing.
	*/
	pcmk__order_remote_connection_actions(data_set);

	crm_trace("Processing fencing and shutdown cases");
	if (any_managed_resources(data_set) == FALSE) {
	crm_notice("Delaying fencing operations until there are resources to manage");
	need_stonith = FALSE;
	}

	/* Check each node for stonith/shutdown */
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	/* Guest nodes are "fenced" by recovering their container resource,
	* so handle them separately.
	*/
	if (pe__is_guest_node(node)) {
	if (node->details->remote_requires_reset && need_stonith
	&& pe_can_fence(data_set, node)) {
	pcmk__fence_guest(node, data_set);
	}
	continue;
	}

	stonith_op = NULL;

	if (node->details->unclean
	&& need_stonith && pe_can_fence(data_set, node)) {

	stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", FALSE, data_set);
	pe_warn("Scheduling Node %s for STONITH", node->details->uname);

	pcmk__order_vs_fence(stonith_op, data_set);

	if (node->details->is_dc) {
	// Remember if the DC is being fenced
	dc_down = stonith_op;

	} else {

	if (!pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)
	&& (stonith_ops != NULL)) {
	/* Concurrent fencing is disabled, so order each non-DC
	* fencing in a chain. If there is any DC fencing or
	* shutdown, it will be ordered after the last action in the
	* chain later.
	*/
	order_actions((pe_action_t *) stonith_ops->data,
	stonith_op, pe_order_optional);
	}

	// Remember all non-DC fencing actions in a separate list
	stonith_ops = g_list_prepend(stonith_ops, stonith_op);
	}

	} else if (node->details->online && node->details->shutdown &&
	/* TODO define what a shutdown op means for a remote node.
	* For now we do not send shutdown operations for remote nodes, but
	* if we can come up with a good use for this in the future, we will. */
	pe__is_guest_or_remote_node(node) == FALSE) {

	pe_action_t *down_op = pcmk__new_shutdown_action(node, data_set);

	if (node->details->is_dc) {
	// Remember if the DC is being shut down
	dc_down = down_op;
	} else {
	// Remember non-DC shutdowns for later ordering
	shutdown_ops = g_list_prepend(shutdown_ops, down_op);
	}
	}

	if (node->details->unclean && stonith_op == NULL) {
	integrity_lost = TRUE;
	pe_warn("Node %s is unclean!", node->details->uname);
	}
	}

	if (integrity_lost) {
	if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
	pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");

	} else if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	crm_notice("Cannot fence unclean nodes until quorum is"
	" attained (or no-quorum-policy is set to ignore)");
	}
	}

	if (dc_down != NULL) {
	/* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
	* DC elections. However, we don't want to order non-DC shutdowns before
	* a DC fencing, because even though we don't want a node that's
	* shutting down to become DC, the DC fencing could be ordered before a
	* clone stop that's also ordered before the shutdowns, thus leading to
	* a graph loop.
	*/
	if (pcmk__str_eq(dc_down->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	for (gIter = shutdown_ops; gIter != NULL; gIter = gIter->next) {
	pe_action_t node_stop = (pe_action_t ) gIter->data;

	crm_debug("Ordering shutdown on %s before %s on DC %s",
	node_stop->node->details->uname,
	dc_down->task, dc_down->node->details->uname);

	order_actions(node_stop, dc_down, pe_order_optional);
	}
	}

	// Order any non-DC fencing before any DC fencing or shutdown

	if (pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)) {
	/* With concurrent fencing, order each non-DC fencing action
	* separately before any DC fencing or shutdown.
	*/
	for (gIter = stonith_ops; gIter != NULL; gIter = gIter->next) {
	order_actions((pe_action_t *) gIter->data, dc_down,
	pe_order_optional);
	}
	} else if (stonith_ops) {
	/* Without concurrent fencing, the non-DC fencing actions are
	* already ordered relative to each other, so we just need to order
	* the DC fencing after the last action in the chain (which is the
	* first item in the list).
	*/
	order_actions((pe_action_t *) stonith_ops->data, dc_down,
	pe_order_optional);
	}
	}
	g_list_free(stonith_ops);
	g_list_free(shutdown_ops);
	return TRUE;
	}
	diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c
	index 1b1b5d34ce..6017c43e5c 100644
	--- a/lib/pacemaker/pcmk_sched_colocation.c
	+++ b/lib/pacemaker/pcmk_sched_colocation.c
	@@ -1,1064 +1,1065 @@
	/*
	- * Copyright 2004-2021 the Pacemaker project contributors
	+ * Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include "crm/common/util.h"
	#include "crm/common/xml_internal.h"
	#include "crm/msg_xml.h"
	#include "libpacemaker_private.h"

	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	__rsc = pcmk__find_constraint_resource(data_set->resources, __name); \
	if (__rsc == NULL) { \
	pcmk__config_err("%s: No resource found for %s", __set, __name); \
	return; \
	} \
	} while(0)

	static gint
	cmp_dependent_priority(gconstpointer a, gconstpointer b)
	{
	const pcmk__colocation_t rsc_constraint1 = (const pcmk__colocation_t ) a;
	const pcmk__colocation_t rsc_constraint2 = (const pcmk__colocation_t ) b;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	CRM_ASSERT(rsc_constraint1->dependent != NULL);
	CRM_ASSERT(rsc_constraint1->primary != NULL);

	if (rsc_constraint1->dependent->priority > rsc_constraint2->dependent->priority) {
	return -1;
	}

	if (rsc_constraint1->dependent->priority < rsc_constraint2->dependent->priority) {
	return 1;
	}

	/* Process clones before primitives and groups */
	if (rsc_constraint1->dependent->variant > rsc_constraint2->dependent->variant) {
	return -1;
	}
	if (rsc_constraint1->dependent->variant < rsc_constraint2->dependent->variant) {
	return 1;
	}

	/* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
	* clones (probably unnecessary, but avoids having to update regression
	* tests)
	*/
	if (rsc_constraint1->dependent->variant == pe_clone) {
	if (pcmk_is_set(rsc_constraint1->dependent->flags, pe_rsc_promotable)
	&& !pcmk_is_set(rsc_constraint2->dependent->flags, pe_rsc_promotable)) {
	return -1;
	} else if (!pcmk_is_set(rsc_constraint1->dependent->flags, pe_rsc_promotable)
	&& pcmk_is_set(rsc_constraint2->dependent->flags, pe_rsc_promotable)) {
	return 1;
	}
	}

	return strcmp(rsc_constraint1->dependent->id,
	rsc_constraint2->dependent->id);
	}

	static gint
	cmp_primary_priority(gconstpointer a, gconstpointer b)
	{
	const pcmk__colocation_t rsc_constraint1 = (const pcmk__colocation_t ) a;
	const pcmk__colocation_t rsc_constraint2 = (const pcmk__colocation_t ) b;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	CRM_ASSERT(rsc_constraint1->dependent != NULL);
	CRM_ASSERT(rsc_constraint1->primary != NULL);

	if (rsc_constraint1->primary->priority > rsc_constraint2->primary->priority) {
	return -1;
	}

	if (rsc_constraint1->primary->priority < rsc_constraint2->primary->priority) {
	return 1;
	}

	/* Process clones before primitives and groups */
	if (rsc_constraint1->primary->variant > rsc_constraint2->primary->variant) {
	return -1;
	} else if (rsc_constraint1->primary->variant < rsc_constraint2->primary->variant) {
	return 1;
	}

	/* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
	* clones (probably unnecessary, but avoids having to update regression
	* tests)
	*/
	if (rsc_constraint1->primary->variant == pe_clone) {
	if (pcmk_is_set(rsc_constraint1->primary->flags, pe_rsc_promotable)
	&& !pcmk_is_set(rsc_constraint2->primary->flags, pe_rsc_promotable)) {
	return -1;
	} else if (!pcmk_is_set(rsc_constraint1->primary->flags, pe_rsc_promotable)
	&& pcmk_is_set(rsc_constraint2->primary->flags, pe_rsc_promotable)) {
	return 1;
	}
	}

	return strcmp(rsc_constraint1->primary->id, rsc_constraint2->primary->id);
	}

	/*!
	* \internal
	* \brief Add orderings necessary for an anti-colocation constraint
	*/
	static void
	anti_colocation_order(pe_resource_t *first_rsc, int first_role,
	pe_resource_t *then_rsc, int then_role,
	pe_working_set_t *data_set)
	{
	const char *first_tasks[] = { NULL, NULL };
	const char *then_tasks[] = { NULL, NULL };

	/* Actions to make first_rsc lose first_role */
	if (first_role == RSC_ROLE_PROMOTED) {
	first_tasks[0] = CRMD_ACTION_DEMOTE;

	} else {
	first_tasks[0] = CRMD_ACTION_STOP;

	if (first_role == RSC_ROLE_UNPROMOTED) {
	first_tasks[1] = CRMD_ACTION_PROMOTE;
	}
	}

	/* Actions to make then_rsc gain then_role */
	if (then_role == RSC_ROLE_PROMOTED) {
	then_tasks[0] = CRMD_ACTION_PROMOTE;

	} else {
	then_tasks[0] = CRMD_ACTION_START;

	if (then_role == RSC_ROLE_UNPROMOTED) {
	then_tasks[1] = CRMD_ACTION_DEMOTE;
	}
	}

	for (int first_lpc = 0;
	(first_lpc <= 1) && (first_tasks[first_lpc] != NULL); first_lpc++) {

	for (int then_lpc = 0;
	(then_lpc <= 1) && (then_tasks[then_lpc] != NULL); then_lpc++) {

	pcmk__order_resource_actions(first_rsc, first_tasks[first_lpc],
	then_rsc, then_tasks[then_lpc],
	pe_order_anti_colocation, data_set);
	}
	}
	}

	/*!
	* \internal
	* \brief Add a new colocation constraint to a cluster working set
	*
	* \param[in] id XML ID for this constraint
	* \param[in] node_attr Colocate by this attribute (or NULL for #uname)
	* \param[in] score Constraint score
	* \param[in] dependent Resource to be colocated
	* \param[in] primary Resource to colocate \p dependent with
	* \param[in] dependent_role Current role of \p dependent
	* \param[in] primary_role Current role of \p primary
	* \param[in] influence Whether colocation constraint has influence
	* \param[in] data_set Cluster working set to add constraint to
	*/
	void
	pcmk__new_colocation(const char id, const char node_attr, int score,
	pe_resource_t dependent, pe_resource_t primary,
	const char dependent_role, const char primary_role,
	bool influence, pe_working_set_t *data_set)
	{
	pcmk__colocation_t *new_con = NULL;

	if (score == 0) {
	crm_trace("Ignoring colocation '%s' because score is 0", id);
	return;
	}
	if ((dependent == NULL) \|\| (primary == NULL)) {
	pcmk__config_err("Ignoring colocation '%s' because resource "
	"does not exist", id);
	return;
	}

	new_con = calloc(1, sizeof(pcmk__colocation_t));
	if (new_con == NULL) {
	return;
	}

	if (pcmk__str_eq(dependent_role, RSC_ROLE_STARTED_S,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	dependent_role = RSC_ROLE_UNKNOWN_S;
	}

	if (pcmk__str_eq(primary_role, RSC_ROLE_STARTED_S,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	primary_role = RSC_ROLE_UNKNOWN_S;
	}

	new_con->id = id;
	new_con->dependent = dependent;
	new_con->primary = primary;
	new_con->score = score;
	new_con->dependent_role = text2role(dependent_role);
	new_con->primary_role = text2role(primary_role);
	new_con->node_attribute = node_attr;
	new_con->influence = influence;

	if (node_attr == NULL) {
	node_attr = CRM_ATTR_UNAME;
	}

	pe_rsc_trace(dependent, "%s ==> %s (%s %d)",
	dependent->id, primary->id, node_attr, score);

	dependent->rsc_cons = g_list_insert_sorted(dependent->rsc_cons, new_con,
	cmp_primary_priority);

	primary->rsc_cons_lhs = g_list_insert_sorted(primary->rsc_cons_lhs, new_con,
	cmp_dependent_priority);

	data_set->colocation_constraints = g_list_append(data_set->colocation_constraints,
	new_con);

	if (score <= -INFINITY) {
	anti_colocation_order(dependent, new_con->dependent_role, primary,
	new_con->primary_role, data_set);
	anti_colocation_order(primary, new_con->primary_role, dependent,
	new_con->dependent_role, data_set);
	}
	}

	/*!
	* \internal
	* \brief Return the boolean influence corresponding to configuration
	*
	* \param[in] coloc_id Colocation XML ID (for error logging)
	* \param[in] rsc Resource involved in constraint (for default)
	* \param[in] influence_s String value of influence option
	*
	* \return true if string evaluates true, false if string evaluates false,
	* or value of resource's critical option if string is NULL or invalid
	*/
	static bool
	unpack_influence(const char coloc_id, const pe_resource_t rsc,
	const char *influence_s)
	{
	if (influence_s != NULL) {
	int influence_i = 0;

	if (crm_str_to_boolean(influence_s, &influence_i) < 0) {
	pcmk__config_err("Constraint '%s' has invalid value for "
	XML_COLOC_ATTR_INFLUENCE " (using default)",
	coloc_id);
	} else {
	return (influence_i != 0);
	}
	}
	return pcmk_is_set(rsc->flags, pe_rsc_critical);
	}

	static void
	unpack_colocation_set(xmlNode set, int score, const char coloc_id,
	const char influence_s, pe_working_set_t data_set)
	{
	xmlNode *xml_rsc = NULL;
	pe_resource_t *with = NULL;
	pe_resource_t *resource = NULL;
	const char *set_id = ID(set);
	const char *role = crm_element_value(set, "role");
	const char *ordering = crm_element_value(set, "ordering");
	int local_score = score;
	bool sequential = false;

	const char *score_s = crm_element_value(set, XML_RULE_ATTR_SCORE);

	if (score_s) {
	local_score = char2score(score_s);
	}
	if (local_score == 0) {
	crm_trace("Ignoring colocation '%s' for set '%s' because score is 0",
	coloc_id, set_id);
	return;
	}

	if (ordering == NULL) {
	ordering = "group";
	}

	if (pcmk__xe_get_bool_attr(set, "sequential", &sequential) == pcmk_rc_ok && !sequential) {
	return;

	} else if ((local_score > 0)
	&& pcmk__str_eq(ordering, "group", pcmk__str_casei)) {
	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	if (with != NULL) {
	pe_rsc_trace(resource, "Colocating %s with %s", resource->id, with->id);
	pcmk__new_colocation(set_id, NULL, local_score, resource,
	with, role, role,
	unpack_influence(coloc_id, resource,
	influence_s), data_set);
	}
	with = resource;
	}

	} else if (local_score > 0) {
	pe_resource_t *last = NULL;

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	if (last != NULL) {
	pe_rsc_trace(resource, "Colocating %s with %s",
	last->id, resource->id);
	pcmk__new_colocation(set_id, NULL, local_score, last,
	resource, role, role,
	unpack_influence(coloc_id, last,
	influence_s), data_set);
	}

	last = resource;
	}

	} else {
	/* Anti-colocating with every prior resource is
	* the only way to ensure the intuitive result
	* (i.e. that no one in the set can run with anyone else in the set)
	*/

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xmlNode *xml_rsc_with = NULL;
	bool influence = true;

	EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	influence = unpack_influence(coloc_id, resource, influence_s);

	for (xml_rsc_with = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc_with != NULL;
	xml_rsc_with = crm_next_same_xml(xml_rsc_with)) {

	if (pcmk__str_eq(resource->id, ID(xml_rsc_with),
	pcmk__str_casei)) {
	break;
	}
	EXPAND_CONSTRAINT_IDREF(set_id, with, ID(xml_rsc_with));
	pe_rsc_trace(resource, "Anti-Colocating %s with %s", resource->id,
	with->id);
	pcmk__new_colocation(set_id, NULL, local_score,
	resource, with, role, role,
	influence, data_set);
	}
	}
	}
	}

	static void
	colocate_rsc_sets(const char id, xmlNode set1, xmlNode *set2, int score,
	const char influence_s, pe_working_set_t data_set)
	{
	xmlNode *xml_rsc = NULL;
	pe_resource_t *rsc_1 = NULL;
	pe_resource_t *rsc_2 = NULL;

	const char *role_1 = crm_element_value(set1, "role");
	const char *role_2 = crm_element_value(set2, "role");

	int rc = pcmk_rc_ok;
	bool sequential = false;

	if (score == 0) {
	crm_trace("Ignoring colocation '%s' between sets because score is 0",
	id);
	return;
	}

	rc = pcmk__xe_get_bool_attr(set1, "sequential", &sequential);
	if (rc != pcmk_rc_ok \|\| sequential) {
	// Get the first one
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	}
	}

	rc = pcmk__xe_get_bool_attr(set2, "sequential", &sequential);
	if (rc != pcmk_rc_ok \|\| sequential) {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2,
	unpack_influence(id, rsc_1, influence_s),
	data_set);

	} else if (rsc_1 != NULL) {
	bool influence = unpack_influence(id, rsc_1, influence_s);

	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	role_2, influence, data_set);
	}

	} else if (rsc_2 != NULL) {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	role_2,
	unpack_influence(id, rsc_1, influence_s),
	data_set);
	}

	} else {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xmlNode *xml_rsc_2 = NULL;
	bool influence = true;

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	influence = unpack_influence(id, rsc_1, influence_s);

	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL;
	xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2,
	role_1, role_2, influence,
	data_set);
	}
	}
	}
	}

	static void
	unpack_simple_colocation(xmlNode xml_obj, const char id,
	const char influence_s, pe_working_set_t data_set)
	{
	int score_i = 0;

	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	const char *dependent_id = crm_element_value(xml_obj,
	XML_COLOC_ATTR_SOURCE);
	const char *primary_id = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	const char *dependent_role = crm_element_value(xml_obj,
	XML_COLOC_ATTR_SOURCE_ROLE);
	const char *primary_role = crm_element_value(xml_obj,
	XML_COLOC_ATTR_TARGET_ROLE);
	const char *attr = crm_element_value(xml_obj, XML_COLOC_ATTR_NODE_ATTR);

	// experimental syntax from pacemaker-next (unlikely to be adopted as-is)
	const char *dependent_instance = crm_element_value(xml_obj,
	XML_COLOC_ATTR_SOURCE_INSTANCE);
	const char *primary_instance = crm_element_value(xml_obj,
	XML_COLOC_ATTR_TARGET_INSTANCE);

	pe_resource_t *dependent = pcmk__find_constraint_resource(data_set->resources,
	dependent_id);
	pe_resource_t *primary = pcmk__find_constraint_resource(data_set->resources,
	primary_id);

	if (dependent == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, dependent_id);
	return;

	} else if (primary == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, primary_id);
	return;

	} else if ((dependent_instance != NULL) && !pe_rsc_is_clone(dependent)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	id, dependent_id, dependent_instance);
	return;

	} else if ((primary_instance != NULL) && !pe_rsc_is_clone(primary)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	id, primary_id, primary_instance);
	return;
	}

	if (dependent_instance != NULL) {
	dependent = find_clone_instance(dependent, dependent_instance, data_set);
	if (dependent == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	id, dependent_id, dependent_instance);
	return;
	}
	}

	if (primary_instance != NULL) {
	primary = find_clone_instance(primary, primary_instance, data_set);
	if (primary == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", id, primary_id, primary_instance);
	return;
	}
	}

	if (pcmk__xe_attr_is_true(xml_obj, XML_CONS_ATTR_SYMMETRICAL)) {
	pcmk__config_warn("The colocation constraint '"
	XML_CONS_ATTR_SYMMETRICAL
	"' attribute has been removed");
	}

	if (score) {
	score_i = char2score(score);
	}

	pcmk__new_colocation(id, attr, score_i, dependent, primary,
	dependent_role, primary_role,
	unpack_influence(id, dependent, influence_s), data_set);
	}

	// \return Standard Pacemaker return code
	static int
	unpack_colocation_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	pe_working_set_t *data_set)
	{
	const char *id = NULL;
	const char *dependent_id = NULL;
	const char *primary_id = NULL;
	const char *dependent_role = NULL;
	const char *primary_role = NULL;

	pe_resource_t *dependent = NULL;
	pe_resource_t *primary = NULL;

	pe_tag_t *dependent_tag = NULL;
	pe_tag_t *primary_tag = NULL;

	xmlNode *dependent_set = NULL;
	xmlNode *primary_set = NULL;
	bool any_sets = false;

	*expanded_xml = NULL;

	CRM_CHECK(xml_obj != NULL, return pcmk_rc_schema_validation);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return pcmk_rc_schema_validation;
	}

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	return pcmk_rc_ok;
	}

	dependent_id = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	primary_id = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	if ((dependent_id == NULL) \|\| (primary_id == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(data_set, dependent_id, &dependent,
	&dependent_tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, dependent_id);
	return pcmk_rc_schema_validation;
	}

	if (!pcmk__valid_resource_or_tag(data_set, primary_id, &primary,
	&primary_tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, primary_id);
	return pcmk_rc_schema_validation;
	}

	if ((dependent != NULL) && (primary != NULL)) {
	/* Neither side references any template/tag. */
	return pcmk_rc_ok;
	}

	if ((dependent_tag != NULL) && (primary_tag != NULL)) {
	// A colocation constraint between two templates/tags makes no sense
	pcmk__config_err("Ignoring constraint '%s' because two templates or "
	"tags cannot be colocated", id);
	return pcmk_rc_schema_validation;
	}

	dependent_role = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
	primary_role = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);

	*expanded_xml = copy_xml(xml_obj);

	// Convert template/tag reference in "rsc" into resource_set under constraint
	if (!pcmk__tag_to_set(*expanded_xml, &dependent_set, XML_COLOC_ATTR_SOURCE,
	true, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_schema_validation;
	}

	if (dependent_set != NULL) {
	if (dependent_role != NULL) {
	// Move "rsc-role" into converted resource_set as "role"
	crm_xml_add(dependent_set, "role", dependent_role);
	xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_SOURCE_ROLE);
	}
	any_sets = true;
	}

	// Convert template/tag reference in "with-rsc" into resource_set under constraint
	if (!pcmk__tag_to_set(*expanded_xml, &primary_set, XML_COLOC_ATTR_TARGET,
	true, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_schema_validation;
	}

	if (primary_set != NULL) {
	if (primary_role != NULL) {
	// Move "with-rsc-role" into converted resource_set as "role"
	crm_xml_add(primary_set, "role", primary_role);
	xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_TARGET_ROLE);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Parse a colocation constraint from XML into a cluster working set
	*
	* \param[in] xml_obj Colocation constraint XML to unpack
	* \param[in] data_set Cluster working set to add constraint to
	*/
	void
	pcmk__unpack_colocation(xmlNode xml_obj, pe_working_set_t data_set)
	{
	int score_i = 0;
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	const char *influence_s = crm_element_value(xml_obj,
	XML_COLOC_ATTR_INFLUENCE);

	if (score) {
	score_i = char2score(score);
	}

	if (unpack_colocation_tags(xml_obj, &expanded_xml,
	data_set) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	set = crm_next_same_xml(set)) {

	set = expand_idref(set, data_set->input);
	if (set == NULL) { // Configuration error, message already logged
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	unpack_colocation_set(set, score_i, id, influence_s, data_set);

	if (last != NULL) {
	colocate_rsc_sets(id, last, set, score_i, influence_s, data_set);
	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	if (last == NULL) {
	unpack_simple_colocation(xml_obj, id, influence_s, data_set);
	}
	}

	static void
	mark_start_blocked(pe_resource_t rsc, pe_resource_t reason,
	pe_working_set_t *data_set)
	{
	char *reason_text = crm_strdup_printf("colocation with %s", reason->id);

	for (GList *gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (pcmk_is_set(action->flags, pe_action_runnable)
	&& pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {

	pe__clear_action_flags(action, pe_action_runnable);
	pe_action_set_reason(action, reason_text, false);
	pcmk__block_colocated_starts(action, data_set);
	pcmk__update_action_for_orderings(action, data_set);
	}
	}
	free(reason_text);
	}

	/*!
	* \internal
	* \brief If a start action is unrunnable, block starts of colocated resources
	*
	* \param[in] action Action to check
	* \param[in] data_set Cluster working set
	*/
	void
	pcmk__block_colocated_starts(pe_action_t action, pe_working_set_t data_set)
	{
	GList *gIter = NULL;
	pe_resource_t *rsc = NULL;

	if (!pcmk_is_set(action->flags, pe_action_runnable)
	&& pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {

	rsc = uber_parent(action->rsc);
	if (rsc->parent) {
	/* For bundles, uber_parent() returns the clone, not the bundle, so
	* the existence of rsc->parent implies this is a bundle.
	* In this case, we need the bundle resource, so that we can check
	* if all containers are stopped/stopping.
	*/
	rsc = rsc->parent;
	}
	}

	if ((rsc == NULL) \|\| (rsc->rsc_cons_lhs == NULL)) {
	return;
	}

	// Block colocated starts only if all children (if any) have unrunnable starts
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t )gIter->data;
	pe_action_t *start = find_first_action(child->actions, NULL, RSC_START, NULL);

	if ((start == NULL) \|\| pcmk_is_set(start->flags, pe_action_runnable)) {
	return;
	}
	}

	for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t colocate_with = (pcmk__colocation_t ) gIter->data;

	if (colocate_with->score == INFINITY) {
	mark_start_blocked(colocate_with->dependent, action->rsc, data_set);
	}
	}
	}

	/*!
	* \internal
	* \brief Determine how a colocation constraint should affect a resource
	*
	* Colocation constraints have different effects at different points in the
	* scheduler sequence. Initially, they affect a resource's location; once that
	* is determined, then for promotable clones they can affect a resource
	* instance's role; after both are determined, the constraints no longer matter.
	* Given a specific colocation constraint, check what has been done so far to
	* determine what should be affected at the current point in the scheduler.
	*
	* \param[in] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] constraint Colocation constraint
	* \param[in] preview If true, pretend resources have already been allocated
	*
	* \return How colocation constraint should be applied at this point
	*/
	enum pcmk__coloc_affects
	pcmk__colocation_affects(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint, bool preview)
	{
	if (!preview && pcmk_is_set(primary->flags, pe_rsc_provisional)) {
	// Primary resource has not been allocated yet, so we can't do anything
	return pcmk__coloc_affects_nothing;
	}

	if ((constraint->dependent_role >= RSC_ROLE_UNPROMOTED)
	&& (dependent->parent != NULL)
	&& pcmk_is_set(dependent->parent->flags, pe_rsc_promotable)
	&& !pcmk_is_set(dependent->flags, pe_rsc_provisional)) {

	/* This is a colocation by role, and the dependent is a promotable clone
	* that has already been allocated, so the colocation should now affect
	* the role.
	*/
	return pcmk__coloc_affects_role;
	}

	if (!preview && !pcmk_is_set(dependent->flags, pe_rsc_provisional)) {
	/* The dependent resource has already been through allocation, so the
	* constraint no longer has any effect. Log an error if a mandatory
	* colocation constraint has been violated.
	*/

	const pe_node_t *primary_node = primary->allocated_to;

	if (dependent->allocated_to == NULL) {
	crm_trace("Skipping colocation '%s': %s will not run anywhere",
	constraint->id, dependent->id);

	} else if (constraint->score >= INFINITY) {
	// Dependent resource must colocate with primary resource

	if ((primary_node == NULL) \|\|
	(primary_node->details != dependent->allocated_to->details)) {
	crm_err("%s must be colocated with %s but is not (%s vs. %s)",
	dependent->id, primary->id,
	dependent->allocated_to->details->uname,
	(primary_node == NULL)? "unallocated" : primary_node->details->uname);
	}

	} else if (constraint->score <= -CRM_SCORE_INFINITY) {
	// Dependent resource must anti-colocate with primary resource

	if ((primary_node != NULL) &&
	(dependent->allocated_to->details == primary_node->details)) {
	crm_err("%s and %s must be anti-colocated but are allocated "
	"to the same node (%s)",
	dependent->id, primary->id, primary_node->details->uname);
	}
	}
	return pcmk__coloc_affects_nothing;
	}

	if ((constraint->score > 0)
	&& (constraint->dependent_role != RSC_ROLE_UNKNOWN)
	&& (constraint->dependent_role != dependent->next_role)) {

	crm_trace("Skipping colocation '%s': dependent limited to %s role "
	"but %s next role is %s",
	constraint->id, role2text(constraint->dependent_role),
	dependent->id, role2text(dependent->next_role));
	return pcmk__coloc_affects_nothing;
	}

	if ((constraint->score > 0)
	&& (constraint->primary_role != RSC_ROLE_UNKNOWN)
	&& (constraint->primary_role != primary->next_role)) {

	crm_trace("Skipping colocation '%s': primary limited to %s role "
	"but %s next role is %s",
	constraint->id, role2text(constraint->primary_role),
	primary->id, role2text(primary->next_role));
	return pcmk__coloc_affects_nothing;
	}

	if ((constraint->score < 0)
	&& (constraint->dependent_role != RSC_ROLE_UNKNOWN)
	&& (constraint->dependent_role == dependent->next_role)) {
	crm_trace("Skipping anti-colocation '%s': dependent role %s matches",
	constraint->id, role2text(constraint->dependent_role));
	return pcmk__coloc_affects_nothing;
	}

	if ((constraint->score < 0)
	&& (constraint->primary_role != RSC_ROLE_UNKNOWN)
	&& (constraint->primary_role == primary->next_role)) {
	crm_trace("Skipping anti-colocation '%s': primary role %s matches",
	constraint->id, role2text(constraint->primary_role));
	return pcmk__coloc_affects_nothing;
	}

	return pcmk__coloc_affects_location;
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent for allocation purposes
	*
	* Update the allocated node weights of the dependent resource in a colocation,
	* for the purposes of allocating it to a node
	*
	* \param[in] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] constraint Colocation constraint
	*/
	void
	pcmk__apply_coloc_to_weights(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint)
	{
	const char *attribute = CRM_ATTR_ID;
	const char *value = NULL;
	GHashTable *work = NULL;
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	if (primary->allocated_to != NULL) {
	value = pe_node_attribute_raw(primary->allocated_to, attribute);

	} else if (constraint->score < 0) {
	// Nothing to do (anti-colocation with something that is not running)
	return;
	}

	work = pcmk__copy_node_table(dependent->allowed_nodes);

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (primary->allocated_to == NULL) {
	pe_rsc_trace(dependent, "%s: %s@%s -= %d (%s inactive)",
	constraint->id, dependent->id, node->details->uname,
	constraint->score, primary->id);
	- node->weight = pe__add_scores(-constraint->score, node->weight);
	+ node->weight = pcmk__add_scores(-constraint->score, node->weight);

	} else if (pcmk__str_eq(pe_node_attribute_raw(node, attribute), value,
	pcmk__str_casei)) {
	if (constraint->score < CRM_SCORE_INFINITY) {
	pe_rsc_trace(dependent, "%s: %s@%s += %d",
	constraint->id, dependent->id,
	node->details->uname, constraint->score);
	- node->weight = pe__add_scores(constraint->score, node->weight);
	+ node->weight = pcmk__add_scores(constraint->score,
	+ node->weight);
	}

	} else if (constraint->score >= CRM_SCORE_INFINITY) {
	pe_rsc_trace(dependent, "%s: %s@%s -= %d (%s mismatch)",
	constraint->id, dependent->id, node->details->uname,
	constraint->score, attribute);
	- node->weight = pe__add_scores(-constraint->score, node->weight);
	+ node->weight = pcmk__add_scores(-constraint->score, node->weight);
	}
	}

	if ((constraint->score <= -INFINITY) \|\| (constraint->score >= INFINITY)
	\|\| pcmk__any_node_available(work)) {

	g_hash_table_destroy(dependent->allowed_nodes);
	dependent->allowed_nodes = work;
	work = NULL;

	} else {
	pe_rsc_info(dependent,
	"%s: Rolling back scores from %s (no available nodes)",
	dependent->id, primary->id);
	}

	if (work != NULL) {
	g_hash_table_destroy(work);
	}
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent for role purposes
	*
	* Update the priority of the dependent resource in a colocation, for the
	* purposes of selecting its role
	*
	* \param[in] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] constraint Colocation constraint
	*/
	void
	pcmk__apply_coloc_to_priority(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint)
	{
	const char *dependent_value = NULL;
	const char *primary_value = NULL;
	const char *attribute = CRM_ATTR_ID;
	int score_multiplier = 1;

	if ((primary->allocated_to == NULL) \|\| (dependent->allocated_to == NULL)) {
	return;
	}

	if (constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	dependent_value = pe_node_attribute_raw(dependent->allocated_to, attribute);
	primary_value = pe_node_attribute_raw(primary->allocated_to, attribute);

	if (!pcmk__str_eq(dependent_value, primary_value, pcmk__str_casei)) {
	if ((constraint->score == INFINITY)
	&& (constraint->dependent_role == RSC_ROLE_PROMOTED)) {
	dependent->priority = -INFINITY;
	}
	return;
	}

	if ((constraint->primary_role != RSC_ROLE_UNKNOWN)
	&& (constraint->primary_role != primary->next_role)) {
	return;
	}

	if (constraint->dependent_role == RSC_ROLE_UNPROMOTED) {
	score_multiplier = -1;
	}

	- dependent->priority = pe__add_scores(score_multiplier * constraint->score,
	- dependent->priority);
	+ dependent->priority = pcmk__add_scores(score_multiplier * constraint->score,
	+ dependent->priority);
	}
	diff --git a/lib/pacemaker/pcmk_sched_location.c b/lib/pacemaker/pcmk_sched_location.c
	index aa4d05bdfa..c3e6e093c7 100644
	--- a/lib/pacemaker/pcmk_sched_location.c
	+++ b/lib/pacemaker/pcmk_sched_location.c
	@@ -1,672 +1,672 @@
	/*
	- * Copyright 2004-2021 the Pacemaker project contributors
	+ * Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	static int
	get_node_score(const char rule, const char score, bool raw,
	pe_node_t node, pe_resource_t rsc)
	{
	int score_f = 0;

	if (score == NULL) {
	pe_err("Rule %s: no score specified. Assuming 0.", rule);

	} else if (raw) {
	score_f = char2score(score);

	} else {
	const char *attr_score = pe_node_attribute_calculated(node, score, rsc);

	if (attr_score == NULL) {
	crm_debug("Rule %s: node %s did not have a value for %s",
	rule, node->details->uname, score);
	score_f = -INFINITY;

	} else {
	crm_debug("Rule %s: node %s had value %s for %s",
	rule, node->details->uname, attr_score, score);
	score_f = char2score(attr_score);
	}
	}
	return score_f;
	}

	static pe__location_t *
	generate_location_rule(pe_resource_t rsc, xmlNode rule_xml,
	const char discovery, crm_time_t next_change,
	pe_working_set_t *data_set,
	pe_re_match_data_t *re_match_data)
	{
	const char *rule_id = NULL;
	const char *score = NULL;
	const char *boolean = NULL;
	const char *role = NULL;

	GList *gIter = NULL;
	GList *match_L = NULL;

	bool do_and = true;
	bool accept = true;
	bool raw_score = true;
	bool score_allocated = false;

	pe__location_t *location_rule = NULL;

	rule_xml = expand_idref(rule_xml, data_set->input);
	if (rule_xml == NULL) {
	return NULL;
	}

	rule_id = crm_element_value(rule_xml, XML_ATTR_ID);
	boolean = crm_element_value(rule_xml, XML_RULE_ATTR_BOOLEAN_OP);
	role = crm_element_value(rule_xml, XML_RULE_ATTR_ROLE);

	crm_trace("Processing rule: %s", rule_id);

	if ((role != NULL) && (text2role(role) == RSC_ROLE_UNKNOWN)) {
	pe_err("Bad role specified for %s: %s", rule_id, role);
	return NULL;
	}

	score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE);
	if (score == NULL) {
	score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE_ATTRIBUTE);
	if (score != NULL) {
	raw_score = false;
	}
	}
	if (pcmk__str_eq(boolean, "or", pcmk__str_casei)) {
	do_and = false;
	}

	location_rule = pcmk__new_location(rule_id, rsc, 0, discovery, NULL,
	data_set);

	if (location_rule == NULL) {
	return NULL;
	}

	if ((re_match_data != NULL) && (re_match_data->nregs > 0)
	&& (re_match_data->pmatch[0].rm_so != -1) && !raw_score) {

	char *result = pe_expand_re_matches(score, re_match_data);

	if (result != NULL) {
	score = result;
	score_allocated = true;
	}
	}

	if (role != NULL) {
	crm_trace("Setting role filter: %s", role);
	location_rule->role_filter = text2role(role);
	if (location_rule->role_filter == RSC_ROLE_UNPROMOTED) {
	/* Any promotable clone cannot be promoted without being in the
	* unpromoted role first. Ergo, any constraint for the unpromoted
	* role applies to every role.
	*/
	location_rule->role_filter = RSC_ROLE_UNKNOWN;
	}
	}
	if (do_and) {
	GList *gIter = NULL;

	match_L = pcmk__copy_node_list(data_set->nodes, true);
	for (gIter = match_L; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	node->weight = get_node_score(rule_id, score, raw_score, node, rsc);
	}
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	int score_f = 0;
	pe_node_t node = (pe_node_t ) gIter->data;
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = pe_rsc_params(rsc, node, data_set),
	.meta = rsc->meta,
	};

	accept = pe_test_rule(rule_xml, node->details->attrs, RSC_ROLE_UNKNOWN,
	data_set->now, next_change, &match_data);

	crm_trace("Rule %s %s on %s", ID(rule_xml), accept? "passed" : "failed",
	node->details->uname);

	score_f = get_node_score(rule_id, score, raw_score, node, rsc);

	if (accept) {
	pe_node_t *local = pe_find_node_id(match_L, node->details->id);

	if ((local == NULL) && do_and) {
	continue;

	} else if (local == NULL) {
	local = pe__copy_node(node);
	match_L = g_list_append(match_L, local);
	}

	if (!do_and) {
	- local->weight = pe__add_scores(local->weight, score_f);
	+ local->weight = pcmk__add_scores(local->weight, score_f);
	}
	crm_trace("node %s now has weight %d",
	node->details->uname, local->weight);

	} else if (do_and && !accept) {
	// Remove it
	pe_node_t *delete = pe_find_node_id(match_L, node->details->id);

	if (delete != NULL) {
	match_L = g_list_remove(match_L, delete);
	crm_trace("node %s did not match", node->details->uname);
	}
	free(delete);
	}
	}

	if (score_allocated) {
	free((char *)score);
	}

	location_rule->node_list_rh = match_L;
	if (location_rule->node_list_rh == NULL) {
	crm_trace("No matching nodes for rule %s", rule_id);
	return NULL;
	}

	crm_trace("%s: %d nodes matched",
	rule_id, g_list_length(location_rule->node_list_rh));
	return location_rule;
	}

	static void
	unpack_rsc_location(xmlNode xml_obj, pe_resource_t rsc_lh, const char *role,
	const char score, pe_working_set_t data_set,
	pe_re_match_data_t *re_match_data)
	{
	pe__location_t *location = NULL;
	const char *id_lh = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *node = crm_element_value(xml_obj, XML_CIB_TAG_NODE);
	const char *discovery = crm_element_value(xml_obj, XML_LOCATION_ATTR_DISCOVERY);

	if (rsc_lh == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, id_lh);
	return;
	}

	if (score == NULL) {
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	}

	if ((node != NULL) && (score != NULL)) {
	int score_i = char2score(score);
	pe_node_t *match = pe_find_node(data_set->nodes, node);

	if (!match) {
	return;
	}
	location = pcmk__new_location(id, rsc_lh, score_i, discovery, match,
	data_set);

	} else {
	bool empty = true;
	crm_time_t *next_change = crm_time_new_undefined();

	/* This loop is logically parallel to pe_evaluate_rules(), except
	* instead of checking whether any rule is active, we set up location
	* constraints for each active rule.
	*/
	for (xmlNode *rule_xml = first_named_child(xml_obj, XML_TAG_RULE);
	rule_xml != NULL; rule_xml = crm_next_same_xml(rule_xml)) {
	empty = false;
	crm_trace("Unpacking %s/%s", id, ID(rule_xml));
	generate_location_rule(rsc_lh, rule_xml, discovery, next_change,
	data_set, re_match_data);
	}

	if (empty) {
	pcmk__config_err("Ignoring constraint '%s' because it contains "
	"no rules", id);
	}

	/* If there is a point in the future when the evaluation of a rule will
	* change, make sure the scheduler is re-run by that time.
	*/
	if (crm_time_is_defined(next_change)) {
	time_t t = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(t, data_set);
	}
	crm_time_free(next_change);
	return;
	}

	if (role == NULL) {
	role = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);
	}

	if ((location != NULL) && (role != NULL)) {
	if (text2role(role) == RSC_ROLE_UNKNOWN) {
	pe_err("Invalid constraint %s: Bad role %s", id, role);
	return;

	} else {
	enum rsc_role_e r = text2role(role);
	switch(r) {
	case RSC_ROLE_UNKNOWN:
	case RSC_ROLE_STARTED:
	case RSC_ROLE_UNPROMOTED:
	/* Applies to all */
	location->role_filter = RSC_ROLE_UNKNOWN;
	break;
	default:
	location->role_filter = r;
	break;
	}
	}
	}
	}

	static void
	unpack_simple_location(xmlNode xml_obj, pe_working_set_t data_set)
	{
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);

	if (value) {
	pe_resource_t *rsc_lh;

	rsc_lh = pcmk__find_constraint_resource(data_set->resources, value);
	unpack_rsc_location(xml_obj, rsc_lh, NULL, NULL, data_set, NULL);
	}

	value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE_PATTERN);
	if (value) {
	regex_t *r_patt = calloc(1, sizeof(regex_t));
	bool invert = false;
	GList *rIter = NULL;

	if (value[0] == '!') {
	value++;
	invert = true;
	}

	if (regcomp(r_patt, value, REG_EXTENDED)) {
	pcmk__config_err("Ignoring constraint '%s' because "
	XML_LOC_ATTR_SOURCE_PATTERN
	" has invalid value '%s'", id, value);
	regfree(r_patt);
	free(r_patt);
	return;
	}

	for (rIter = data_set->resources; rIter; rIter = rIter->next) {
	pe_resource_t *r = rIter->data;
	int nregs = 0;
	regmatch_t *pmatch = NULL;
	int status;

	if(r_patt->re_nsub > 0) {
	nregs = r_patt->re_nsub + 1;
	} else {
	nregs = 1;
	}
	pmatch = calloc(nregs, sizeof(regmatch_t));

	status = regexec(r_patt, r->id, nregs, pmatch, 0);

	if (!invert && (status == 0)) {
	pe_re_match_data_t re_match_data = {
	.string = r->id,
	.nregs = nregs,
	.pmatch = pmatch
	};

	crm_debug("'%s' matched '%s' for %s", r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, data_set,
	&re_match_data);

	} else if (invert && (status != 0)) {
	crm_debug("'%s' is an inverted match of '%s' for %s",
	r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, data_set, NULL);

	} else {
	crm_trace("'%s' does not match '%s' for %s", r->id, value, id);
	}

	free(pmatch);
	}

	regfree(r_patt);
	free(r_patt);
	}
	}

	// \return Standard Pacemaker return code
	static int
	unpack_location_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	pe_working_set_t *data_set)
	{
	const char *id = NULL;
	const char *id_lh = NULL;
	const char *state_lh = NULL;
	pe_resource_t *rsc_lh = NULL;
	pe_tag_t *tag_lh = NULL;
	xmlNode *rsc_set_lh = NULL;

	*expanded_xml = NULL;

	CRM_CHECK(xml_obj != NULL, return pcmk_rc_schema_validation);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return pcmk_rc_schema_validation;
	}

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_location");
	return pcmk_rc_ok;
	}

	id_lh = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
	if (id_lh == NULL) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, id_lh);
	return pcmk_rc_schema_validation;

	} else if (rsc_lh != NULL) {
	// No template is referenced
	return pcmk_rc_ok;
	}

	state_lh = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);

	*expanded_xml = copy_xml(xml_obj);

	// Convert template/tag reference in "rsc" into resource_set under constraint
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_lh, XML_LOC_ATTR_SOURCE,
	false, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_schema_validation;
	}

	if (rsc_set_lh != NULL) {
	if (state_lh != NULL) {
	// Move "rsc-role" into converted resource_set as "role" attribute
	crm_xml_add(rsc_set_lh, "role", state_lh);
	xml_remove_prop(*expanded_xml, XML_RULE_ATTR_ROLE);
	}
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_location");

	} else {
	// No sets
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	// \return Standard Pacemaker return code
	static int
	unpack_location_set(xmlNode location, xmlNode set, pe_working_set_t *data_set)
	{
	xmlNode *xml_rsc = NULL;
	pe_resource_t *resource = NULL;
	const char *set_id;
	const char *role;
	const char *local_score;

	CRM_CHECK(set != NULL, return pcmk_rc_schema_validation);

	set_id = ID(set);
	if (set_id == NULL) {
	pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_SET " without "
	XML_ATTR_ID " in constraint '%s'",
	crm_str(ID(location)));
	return pcmk_rc_schema_validation;
	}

	role = crm_element_value(set, "role");
	local_score = crm_element_value(set, XML_RULE_ATTR_SCORE);

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	resource = pcmk__find_constraint_resource(data_set->resources,
	ID(xml_rsc));
	if (resource == NULL) {
	pcmk__config_err("%s: No resource found for %s",
	set_id, ID(xml_rsc));
	return pcmk_rc_schema_validation;
	}

	unpack_rsc_location(location, resource, role, local_score, data_set,
	NULL);
	}

	return pcmk_rc_ok;
	}

	void
	pcmk__unpack_location(xmlNode xml_obj, pe_working_set_t data_set)
	{
	xmlNode *set = NULL;
	bool any_sets = false;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	if (unpack_location_tags(xml_obj, &expanded_xml, data_set) != pcmk_rc_ok) {
	return;
	}

	if (expanded_xml) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	set = crm_next_same_xml(set)) {

	any_sets = true;
	set = expand_idref(set, data_set->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_location_set(xml_obj, set, data_set) != pcmk_rc_ok)) {

	if (expanded_xml) {
	free_xml(expanded_xml);
	}
	return;
	}
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	if (!any_sets) {
	unpack_simple_location(xml_obj, data_set);
	}
	}

	/*!
	* \internal
	* \brief Add a new location constraint to a cluster working set
	*
	* \param[in] id XML ID of location constraint
	* \param[in] rsc Resource in location constraint
	* \param[in] node_weight Constraint score
	* \param[in] discover_mode Resource discovery option for constraint
	* \param[in] node Node in location constraint (or NULL if rule-based)
	* \param[in] data_set Cluster working set to add constraint to
	*
	* \return Newly allocated location constraint
	* \note The result will be added to \p data_set and should not be freed
	* separately.
	*/
	pe__location_t *
	pcmk__new_location(const char id, pe_resource_t rsc,
	int node_weight, const char *discover_mode,
	pe_node_t node, pe_working_set_t data_set)
	{
	pe__location_t *new_con = NULL;

	if ((rsc == NULL) \|\| (id == NULL)) {
	pe_err("Invalid constraint %s for rsc=%p", crm_str(id), rsc);
	return NULL;

	} else if (node == NULL) {
	CRM_CHECK(node_weight == 0, return NULL);
	}

	new_con = calloc(1, sizeof(pe__location_t));
	if (new_con != NULL) {
	new_con->id = strdup(id);
	new_con->rsc_lh = rsc;
	new_con->node_list_rh = NULL;
	new_con->role_filter = RSC_ROLE_UNKNOWN;

	if (pcmk__str_eq(discover_mode, "always",
	pcmk__str_null_matches\|pcmk__str_casei)) {
	new_con->discover_mode = pe_discover_always;

	} else if (pcmk__str_eq(discover_mode, "never", pcmk__str_casei)) {
	new_con->discover_mode = pe_discover_never;

	} else if (pcmk__str_eq(discover_mode, "exclusive", pcmk__str_casei)) {
	new_con->discover_mode = pe_discover_exclusive;
	rsc->exclusive_discover = TRUE;

	} else {
	pe_err("Invalid " XML_LOCATION_ATTR_DISCOVERY " value %s "
	"in location constraint", discover_mode);
	}

	if (node != NULL) {
	pe_node_t *copy = pe__copy_node(node);

	copy->weight = node_weight;
	new_con->node_list_rh = g_list_prepend(NULL, copy);
	}

	data_set->placement_constraints = g_list_prepend(data_set->placement_constraints,
	new_con);
	rsc->rsc_location = g_list_prepend(rsc->rsc_location, new_con);
	}

	return new_con;
	}

	/*!
	* \internal
	* \brief Apply all location constraints
	*
	* \param[in] data_set Cluster working set
	*/
	void
	pcmk__apply_locations(pe_working_set_t *data_set)
	{
	for (GList *iter = data_set->placement_constraints;
	iter != NULL; iter = iter->next) {
	pe__location_t *location = iter->data;

	location->rsc_lh->cmds->rsc_location(location->rsc_lh, location);
	}
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a resource's allowed node weights
	*
	* \param[in] constraint Location constraint to apply
	* \param[in] rsc Resource to apply constraint to
	*/
	void
	pcmk__apply_location(pe__location_t constraint, pe_resource_t rsc)
	{
	bool need_role = false;

	CRM_CHECK((constraint != NULL) && (rsc != NULL), return);

	// If a role was specified, ensure constraint is applicable
	need_role = (constraint->role_filter > RSC_ROLE_UNKNOWN);
	if (need_role && (constraint->role_filter != rsc->next_role)) {
	pe_rsc_trace(rsc,
	"Not applying %s to %s because role will be %s not %s",
	constraint->id, rsc->id, role2text(rsc->next_role),
	role2text(constraint->role_filter));
	return;
	}

	if (constraint->node_list_rh == NULL) {
	pe_rsc_trace(rsc, "Not applying %s to %s because no nodes match",
	constraint->id, rsc->id);
	return;
	}

	pe_rsc_trace(rsc, "Applying %s%s%s to %s", constraint->id,
	(need_role? " for role " : ""),
	(need_role? role2text(constraint->role_filter) : ""), rsc->id);

	for (GList *gIter = constraint->node_list_rh; gIter != NULL;
	gIter = gIter->next) {

	pe_node_t node = (pe_node_t ) gIter->data;
	pe_node_t *weighted_node = NULL;

	weighted_node = (pe_node_t *) pe_hash_table_lookup(rsc->allowed_nodes,
	node->details->id);
	if (weighted_node == NULL) {
	pe_rsc_trace(rsc, "* = %d on %s",
	node->weight, node->details->uname);
	weighted_node = pe__copy_node(node);
	g_hash_table_insert(rsc->allowed_nodes,
	(gpointer) weighted_node->details->id,
	weighted_node);
	} else {
	pe_rsc_trace(rsc, "* + %d on %s",
	node->weight, node->details->uname);
	- weighted_node->weight = pe__add_scores(weighted_node->weight,
	- node->weight);
	+ weighted_node->weight = pcmk__add_scores(weighted_node->weight,
	+ node->weight);
	}

	if (weighted_node->rsc_discover_mode < constraint->discover_mode) {
	if (constraint->discover_mode == pe_discover_exclusive) {
	rsc->exclusive_discover = TRUE;
	}
	/* exclusive > never > always... always is default */
	weighted_node->rsc_discover_mode = constraint->discover_mode;
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
	index 1a9ee44378..97d6dd8233 100644
	--- a/lib/pacemaker/pcmk_sched_native.c
	+++ b/lib/pacemaker/pcmk_sched_native.c
	@@ -1,2407 +1,2407 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/pengine/rules.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>
	#include <pacemaker-internal.h>
	#include <crm/services.h>

	#include "libpacemaker_private.h"

	// The controller removes the resource from the CIB, making this redundant
	// #define DELETE_THEN_REFRESH 1

	#define INFINITY_HACK (INFINITY * -100)

	#define VARIANT_NATIVE 1
	#include <lib/pengine/variant.h>

	extern bool pcmk__is_daemon;

	static void Recurring(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	pe_working_set_t *data_set);
	static void RecurringOp(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	xmlNode operation, pe_working_set_t data_set);
	static void Recurring_Stopped(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	pe_working_set_t *data_set);
	static void RecurringOp_Stopped(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	xmlNode operation, pe_working_set_t data_set);

	gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
	gboolean StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional,
	pe_working_set_t * data_set);
	gboolean RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);

	/* This array says what the next role should be when transitioning from one
	* role to another. For example going from Stopped to Promoted, the next role is
	* RSC_ROLE_UNPROMOTED, because the resource must be started before being promoted.
	* The current state then becomes Started, which is fed into this array again,
	* giving a next role of RSC_ROLE_PROMOTED.
	*/
	static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* Current state Next state*/
	/* Unknown Stopped Started Unpromoted Promoted */
	/* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED },
	/* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_UNPROMOTED },
	/* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_PROMOTED },
	/* Unpromoted */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_UNPROMOTED, RSC_ROLE_PROMOTED },
	/* Promoted */ { RSC_ROLE_STOPPED, RSC_ROLE_UNPROMOTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_PROMOTED },
	};

	typedef gboolean (rsc_transition_fn)(pe_resource_t rsc, pe_node_t *next,
	gboolean optional,
	pe_working_set_t *data_set);

	// This array picks the function needed to transition from one role to another
	static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* Current state Next state */
	/* Unknown Stopped Started Unpromoted Promoted */
	/* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
	/* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
	/* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
	/* Unpromoted */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, },
	/* Promoted */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp, },
	};

	#define clear_node_weights_flags(nw_flags, nw_rsc, flags_to_clear) do { \
	flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Node weight", (nw_rsc)->id, (flags), \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	static bool
	native_choose_node(pe_resource_t * rsc, pe_node_t * prefer, pe_working_set_t * data_set)
	{
	GList *nodes = NULL;
	pe_node_t *chosen = NULL;
	pe_node_t *best = NULL;
	int multiple = 1;
	int length = 0;
	bool result = false;

	pcmk__ban_insufficient_capacity(rsc, &prefer, data_set);

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to != NULL;
	}

	// Sort allowed nodes by weight
	if (rsc->allowed_nodes) {
	length = g_hash_table_size(rsc->allowed_nodes);
	}
	if (length > 0) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	nodes = pcmk__sort_nodes(nodes, pe__current_node(rsc), data_set);

	// First node in sorted list has the best score
	best = g_list_nth_data(nodes, 0);
	}

	if (prefer && nodes) {
	chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);

	if (chosen == NULL) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
	prefer->details->uname, rsc->id);

	/* Favor the preferred node as long as its weight is at least as good as
	* the best allowed node's.
	*
	* An alternative would be to favor the preferred node even if the best
	* node is better, when the best node's weight is less than INFINITY.
	*/
	} else if ((chosen->weight < 0) \|\| (chosen->weight < best->weight)) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
	chosen->details->uname, rsc->id);
	chosen = NULL;

	} else if (!pcmk__node_available(chosen)) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
	chosen->details->uname, rsc->id);
	chosen = NULL;

	} else {
	pe_rsc_trace(rsc,
	"Chose preferred node %s for %s (ignoring %d candidates)",
	chosen->details->uname, rsc->id, length);
	}
	}

	if ((chosen == NULL) && nodes) {
	/* Either there is no preferred node, or the preferred node is not
	* available, but there are other nodes allowed to run the resource.
	*/

	chosen = best;
	pe_rsc_trace(rsc, "Chose node %s for %s from %d candidates",
	chosen ? chosen->details->uname : "<none>", rsc->id, length);

	if (!pe_rsc_is_unique_clone(rsc->parent)
	&& chosen && (chosen->weight > 0) && pcmk__node_available(chosen)) {
	/* If the resource is already running on a node, prefer that node if
	* it is just as good as the chosen node.
	*
	* We don't do this for unique clone instances, because
	* distribute_children() has already assigned instances to their
	* running nodes when appropriate, and if we get here, we don't want
	* remaining unallocated instances to prefer a node that's already
	* running another instance.
	*/
	pe_node_t *running = pe__current_node(rsc);

	if ((running != NULL) && !pcmk__node_available(running)) {
	pe_rsc_trace(rsc, "Current node for %s (%s) can't run resources",
	rsc->id, running->details->uname);
	} else if (running) {
	for (GList *iter = nodes->next; iter; iter = iter->next) {
	pe_node_t tmp = (pe_node_t ) iter->data;

	if (tmp->weight != chosen->weight) {
	// The nodes are sorted by weight, so no more are equal
	break;
	}
	if (tmp->details == running->details) {
	// Scores are equal, so prefer the current node
	chosen = tmp;
	}
	multiple++;
	}
	}
	}
	}

	if (multiple > 1) {
	static char score[33];
	int log_level = (chosen->weight >= INFINITY)? LOG_WARNING : LOG_INFO;

	score2char_stack(chosen->weight, score, sizeof(score));
	do_crm_log(log_level,
	"Chose node %s for %s from %d nodes with score %s",
	chosen->details->uname, rsc->id, multiple, score);
	}

	result = pcmk__assign_primitive(rsc, chosen, false);
	g_list_free(nodes);
	return result;
	}

	/*!
	* \internal
	* \brief Find score of highest-scored node that matches colocation attribute
	*
	* \param[in] rsc Resource whose allowed nodes should be searched
	* \param[in] attr Colocation attribute name (must not be NULL)
	* \param[in] value Colocation attribute value to require
	*/
	static int
	best_node_score_matching_attr(const pe_resource_t rsc, const char attr,
	const char *value)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	int best_score = -INFINITY;
	const char *best_node = NULL;

	// Find best allowed node with matching attribute
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {

	if ((node->weight > best_score) && pcmk__node_available(node)
	&& pcmk__str_eq(value, pe_node_attribute_raw(node, attr), pcmk__str_casei)) {

	best_score = node->weight;
	best_node = node->details->uname;
	}
	}

	if (!pcmk__str_eq(attr, CRM_ATTR_UNAME, pcmk__str_casei)) {
	if (best_node == NULL) {
	crm_info("No allowed node for %s matches node attribute %s=%s",
	rsc->id, attr, value);
	} else {
	crm_info("Allowed node %s for %s had best score (%d) "
	"of those matching node attribute %s=%s",
	best_node, rsc->id, best_score, attr, value);
	}
	}
	return best_score;
	}

	/*!
	* \internal
	* \brief Add resource's colocation matches to current node allocation scores
	*
	* For each node in a given table, if any of a given resource's allowed nodes
	* have a matching value for the colocation attribute, add the highest of those
	* nodes' scores to the node's score.
	*
	* \param[in,out] nodes Hash table of nodes with allocation scores so far
	* \param[in] rsc Resource whose allowed nodes should be compared
	* \param[in] attr Colocation attribute that must match (NULL for default)
	* \param[in] factor Factor by which to multiply scores being added
	* \param[in] only_positive Whether to add only positive scores
	*/
	static void
	add_node_scores_matching_attr(GHashTable nodes, const pe_resource_t rsc,
	const char *attr, float factor,
	bool only_positive)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (attr == NULL) {
	attr = CRM_ATTR_UNAME;
	}

	// Iterate through each node
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	float weight_f = 0;
	int weight = 0;
	int score = 0;
	int new_score = 0;

	score = best_node_score_matching_attr(rsc, attr,
	pe_node_attribute_raw(node, attr));

	if ((factor < 0) && (score < 0)) {
	/* Negative preference for a node with a negative score
	* should not become a positive preference.
	*
	* @TODO Consider filtering only if weight is -INFINITY
	*/
	crm_trace("%s: Filtering %d + %f * %d (double negative disallowed)",
	node->details->uname, node->weight, factor, score);
	continue;
	}

	if (node->weight == INFINITY_HACK) {
	crm_trace("%s: Filtering %d + %f * %d (node was marked unusable)",
	node->details->uname, node->weight, factor, score);
	continue;
	}

	weight_f = factor * score;

	// Round the number; see http://c-faq.com/fp/round.html
	weight = (int) ((weight_f < 0)? (weight_f - 0.5) : (weight_f + 0.5));

	/* Small factors can obliterate the small scores that are often actually
	* used in configurations. If the score and factor are nonzero, ensure
	* that the result is nonzero as well.
	*/
	if ((weight == 0) && (score != 0)) {
	if (factor > 0.0) {
	weight = 1;
	} else if (factor < 0.0) {
	weight = -1;
	}
	}

	- new_score = pe__add_scores(weight, node->weight);
	+ new_score = pcmk__add_scores(weight, node->weight);

	if (only_positive && (new_score < 0) && (node->weight > 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d "
	"(negative disallowed, marking node unusable)",
	node->details->uname, node->weight, factor, score,
	new_score);
	node->weight = INFINITY_HACK;
	continue;
	}

	if (only_positive && (new_score < 0) && (node->weight == 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d (negative disallowed)",
	node->details->uname, node->weight, factor, score,
	new_score);
	continue;
	}

	crm_trace("%s: %d + %f * %d = %d", node->details->uname,
	node->weight, factor, score, new_score);
	node->weight = new_score;
	}
	}

	static inline bool
	is_nonempty_group(pe_resource_t *rsc)
	{
	return rsc && (rsc->variant == pe_group) && (rsc->children != NULL);
	}

	/*!
	* \internal
	* \brief Incorporate colocation constraint scores into node weights
	*
	* \param[in,out] rsc Resource being placed
	* \param[in] primary_id ID of primary resource in constraint
	* \param[in,out] nodes Nodes, with scores as of this point
	* \param[in] attr Colocation attribute (ID by default)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pe_weights values
	*
	* \return Nodes, with scores modified by this constraint
	* \note This function assumes ownership of the nodes argument. The caller
	* should free the returned copy rather than the original.
	*/
	GHashTable *
	pcmk__native_merge_weights(pe_resource_t rsc, const char primary_id,
	GHashTable nodes, const char attr, float factor,
	uint32_t flags)
	{
	GHashTable *work = NULL;

	// Avoid infinite recursion
	if (pcmk_is_set(rsc->flags, pe_rsc_merging)) {
	pe_rsc_info(rsc, "%s: Breaking dependency loop at %s",
	primary_id, rsc->id);
	return nodes;
	}
	pe__set_resource_flags(rsc, pe_rsc_merging);

	if (pcmk_is_set(flags, pe_weights_init)) {
	if (is_nonempty_group(rsc)) {
	GList *last = g_list_last(rsc->children);
	pe_resource_t *last_rsc = last->data;

	pe_rsc_trace(rsc, "%s: Merging scores from group %s "
	"using last member %s (at %.6f)",
	primary_id, rsc->id, last_rsc->id, factor);
	work = pcmk__native_merge_weights(last_rsc, primary_id, NULL, attr,
	factor, flags);
	} else {
	work = pcmk__copy_node_table(rsc->allowed_nodes);
	}
	clear_node_weights_flags(flags, rsc, pe_weights_init);

	} else if (is_nonempty_group(rsc)) {
	/* The first member of the group will recursively incorporate any
	* constraints involving other members (including the group internal
	* colocation).
	*
	* @TODO The indirect colocations from the dependent group's other
	* members will be incorporated at full strength rather than by
	* factor, so the group's combined stickiness will be treated as
	* (factor + (#members - 1)) * stickiness. It is questionable what
	* the right approach should be.
	*/
	pe_rsc_trace(rsc, "%s: Merging scores from first member of group %s "
	"(at %.6f)", primary_id, rsc->id, factor);
	work = pcmk__copy_node_table(nodes);
	work = pcmk__native_merge_weights(rsc->children->data, primary_id, work,
	attr, factor, flags);

	} else {
	pe_rsc_trace(rsc, "%s: Merging scores from %s (at %.6f)",
	primary_id, rsc->id, factor);
	work = pcmk__copy_node_table(nodes);
	add_node_scores_matching_attr(work, rsc, attr, factor,
	pcmk_is_set(flags, pe_weights_positive));
	}

	if (pcmk__any_node_available(work)) {
	GList *gIter = NULL;
	int multiplier = (factor < 0)? -1 : 1;

	if (pcmk_is_set(flags, pe_weights_forward)) {
	gIter = rsc->rsc_cons;
	pe_rsc_trace(rsc,
	"Checking additional %d optional '%s with' constraints",
	g_list_length(gIter), rsc->id);

	} else if (is_nonempty_group(rsc)) {
	pe_resource_t *last_rsc = g_list_last(rsc->children)->data;

	gIter = last_rsc->rsc_cons_lhs;
	pe_rsc_trace(rsc, "Checking additional %d optional 'with group %s' "
	"constraints using last member %s",
	g_list_length(gIter), rsc->id, last_rsc->id);

	} else {
	gIter = rsc->rsc_cons_lhs;
	pe_rsc_trace(rsc,
	"Checking additional %d optional 'with %s' constraints",
	g_list_length(gIter), rsc->id);
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *other = NULL;
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	if (pcmk_is_set(flags, pe_weights_forward)) {
	other = constraint->primary;
	} else if (!pcmk__colocation_has_influence(constraint, NULL)) {
	continue;
	} else {
	other = constraint->dependent;
	}

	pe_rsc_trace(rsc, "Optionally merging score of '%s' constraint (%s with %s)",
	constraint->id, constraint->dependent->id,
	constraint->primary->id);
	work = pcmk__native_merge_weights(other, primary_id, work,
	constraint->node_attribute,
	multiplier * constraint->score / (float) INFINITY,
	flags\|pe_weights_rollback);
	pe__show_node_weights(true, NULL, primary_id, work, rsc->cluster);
	}

	} else if (pcmk_is_set(flags, pe_weights_rollback)) {
	pe_rsc_info(rsc, "%s: Rolling back optional scores from %s",
	primary_id, rsc->id);
	g_hash_table_destroy(work);
	pe__clear_resource_flags(rsc, pe_rsc_merging);
	return nodes;
	}


	if (pcmk_is_set(flags, pe_weights_positive)) {
	pe_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->weight == INFINITY_HACK) {
	node->weight = 1;
	}
	}
	}

	if (nodes) {
	g_hash_table_destroy(nodes);
	}

	pe__clear_resource_flags(rsc, pe_rsc_merging);
	return work;
	}

	pe_node_t *
	pcmk__native_allocate(pe_resource_t rsc, pe_node_t prefer,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;

	if (rsc->parent && !pcmk_is_set(rsc->parent->flags, pe_rsc_allocating)) {
	/* never allocate children on their own */
	pe_rsc_debug(rsc, "Escalating allocation of %s to its parent: %s", rsc->id,
	rsc->parent->id);
	rsc->parent->cmds->allocate(rsc->parent, prefer, data_set);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	pe__set_resource_flags(rsc, pe_rsc_allocating);
	pe__show_node_weights(true, rsc, "Pre-alloc", rsc->allowed_nodes, data_set);

	for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	GHashTable *archive = NULL;
	pe_resource_t *primary = constraint->primary;

	if ((constraint->dependent_role >= RSC_ROLE_PROMOTED)
	\|\| (constraint->score < 0 && constraint->score > -INFINITY)) {
	archive = pcmk__copy_node_table(rsc->allowed_nodes);
	}

	pe_rsc_trace(rsc,
	"%s: Allocating %s first (constraint=%s score=%d role=%s)",
	rsc->id, primary->id, constraint->id,
	constraint->score, role2text(constraint->dependent_role));
	primary->cmds->allocate(primary, NULL, data_set);
	rsc->cmds->rsc_colocation_lh(rsc, primary, constraint, data_set);
	if (archive && !pcmk__any_node_available(rsc->allowed_nodes)) {
	pe_rsc_info(rsc, "%s: Rolling back scores from %s",
	rsc->id, primary->id);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive) {
	g_hash_table_destroy(archive);
	}
	}

	pe__show_node_weights(true, rsc, "Post-coloc", rsc->allowed_nodes, data_set);

	for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	if (!pcmk__colocation_has_influence(constraint, NULL)) {
	continue;
	}
	pe_rsc_trace(rsc, "Merging score of '%s' constraint (%s with %s)",
	constraint->id, constraint->dependent->id,
	constraint->primary->id);
	rsc->allowed_nodes = constraint->dependent->cmds->merge_weights(
	constraint->dependent, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute, constraint->score / (float) INFINITY,
	pe_weights_rollback);
	}

	if (rsc->next_role == RSC_ROLE_STOPPED) {
	pe_rsc_trace(rsc, "Making sure %s doesn't get allocated", rsc->id);
	/* make sure it doesn't come up again */
	resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);

	} else if(rsc->next_role > rsc->role
	&& !pcmk_is_set(data_set->flags, pe_flag_have_quorum)
	&& data_set->no_quorum_policy == no_quorum_freeze) {
	crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role));
	pe__set_next_role(rsc, rsc->role, "no-quorum-policy=freeze");
	}

	pe__show_node_weights(!pcmk_is_set(data_set->flags, pe_flag_show_scores),
	rsc, __func__, rsc->allowed_nodes, data_set);
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)
	&& !pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	const char *reason = NULL;
	pe_node_t *assign_to = NULL;

	pe__set_next_role(rsc, rsc->role, "unmanaged");
	assign_to = pe__current_node(rsc);
	if (assign_to == NULL) {
	reason = "inactive";
	} else if (rsc->role == RSC_ROLE_PROMOTED) {
	reason = "promoted";
	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	reason = "failed";
	} else {
	reason = "active";
	}
	pe_rsc_info(rsc, "Unmanaged resource %s allocated to %s: %s", rsc->id,
	(assign_to? assign_to->details->uname : "no node"), reason);
	pcmk__assign_primitive(rsc, assign_to, true);

	} else if (pcmk_is_set(data_set->flags, pe_flag_stop_everything)) {
	pe_rsc_debug(rsc, "Forcing %s to stop", rsc->id);
	pcmk__assign_primitive(rsc, NULL, true);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_provisional)
	&& native_choose_node(rsc, prefer, data_set)) {
	pe_rsc_trace(rsc, "Allocated resource %s to %s", rsc->id,
	rsc->allocated_to->details->uname);

	} else if (rsc->allocated_to == NULL) {
	if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
	} else if (rsc->running_on != NULL) {
	pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
	}

	} else {
	pe_rsc_debug(rsc, "Pre-Allocated resource %s to %s", rsc->id,
	rsc->allocated_to->details->uname);
	}

	pe__clear_resource_flags(rsc, pe_rsc_allocating);

	if (rsc->is_remote_node) {
	pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);

	CRM_ASSERT(remote_node != NULL);
	if (rsc->allocated_to && rsc->next_role != RSC_ROLE_STOPPED) {
	crm_trace("Setting Pacemaker Remote node %s to ONLINE",
	remote_node->details->id);
	remote_node->details->online = TRUE;
	/* We shouldn't consider an unseen remote-node unclean if we are going
	* to try and connect to it. Otherwise we get an unnecessary fence */
	if (remote_node->details->unseen == TRUE) {
	remote_node->details->unclean = FALSE;
	}

	} else {
	crm_trace("Setting Pacemaker Remote node %s to SHUTDOWN (next role %s, %sallocated)",
	remote_node->details->id, role2text(rsc->next_role),
	(rsc->allocated_to? "" : "un"));
	remote_node->details->shutdown = TRUE;
	}
	}

	return rsc->allocated_to;
	}

	static gboolean
	is_op_dup(pe_resource_t rsc, const char name, guint interval_ms)
	{
	gboolean dup = FALSE;
	const char *id = NULL;
	const char *value = NULL;
	xmlNode *operation = NULL;
	guint interval2_ms = 0;

	CRM_ASSERT(rsc);
	for (operation = pcmk__xe_first_child(rsc->ops_xml); operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	value = crm_element_value(operation, "name");
	if (!pcmk__str_eq(value, name, pcmk__str_casei)) {
	continue;
	}

	value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval2_ms = crm_parse_interval_spec(value);
	if (interval_ms != interval2_ms) {
	continue;
	}

	if (id == NULL) {
	id = ID(operation);

	} else {
	pcmk__config_err("Operation %s is duplicate of %s (do not use "
	"same name and interval combination more "
	"than once per resource)", ID(operation), id);
	dup = TRUE;
	}
	}
	}

	return dup;
	}

	static bool
	op_cannot_recur(const char *name)
	{
	return pcmk__strcase_any_of(name, RSC_STOP, RSC_START, RSC_DEMOTE, RSC_PROMOTE, NULL);
	}

	static void
	RecurringOp(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *node_uname = node? node->details->uname : "n/a";

	guint interval_ms = 0;
	pe_action_t *mon = NULL;
	gboolean is_optional = TRUE;
	GList *possible_matches = NULL;

	CRM_ASSERT(rsc);

	/* Only process for the operations without role="Stopped" */
	role = crm_element_value(operation, "role");
	if (role && text2role(role) == RSC_ROLE_STOPPED) {
	return;
	}

	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_parse_interval_spec(interval_spec);
	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval_ms)) {
	crm_trace("Not creating duplicate recurring action %s for %dms %s",
	ID(operation), interval_ms, name);
	return;
	}

	if (op_cannot_recur(name)) {
	pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
	ID(operation), name);
	return;
	}

	key = pcmk__op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	crm_trace("Not creating recurring action %s for disabled resource %s",
	ID(operation), rsc->id);
	free(key);
	return;
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s in role %s on %s",
	ID(operation), rsc->id, role2text(rsc->next_role), node_uname);

	if (start != NULL) {
	pe_rsc_trace(rsc, "Marking %s %s due to %s", key,
	pcmk_is_set(start->flags, pe_action_optional)? "optional" : "mandatory",
	start->uuid);
	is_optional = (rsc->cmds->action_flags(start, NULL) & pe_action_optional);
	} else {
	pe_rsc_trace(rsc, "Marking %s optional", key);
	is_optional = TRUE;
	}

	/* start a monitor for an already active resource */
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches == NULL) {
	is_optional = FALSE;
	pe_rsc_trace(rsc, "Marking %s mandatory: not active", key);

	} else {
	GList *gIter = NULL;

	for (gIter = possible_matches; gIter != NULL; gIter = gIter->next) {
	pe_action_t op = (pe_action_t ) gIter->data;

	if (pcmk_is_set(op->flags, pe_action_reschedule)) {
	is_optional = FALSE;
	break;
	}
	}
	g_list_free(possible_matches);
	}

	if (((rsc->next_role == RSC_ROLE_PROMOTED) && (role == NULL))
	\|\| (role != NULL && text2role(role) != rsc->next_role)) {
	int log_level = LOG_TRACE;
	const char *result = "Ignoring";

	if (is_optional) {
	char *after_key = NULL;
	pe_action_t *cancel_op = NULL;

	// It's running, so cancel it
	log_level = LOG_INFO;
	result = "Cancelling";
	cancel_op = pcmk__new_cancel_action(rsc, name, interval_ms, node);

	switch (rsc->role) {
	case RSC_ROLE_UNPROMOTED:
	case RSC_ROLE_STARTED:
	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	after_key = promote_key(rsc);

	} else if (rsc->next_role == RSC_ROLE_STOPPED) {
	after_key = stop_key(rsc);
	}

	break;
	case RSC_ROLE_PROMOTED:
	after_key = demote_key(rsc);
	break;
	default:
	break;
	}

	if (after_key) {
	pcmk__new_ordering(rsc, NULL, cancel_op, rsc, after_key, NULL,
	pe_order_runnable_left, data_set);
	}
	}

	do_crm_log(log_level, "%s action %s (%s vs. %s)",
	result, key, role ? role : role2text(RSC_ROLE_UNPROMOTED),
	role2text(rsc->next_role));

	free(key);
	return;
	}

	mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set);
	key = mon->uuid;
	if (is_optional) {
	pe_rsc_trace(rsc, "%s\t %s (optional)", node_uname, mon->uuid);
	}

	if ((start == NULL) \|\| !pcmk_is_set(start->flags, pe_action_runnable)) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : start un-runnable)",
	node_uname, mon->uuid);
	pe__clear_action_flags(mon, pe_action_runnable);

	} else if (node == NULL \|\| node->details->online == FALSE \|\| node->details->unclean) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)",
	node_uname, mon->uuid);
	pe__clear_action_flags(mon, pe_action_runnable);

	} else if (!pcmk_is_set(mon->flags, pe_action_optional)) {
	pe_rsc_info(rsc, " Start recurring %s (%us) for %s on %s",
	mon->task, interval_ms / 1000, rsc->id, node_uname);
	}

	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	char *running_promoted = pcmk__itoa(PCMK_OCF_RUNNING_PROMOTED);

	add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_promoted);
	free(running_promoted);
	}

	if ((node == NULL) \|\| pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pcmk__new_ordering(rsc, start_key(rsc), NULL, NULL, strdup(key), mon,
	pe_order_implies_then\|pe_order_runnable_left,
	data_set);

	pcmk__new_ordering(rsc, reload_key(rsc), NULL, NULL, strdup(key), mon,
	pe_order_implies_then\|pe_order_runnable_left,
	data_set);

	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	pcmk__new_ordering(rsc, promote_key(rsc), NULL, rsc, NULL, mon,
	pe_order_optional\|pe_order_runnable_left,
	data_set);

	} else if (rsc->role == RSC_ROLE_PROMOTED) {
	pcmk__new_ordering(rsc, demote_key(rsc), NULL, rsc, NULL, mon,
	pe_order_optional\|pe_order_runnable_left,
	data_set);
	}
	}
	}

	static void
	Recurring(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_maintenance) &&
	(node == NULL \|\| node->details->maintenance == FALSE)) {
	xmlNode *operation = NULL;

	for (operation = pcmk__xe_first_child(rsc->ops_xml);
	operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	RecurringOp(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	static void
	RecurringOp_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *node_uname = node? node->details->uname : "n/a";

	guint interval_ms = 0;
	GList *possible_matches = NULL;
	GList *gIter = NULL;

	/* Only process for the operations with role="Stopped" */
	role = crm_element_value(operation, "role");
	if (role == NULL \|\| text2role(role) != RSC_ROLE_STOPPED) {
	return;
	}

	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_parse_interval_spec(interval_spec);
	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval_ms)) {
	crm_trace("Not creating duplicate recurring action %s for %dms %s",
	ID(operation), interval_ms, name);
	return;
	}

	if (op_cannot_recur(name)) {
	pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
	ID(operation), name);
	return;
	}

	key = pcmk__op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	crm_trace("Not creating recurring action %s for disabled resource %s",
	ID(operation), rsc->id);
	free(key);
	return;
	}

	// @TODO add support
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	crm_notice("Ignoring %s (recurring monitors for Stopped role are "
	"not supported for anonymous clones)",
	ID(operation));
	return;
	}

	pe_rsc_trace(rsc,
	"Creating recurring action %s for %s in role %s on nodes where it should not be running",
	ID(operation), rsc->id, role2text(rsc->next_role));

	/* if the monitor exists on the node where the resource will be running, cancel it */
	if (node != NULL) {
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches) {
	pe_action_t *cancel_op = NULL;

	g_list_free(possible_matches);

	cancel_op = pcmk__new_cancel_action(rsc, name, interval_ms, node);

	if ((rsc->next_role == RSC_ROLE_STARTED)
	\|\| (rsc->next_role == RSC_ROLE_UNPROMOTED)) {
	/* rsc->role == RSC_ROLE_STOPPED: cancel the monitor before start */
	/* rsc->role == RSC_ROLE_STARTED: for a migration, cancel the monitor on the target node before start */
	pcmk__new_ordering(rsc, NULL, cancel_op, rsc, start_key(rsc),
	NULL, pe_order_runnable_left, data_set);
	}

	pe_rsc_info(rsc, "Cancel action %s (%s vs. %s) on %s",
	key, role, role2text(rsc->next_role), node_uname);
	}
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t stop_node = (pe_node_t ) gIter->data;
	const char *stop_node_uname = stop_node->details->uname;
	gboolean is_optional = TRUE;
	gboolean probe_is_optional = TRUE;
	gboolean stop_is_optional = TRUE;
	pe_action_t *stopped_mon = NULL;
	char *rc_inactive = NULL;
	GList *stop_ops = NULL;
	GList *local_gIter = NULL;

	if (node && pcmk__str_eq(stop_node_uname, node_uname, pcmk__str_casei)) {
	continue;
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s on %s",
	ID(operation), rsc->id, crm_str(stop_node_uname));

	/* start a monitor for an already stopped resource */
	possible_matches = find_actions_exact(rsc->actions, key, stop_node);
	if (possible_matches == NULL) {
	pe_rsc_trace(rsc, "Marking %s mandatory on %s: not active", key,
	crm_str(stop_node_uname));
	is_optional = FALSE;
	} else {
	pe_rsc_trace(rsc, "Marking %s optional on %s: already active", key,
	crm_str(stop_node_uname));
	is_optional = TRUE;
	g_list_free(possible_matches);
	}

	stopped_mon = custom_action(rsc, strdup(key), name, stop_node, is_optional, TRUE, data_set);

	rc_inactive = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
	add_hash_param(stopped_mon->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
	free(rc_inactive);

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	GList *probes = pe__resource_actions(rsc, stop_node, RSC_STATUS,
	FALSE);
	GList *pIter = NULL;

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	pe_action_t probe = (pe_action_t ) pIter->data;

	order_actions(probe, stopped_mon, pe_order_runnable_left);
	crm_trace("%s then %s on %s", probe->uuid, stopped_mon->uuid, stop_node->details->uname);
	}

	g_list_free(probes);
	}

	stop_ops = pe__resource_actions(rsc, stop_node, RSC_STOP, TRUE);

	for (local_gIter = stop_ops; local_gIter != NULL; local_gIter = local_gIter->next) {
	pe_action_t stop = (pe_action_t ) local_gIter->data;

	if (!pcmk_is_set(stop->flags, pe_action_optional)) {
	stop_is_optional = FALSE;
	}

	if (!pcmk_is_set(stop->flags, pe_action_runnable)) {
	crm_debug("%s\t %s (cancelled : stop un-runnable)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	pe__clear_action_flags(stopped_mon, pe_action_runnable);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pcmk__new_ordering(rsc, stop_key(rsc), stop, NULL, strdup(key),
	stopped_mon,
	pe_order_implies_then\|pe_order_runnable_left,
	data_set);
	}

	}

	if (stop_ops) {
	g_list_free(stop_ops);
	}

	if (is_optional == FALSE && probe_is_optional && stop_is_optional
	&& !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Marking %s optional on %s due to unmanaged",
	key, crm_str(stop_node_uname));
	pe__set_action_flags(stopped_mon, pe_action_optional);
	}

	if (pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
	pe_rsc_trace(rsc, "%s\t %s (optional)", crm_str(stop_node_uname), stopped_mon->uuid);
	}

	if (stop_node->details->online == FALSE \|\| stop_node->details->unclean) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	pe__clear_action_flags(stopped_mon, pe_action_runnable);
	}

	if (pcmk_is_set(stopped_mon->flags, pe_action_runnable)
	&& !pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
	crm_notice(" Start recurring %s (%us) for %s on %s", stopped_mon->task,
	interval_ms / 1000, rsc->id, crm_str(stop_node_uname));
	}
	}

	free(key);
	}

	static void
	Recurring_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_maintenance) &&
	(node == NULL \|\| node->details->maintenance == FALSE)) {
	xmlNode *operation = NULL;

	for (operation = pcmk__xe_first_child(rsc->ops_xml);
	operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	RecurringOp_Stopped(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	static void
	handle_migration_actions(pe_resource_t * rsc, pe_node_t current, pe_node_t chosen, pe_working_set_t * data_set)
	{
	pe_action_t *migrate_to = NULL;
	pe_action_t *migrate_from = NULL;
	pe_action_t *start = NULL;
	pe_action_t *stop = NULL;
	gboolean partial = rsc->partial_migration_target ? TRUE : FALSE;

	pe_rsc_trace(rsc, "Processing migration actions %s moving from %s to %s . partial migration = %s",
	rsc->id, current->details->id, chosen->details->id, partial ? "TRUE" : "FALSE");
	start = start_action(rsc, chosen, TRUE);
	stop = stop_action(rsc, current, TRUE);

	if (partial == FALSE) {
	migrate_to = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	RSC_MIGRATE, current, TRUE, TRUE, data_set);
	}

	migrate_from = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
	RSC_MIGRATED, chosen, TRUE, TRUE, data_set);

	if ((migrate_to && migrate_from) \|\| (migrate_from && partial)) {

	pe__set_action_flags(start, pe_action_migrate_runnable);
	pe__set_action_flags(stop, pe_action_migrate_runnable);

	// This is easier than trying to delete it from the graph
	pe__set_action_flags(start, pe_action_pseudo);

	/* order probes before migrations */
	if (partial) {
	pe__set_action_flags(migrate_from, pe_action_migrate_runnable);
	migrate_from->needs = start->needs;

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
	NULL, pe_order_optional, data_set);

	} else {
	pe__set_action_flags(migrate_from, pe_action_migrate_runnable);
	pe__set_action_flags(migrate_to, pe_action_migrate_runnable);
	migrate_to->needs = start->needs;

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	NULL, pe_order_optional, data_set);
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
	NULL,
	pe_order_optional\|pe_order_implies_first_migratable,
	data_set);
	}

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_optional\|pe_order_implies_first_migratable,
	data_set);
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	pe_order_optional\|pe_order_implies_first_migratable\|pe_order_pseudo_left,
	data_set);
	}

	if (migrate_to) {
	add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
	add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);

	/* Pacemaker Remote connections don't require pending to be recorded in
	* the CIB. We can reduce CIB writes by not setting PENDING for them.
	*/
	if (rsc->is_remote_node == FALSE) {
	/* migrate_to takes place on the source node, but can
	* have an effect on the target node depending on how
	* the agent is written. Because of this, we have to maintain
	* a record that the migrate_to occurred, in case the source node
	* loses membership while the migrate_to action is still in-flight.
	*/
	add_hash_param(migrate_to->meta, XML_OP_ATTR_PENDING, "true");
	}
	}

	if (migrate_from) {
	add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
	add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
	}
	}

	void
	native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_action_t *start = NULL;
	pe_node_t *chosen = NULL;
	pe_node_t *current = NULL;
	gboolean need_stop = FALSE;
	bool need_promote = FALSE;
	gboolean is_moving = FALSE;
	gboolean allow_migrate = pcmk_is_set(rsc->flags, pe_rsc_allow_migrate)? TRUE : FALSE;

	GList *gIter = NULL;
	unsigned int num_all_active = 0;
	unsigned int num_clean_active = 0;
	bool multiply_active = FALSE;
	enum rsc_role_e role = RSC_ROLE_UNKNOWN;
	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;

	CRM_ASSERT(rsc);
	chosen = rsc->allocated_to;
	next_role = rsc->next_role;
	if (next_role == RSC_ROLE_UNKNOWN) {
	pe__set_next_role(rsc,
	(chosen == NULL)? RSC_ROLE_STOPPED : RSC_ROLE_STARTED,
	"allocation");
	}
	pe_rsc_trace(rsc, "Creating all actions for %s transition from %s to %s (%s) on %s",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role),
	((next_role == RSC_ROLE_UNKNOWN)? "implicit" : "explicit"),
	((chosen == NULL)? "no node" : chosen->details->uname));

	current = pe__find_active_on(rsc, &num_all_active, &num_clean_active);

	for (gIter = rsc->dangling_migrations; gIter != NULL; gIter = gIter->next) {
	pe_node_t dangling_source = (pe_node_t ) gIter->data;

	pe_action_t *stop = NULL;

	pe_rsc_trace(rsc, "Creating stop action %sfor %s on %s due to dangling migration",
	pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)? "and cleanup " : "",
	rsc->id, dangling_source->details->uname);
	stop = stop_action(rsc, dangling_source, FALSE);
	pe__set_action_flags(stop, pe_action_dangle);
	if (pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, dangling_source, FALSE, data_set);
	}
	}

	if ((num_all_active == 2) && (num_clean_active == 2) && chosen
	&& rsc->partial_migration_source && rsc->partial_migration_target
	&& (current->details == rsc->partial_migration_source->details)
	&& (chosen->details == rsc->partial_migration_target->details)) {

	/* The chosen node is still the migration target from a partial
	* migration. Attempt to continue the migration instead of recovering
	* by stopping the resource everywhere and starting it on a single node.
	*/
	pe_rsc_trace(rsc, "Will attempt to continue with partial migration "
	"to target %s from %s",
	rsc->partial_migration_target->details->id,
	rsc->partial_migration_source->details->id);

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	/* If a resource has "requires" set to nothing or quorum, don't consider
	* it active on unclean nodes (similar to how all resources behave when
	* stonith-enabled is false). We can start such resources elsewhere
	* before fencing completes, and if we considered the resource active on
	* the failed node, we would attempt recovery for being active on
	* multiple nodes.
	*/
	multiply_active = (num_clean_active > 1);
	} else {
	multiply_active = (num_all_active > 1);
	}

	if (multiply_active) {
	if (rsc->partial_migration_target && rsc->partial_migration_source) {
	// Migration was in progress, but we've chosen a different target
	crm_notice("Resource %s can no longer migrate from %s to %s "
	"(will stop on both nodes)",
	rsc->id, rsc->partial_migration_source->details->uname,
	rsc->partial_migration_target->details->uname);

	} else {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	// Resource was (possibly) incorrectly multiply active
	pe_proc_err("%s resource %s might be active on %u nodes (%s)",
	crm_str(class), rsc->id, num_all_active,
	recovery2text(rsc->recovery_type));
	crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ#Resource_is_Too_Active for more information");
	}

	if (rsc->recovery_type == recovery_stop_start) {
	need_stop = TRUE;
	}

	/* If by chance a partial migration is in process, but the migration
	* target is not chosen still, clear all partial migration data.
	*/
	rsc->partial_migration_source = rsc->partial_migration_target = NULL;
	allow_migrate = FALSE;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
	pe_rsc_trace(rsc, "Creating start action for %s to represent already pending start",
	rsc->id);
	start = start_action(rsc, chosen, TRUE);
	pe__set_action_flags(start, pe_action_print_always);
	}

	if (current && chosen && current->details != chosen->details) {
	pe_rsc_trace(rsc, "Moving %s from %s to %s",
	rsc->id, crm_str(current->details->uname),
	crm_str(chosen->details->uname));
	is_moving = TRUE;
	need_stop = TRUE;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	if (pcmk_is_set(rsc->flags, pe_rsc_stop)) {
	need_stop = TRUE;
	pe_rsc_trace(rsc, "Recovering %s", rsc->id);
	} else {
	pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	need_promote = TRUE;
	}
	}

	} else if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	pe_rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
	need_stop = TRUE;

	} else if (rsc->role > RSC_ROLE_STARTED && current != NULL && chosen != NULL) {
	pe_rsc_trace(rsc, "Creating start action for promoted resource %s",
	rsc->id);
	start = start_action(rsc, chosen, TRUE);
	if (!pcmk_is_set(start->flags, pe_action_optional)) {
	// Recovery of a promoted resource
	pe_rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
	need_stop = TRUE;
	}
	}

	/* Create any additional actions required when bringing resource down and
	* back up to same level.
	*/
	role = rsc->role;
	while (role != RSC_ROLE_STOPPED) {
	next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
	pe_rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
	(need_stop? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	if (rsc_action_matrix[role][next_role] (rsc, current, !need_stop, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}


	while ((rsc->role <= rsc->next_role) && (role != rsc->role)
	&& !pcmk_is_set(rsc->flags, pe_rsc_block)) {
	bool required = need_stop;

	next_role = rsc_state_matrix[role][rsc->role];
	if ((next_role == RSC_ROLE_PROMOTED) && need_promote) {
	required = true;
	}
	pe_rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
	(required? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	if (rsc_action_matrix[role][next_role](rsc, chosen, !required,
	data_set) == FALSE) {
	break;
	}
	role = next_role;
	}
	role = rsc->role;

	/* Required steps from this role to the next */
	while (role != rsc->next_role) {
	next_role = rsc_state_matrix[role][rsc->next_role];
	pe_rsc_trace(rsc, "Creating action to take %s from %s to %s (ending at %s)",
	rsc->id, role2text(role), role2text(next_role),
	role2text(rsc->next_role));
	if (rsc_action_matrix[role][next_role] (rsc, chosen, FALSE, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	pe_rsc_trace(rsc, "Not creating recurring monitors for blocked resource %s",
	rsc->id);

	} else if ((rsc->next_role != RSC_ROLE_STOPPED)
	\|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Creating recurring monitors for %s resource %s",
	((rsc->next_role == RSC_ROLE_STOPPED)? "unmanaged" : "active"),
	rsc->id);
	start = start_action(rsc, chosen, TRUE);
	Recurring(rsc, start, chosen, data_set);
	Recurring_Stopped(rsc, start, chosen, data_set);

	} else {
	pe_rsc_trace(rsc, "Creating recurring monitors for inactive resource %s",
	rsc->id);
	Recurring_Stopped(rsc, NULL, NULL, data_set);
	}

	/* if we are stuck in a partial migration, where the target
	* of the partial migration no longer matches the chosen target.
	* A full stop/start is required */
	if (rsc->partial_migration_target && (chosen == NULL \|\| rsc->partial_migration_target->details != chosen->details)) {
	pe_rsc_trace(rsc, "Not allowing partial migration of %s to continue",
	rsc->id);
	allow_migrate = FALSE;

	} else if (!is_moving \|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)
	\|\| pcmk_any_flags_set(rsc->flags,
	pe_rsc_failed\|pe_rsc_start_pending)
	\|\| (current && current->details->unclean)
	\|\| rsc->next_role < RSC_ROLE_STARTED) {

	allow_migrate = FALSE;
	}

	if (allow_migrate) {
	handle_migration_actions(rsc, current, chosen, data_set);
	}
	}

	static void
	rsc_avoids_remote_nodes(pe_resource_t *rsc)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->details->remote_rsc) {
	node->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Return allowed nodes as (possibly sorted) list
	*
	* Convert a resource's hash table of allowed nodes to a list. If printing to
	* stdout, sort the list, to keep action ID numbers consistent for regression
	* test output (while avoiding the performance hit on a live cluster).
	*
	* \param[in] rsc Resource to check for allowed nodes
	* \param[in] data_set Cluster working set
	*
	* \return List of resource's allowed nodes
	* \note Callers should take care not to rely on the list being sorted.
	*/
	static GList *
	allowed_nodes_as_list(pe_resource_t rsc, pe_working_set_t data_set)
	{
	GList *allowed_nodes = NULL;

	if (rsc->allowed_nodes) {
	allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	if (!pcmk__is_daemon) {
	allowed_nodes = g_list_sort(allowed_nodes, sort_node_uname);
	}

	return allowed_nodes;
	}

	void
	native_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	/* This function is on the critical path and worth optimizing as much as possible */

	pe_resource_t *top = NULL;
	GList *allowed_nodes = NULL;
	bool check_unfencing = FALSE;
	bool check_utilization = false;

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc,
	"Skipping native constraints for unmanaged resource: %s",
	rsc->id);
	return;
	}

	top = uber_parent(rsc);

	// Whether resource requires unfencing
	check_unfencing = !pcmk_is_set(rsc->flags, pe_rsc_fence_device)
	&& pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)
	&& pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing);

	// Whether a non-default placement strategy is used
	check_utilization = (g_hash_table_size(rsc->utilization) > 0)
	&& !pcmk__str_eq(data_set->placement_strategy,
	"default", pcmk__str_casei);

	// Order stops before starts (i.e. restart)
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	pe_order_optional\|pe_order_implies_then\|pe_order_restart,
	data_set);

	// Promotable ordering: demote before stop, start before promote
	if (pcmk_is_set(top->flags, pe_rsc_promotable)
	\|\| (rsc->role > RSC_ROLE_UNPROMOTED)) {

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_DEMOTE, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_promoted_implies_first, data_set);

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_PROMOTE, 0), NULL,
	pe_order_runnable_left, data_set);
	}

	// Don't clear resource history if probing on same node
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0),
	NULL, rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0),
	NULL, pe_order_same_node\|pe_order_then_cancels_first,
	data_set);

	// Certain checks need allowed nodes
	if (check_unfencing \|\| check_utilization \|\| rsc->container) {
	allowed_nodes = allowed_nodes_as_list(rsc, data_set);
	}

	if (check_unfencing) {
	/* Check if the node needs to be unfenced first */

	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *node = item->data;
	pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);

	crm_debug("Ordering any stops of %s before %s, and any starts after",
	rsc->id, unfence->uuid);

	/*
	* It would be more efficient to order clone resources once,
	* rather than order each instance, but ordering the instance
	* allows us to avoid unnecessary dependencies that might conflict
	* with user constraints.
	*
	* @TODO: This constraint can still produce a transition loop if the
	* resource has a stop scheduled on the node being unfenced, and
	* there is a user ordering constraint to start some other resource
	* (which will be ordered after the unfence) before stopping this
	* resource. An example is "start some slow-starting cloned service
	* before stopping an associated virtual IP that may be moving to
	* it":
	* stop this -> unfencing -> start that -> stop this
	*/
	pcmk__new_ordering(rsc, stop_key(rsc), NULL,
	NULL, strdup(unfence->uuid), unfence,
	pe_order_optional\|pe_order_same_node, data_set);

	pcmk__new_ordering(NULL, strdup(unfence->uuid), unfence,
	rsc, start_key(rsc), NULL,
	pe_order_implies_then_on_node\|pe_order_same_node,
	data_set);
	}
	}

	if (check_utilization) {
	pcmk__create_utilization_constraints(rsc, allowed_nodes);
	}

	if (rsc->container) {
	pe_resource_t *remote_rsc = NULL;

	if (rsc->is_remote_node) {
	// rsc is the implicit remote connection for a guest or bundle node

	/* Do not allow a guest resource to live on a Pacemaker Remote node,
	* to avoid nesting remotes. However, allow bundles to run on remote
	* nodes.
	*/
	if (!pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
	rsc_avoids_remote_nodes(rsc->container);
	}

	/* If someone cleans up a guest or bundle node's container, we will
	* likely schedule a (re-)probe of the container and recovery of the
	* connection. Order the connection stop after the container probe,
	* so that if we detect the container running, we will trigger a new
	* transition and avoid the unnecessary recovery.
	*/
	pcmk__order_resource_actions(rsc->container, RSC_STATUS, rsc,
	RSC_STOP, pe_order_optional, data_set);

	/* A user can specify that a resource must start on a Pacemaker Remote
	* node by explicitly configuring it with the container=NODENAME
	* meta-attribute. This is of questionable merit, since location
	* constraints can accomplish the same thing. But we support it, so here
	* we check whether a resource (that is not itself a remote connection)
	* has container set to a remote node or guest node resource.
	*/
	} else if (rsc->container->is_remote_node) {
	remote_rsc = rsc->container;
	} else {
	remote_rsc = pe__resource_contains_guest_node(data_set,
	rsc->container);
	}

	if (remote_rsc) {
	/* Force the resource on the Pacemaker Remote node instead of
	* colocating the resource with the container resource.
	*/
	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *node = item->data;

	if (node->details->remote_rsc != remote_rsc) {
	node->weight = -INFINITY;
	}
	}

	} else {
	/* This resource is either a filler for a container that does NOT
	* represent a Pacemaker Remote node, or a Pacemaker Remote
	* connection resource for a guest node or bundle.
	*/
	int score;

	crm_trace("Order and colocate %s relative to its container %s",
	rsc->id, rsc->container->id);

	pcmk__new_ordering(rsc->container,
	pcmk__op_key(rsc->container->id, RSC_START, 0),
	NULL, rsc, pcmk__op_key(rsc->id, RSC_START, 0),
	NULL,
	pe_order_implies_then\|pe_order_runnable_left,
	data_set);

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc->container,
	pcmk__op_key(rsc->container->id, RSC_STOP, 0),
	NULL, pe_order_implies_first, data_set);

	if (pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
	score = 10000; /* Highly preferred but not essential */
	} else {
	score = INFINITY; /* Force them to run on the same host */
	}
	pcmk__new_colocation("resource-with-container", NULL, score, rsc,
	rsc->container, NULL, NULL, true, data_set);
	}
	}

	if (rsc->is_remote_node \|\| pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
	/* don't allow remote nodes to run stonith devices
	* or remote connection resources.*/
	rsc_avoids_remote_nodes(rsc);
	}
	g_list_free(allowed_nodes);
	}

	void
	native_rsc_colocation_lh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	if (dependent == NULL) {
	pe_err("dependent was NULL for %s", constraint->id);
	return;

	} else if (constraint->primary == NULL) {
	pe_err("primary was NULL for %s", constraint->id);
	return;
	}

	pe_rsc_trace(dependent,
	"Processing colocation constraint between %s and %s",
	dependent->id, primary->id);

	primary->cmds->rsc_colocation_rh(dependent, primary, constraint, data_set);
	}

	void
	native_rsc_colocation_rh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	enum pcmk__coloc_affects filter_results;

	CRM_ASSERT((dependent != NULL) && (primary != NULL));
	filter_results = pcmk__colocation_affects(dependent, primary, constraint,
	false);
	pe_rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
	((constraint->score > 0)? "Colocating" : "Anti-colocating"),
	dependent->id, primary->id, constraint->id, constraint->score,
	filter_results);

	switch (filter_results) {
	case pcmk__coloc_affects_role:
	pcmk__apply_coloc_to_priority(dependent, primary, constraint);
	break;
	case pcmk__coloc_affects_location:
	pcmk__apply_coloc_to_weights(dependent, primary, constraint);
	break;
	case pcmk__coloc_affects_nothing:
	default:
	return;
	}
	}

	enum pe_action_flags
	native_action_flags(pe_action_t * action, pe_node_t * node)
	{
	return action->flags;
	}

	static inline bool
	is_primitive_action(pe_action_t *action)
	{
	return action && action->rsc && (action->rsc->variant == pe_native);
	}

	/*!
	* \internal
	* \brief Clear a single action flag and set reason text
	*
	* \param[in] action Action whose flag should be cleared
	* \param[in] flag Action flag that should be cleared
	* \param[in] reason Action that is the reason why flag is being cleared
	*/
	#define clear_action_flag_because(action, flag, reason) do { \
	if (pcmk_is_set((action)->flags, (flag))) { \
	pe__clear_action_flags(action, flag); \
	if ((action)->rsc != (reason)->rsc) { \
	char *reason_text = pe__action2reason((reason), (flag)); \
	pe_action_set_reason((action), reason_text, \
	((flag) == pe_action_migrate_runnable)); \
	free(reason_text); \
	} \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Set action bits appropriately when pe_restart_order is used
	*
	* \param[in] first 'First' action in an ordering with pe_restart_order
	* \param[in] then 'Then' action in an ordering with pe_restart_order
	* \param[in] filter What ordering flags to care about
	*
	* \note pe_restart_order is set for "stop resource before starting it" and
	* "stop later group member before stopping earlier group member"
	*/
	static void
	handle_restart_ordering(pe_action_t first, pe_action_t then,
	enum pe_action_flags filter)
	{
	const char *reason = NULL;

	CRM_ASSERT(is_primitive_action(first));
	CRM_ASSERT(is_primitive_action(then));

	// We need to update the action in two cases:

	// ... if 'then' is required
	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {
	reason = "restart";
	}

	/* ... if 'then' is unrunnable action on same resource (if a resource
	* should restart but can't start, we still want to stop)
	*/
	if (pcmk_is_set(filter, pe_action_runnable)
	&& !pcmk_is_set(then->flags, pe_action_runnable)
	&& pcmk_is_set(then->rsc->flags, pe_rsc_managed)
	&& (first->rsc == then->rsc)) {
	reason = "stop";
	}

	if (reason == NULL) {
	return;
	}

	pe_rsc_trace(first->rsc, "Handling %s -> %s for %s",
	first->uuid, then->uuid, reason);

	// Make 'first' required if it is runnable
	if (pcmk_is_set(first->flags, pe_action_runnable)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}

	// Make 'first' required if 'then' is required
	if (!pcmk_is_set(then->flags, pe_action_optional)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}

	// Make 'first' unmigratable if 'then' is unmigratable
	if (!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	clear_action_flag_because(first, pe_action_migrate_runnable, then);
	}

	// Make 'then' unrunnable if 'first' is required but unrunnable
	if (!pcmk_is_set(first->flags, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_runnable)) {
	clear_action_flag_because(then, pe_action_runnable, first);
	}
	}

	/* \param[in] flags Flags from action_flags_for_ordering()
	*/
	enum pe_graph_flags
	native_update_actions(pe_action_t first, pe_action_t then, pe_node_t *node,
	enum pe_action_flags flags, enum pe_action_flags filter,
	enum pe_ordering type, pe_working_set_t *data_set)
	{
	enum pe_graph_flags changed = pe_graph_none;
	enum pe_action_flags then_flags = then->flags;
	enum pe_action_flags first_flags = first->flags;

	if (type & pe_order_asymmetrical) {
	pe_resource_t *then_rsc = then->rsc;
	enum rsc_role_e then_rsc_role = then_rsc ? then_rsc->fns->state(then_rsc, TRUE) : 0;

	if (!then_rsc) {
	/* ignore */
	} else if ((then_rsc_role == RSC_ROLE_STOPPED) && pcmk__str_eq(then->task, RSC_STOP, pcmk__str_casei)) {
	/* ignore... if 'then' is supposed to be stopped after 'first', but
	* then is already stopped, there is nothing to be done when non-symmetrical. */
	} else if ((then_rsc_role >= RSC_ROLE_STARTED)
	&& pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)
	&& pcmk_is_set(then->flags, pe_action_optional)
	&& then->node
	&& pcmk__list_of_1(then_rsc->running_on)
	&& then->node->details == ((pe_node_t *) then_rsc->running_on->data)->details) {
	/* Ignore. If 'then' is supposed to be started after 'first', but
	* 'then' is already started, there is nothing to be done when
	* asymmetrical -- unless the start is mandatory, which indicates
	* the resource is restarting, and the ordering is still needed.
	*/
	} else if (!(first->flags & pe_action_runnable)) {
	/* prevent 'then' action from happening if 'first' is not runnable and
	* 'then' has not yet occurred. */
	clear_action_flag_because(then, pe_action_optional, first);
	clear_action_flag_because(then, pe_action_runnable, first);
	} else {
	/* ignore... then is allowed to start/stop if it wants to. */
	}
	}

	if (pcmk_is_set(type, pe_order_implies_first)
	&& !pcmk_is_set(then_flags, pe_action_optional)) {
	// Then is required, and implies first should be, too

	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(flags, pe_action_optional)
	&& pcmk_is_set(first_flags, pe_action_optional)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}

	if (pcmk_is_set(flags, pe_action_migrate_runnable) &&
	!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	clear_action_flag_because(first, pe_action_migrate_runnable, then);
	}
	}

	if (type & pe_order_promoted_implies_first) {
	if ((filter & pe_action_optional) &&
	((then->flags & pe_action_optional) == FALSE) &&
	(then->rsc != NULL) && (then->rsc->role == RSC_ROLE_PROMOTED)) {

	clear_action_flag_because(first, pe_action_optional, then);

	if (pcmk_is_set(first->flags, pe_action_migrate_runnable) &&
	!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	clear_action_flag_because(first, pe_action_migrate_runnable,
	then);
	}
	}
	}

	if ((type & pe_order_implies_first_migratable)
	&& pcmk_is_set(filter, pe_action_optional)) {

	if (((then->flags & pe_action_migrate_runnable) == FALSE) \|\|
	((then->flags & pe_action_runnable) == FALSE)) {
	clear_action_flag_because(first, pe_action_runnable, then);
	}

	if ((then->flags & pe_action_optional) == 0) {
	clear_action_flag_because(first, pe_action_optional, then);
	}
	}

	if ((type & pe_order_pseudo_left)
	&& pcmk_is_set(filter, pe_action_optional)) {

	if ((first->flags & pe_action_runnable) == FALSE) {
	clear_action_flag_because(then, pe_action_migrate_runnable, first);
	pe__clear_action_flags(then, pe_action_pseudo);
	}
	}

	if (pcmk_is_set(type, pe_order_runnable_left)
	&& pcmk_is_set(filter, pe_action_runnable)
	&& pcmk_is_set(then->flags, pe_action_runnable)
	&& !pcmk_is_set(flags, pe_action_runnable)) {

	clear_action_flag_because(then, pe_action_runnable, first);
	clear_action_flag_because(then, pe_action_migrate_runnable, first);
	}

	if (pcmk_is_set(type, pe_order_implies_then)
	&& pcmk_is_set(filter, pe_action_optional)
	&& pcmk_is_set(then->flags, pe_action_optional)
	&& !pcmk_is_set(flags, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_migrate_runnable)) {

	clear_action_flag_because(then, pe_action_optional, first);
	}

	if (pcmk_is_set(type, pe_order_restart)) {
	handle_restart_ordering(first, then, filter);
	}

	if (then_flags != then->flags) {
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	pe_rsc_trace(then->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'first' %s (%#.6x)",
	then->uuid,
	then->node? then->node->details->uname : "no node",
	then->flags, then_flags, first->uuid, first->flags);

	if(then->rsc && then->rsc->parent) {
	/* "X_stop then X_start" doesn't get handled for cloned groups unless we do this */
	pcmk__update_action_for_orderings(then, data_set);
	}
	}

	if (first_flags != first->flags) {
	pe__set_graph_flags(changed, first, pe_graph_updated_first);
	pe_rsc_trace(first->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'then' %s (%#.6x)",
	first->uuid,
	first->node? first->node->details->uname : "no node",
	first->flags, first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	void
	native_rsc_location(pe_resource_t rsc, pe__location_t constraint)
	{
	pcmk__apply_location(constraint, rsc);
	}

	void
	native_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Processing actions from %s", rsc->id);

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	crm_trace("processing action %d for rsc=%s", action->id, rsc->id);
	pcmk__add_action_to_graph(action, data_set);
	}

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->expand(child_rsc, data_set);
	}
	}

	gboolean
	StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t current = (pe_node_t ) gIter->data;
	pe_action_t *stop;

	if (rsc->partial_migration_target) {
	if (rsc->partial_migration_target->details == current->details) {
	pe_rsc_trace(rsc, "Filtered %s -> %s %s", current->details->uname,
	next->details->uname, rsc->id);
	continue;
	} else {
	pe_rsc_trace(rsc, "Forced on %s %s", current->details->uname, rsc->id);
	optional = FALSE;
	}
	}

	pe_rsc_trace(rsc, "%s on %s", rsc->id, current->details->uname);
	stop = stop_action(rsc, current, optional);

	if(rsc->allocated_to == NULL) {
	pe_action_set_reason(stop, "node availability", TRUE);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe__clear_action_flags(stop, pe_action_runnable);
	}

	if (pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, current, optional, data_set);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
	pe_action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, FALSE, data_set);

	order_actions(stop, unfence, pe_order_implies_first);
	if (!pcmk__node_unfenced(current)) {
	pe_proc_err("Stopping %s until %s can be unfenced", rsc->id, current->details->uname);
	}
	}
	}

	return TRUE;
	}

	gboolean
	StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	pe_action_t *start = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s on %s %d %d", rsc->id, next ? next->details->uname : "N/A", optional, next ? next->weight : 0);
	start = start_action(rsc, next, TRUE);

	pcmk__order_vs_unfence(rsc, next, start, pe_order_implies_then, data_set);

	if (pcmk_is_set(start->flags, pe_action_runnable) && !optional) {
	pe__clear_action_flags(start, pe_action_optional);
	}


	return TRUE;
	}

	gboolean
	PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	gboolean runnable = TRUE;
	GList *action_list = NULL;

	CRM_ASSERT(rsc);
	CRM_CHECK(next != NULL, return FALSE);
	pe_rsc_trace(rsc, "%s on %s", rsc->id, next->details->uname);

	action_list = pe__resource_actions(rsc, next, RSC_START, TRUE);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t start = (pe_action_t ) gIter->data;

	if (!pcmk_is_set(start->flags, pe_action_runnable)) {
	runnable = FALSE;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	promote_action(rsc, next, optional);
	return TRUE;
	}

	pe_rsc_debug(rsc, "%s\tPromote %s (canceled)", next->details->uname, rsc->id);

	action_list = pe__resource_actions(rsc, next, RSC_PROMOTE, TRUE);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t promote = (pe_action_t ) gIter->data;

	pe__clear_action_flags(promote, pe_action_runnable);
	}

	g_list_free(action_list);
	return TRUE;
	}

	gboolean
	DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);

	/* CRM_CHECK(rsc->next_role == RSC_ROLE_UNPROMOTED, return FALSE); */
	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t current = (pe_node_t ) gIter->data;

	pe_rsc_trace(rsc, "%s on %s", rsc->id, next ? next->details->uname : "N/A");
	demote_action(rsc, current, optional);
	}
	return TRUE;
	}

	gboolean
	RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	CRM_ASSERT(rsc);
	crm_err("%s on %s", rsc->id, next ? next->details->uname : "N/A");
	CRM_CHECK(FALSE, return FALSE);
	return FALSE;
	}

	gboolean
	NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);
	return FALSE;
	}

	gboolean
	DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set)
	{
	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_trace(rsc, "Resource %s not deleted from %s: failed", rsc->id, node->details->uname);
	return FALSE;

	} else if (node == NULL) {
	pe_rsc_trace(rsc, "Resource %s not deleted: NULL node", rsc->id);
	return FALSE;

	} else if (node->details->unclean \|\| node->details->online == FALSE) {
	pe_rsc_trace(rsc, "Resource %s not deleted from %s: unrunnable", rsc->id,
	node->details->uname);
	return FALSE;
	}

	crm_notice("Removing %s from %s", rsc->id, node->details->uname);

	delete_action(rsc, node, optional);

	pcmk__order_resource_actions(rsc, RSC_STOP, rsc, RSC_DELETE,
	optional? pe_order_implies_then : pe_order_optional,
	data_set);

	pcmk__order_resource_actions(rsc, RSC_DELETE, rsc, RSC_START,
	optional? pe_order_implies_then : pe_order_optional,
	data_set);

	return TRUE;
	}

	gboolean
	native_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
	gboolean force, pe_working_set_t * data_set)
	{
	enum pe_ordering flags = pe_order_optional;
	char *key = NULL;
	pe_action_t *probe = NULL;
	pe_node_t *running = NULL;
	pe_node_t *allowed = NULL;
	pe_resource_t *top = uber_parent(rsc);

	static const char *rc_promoted = NULL;
	static const char *rc_inactive = NULL;

	if (rc_inactive == NULL) {
	rc_inactive = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
	rc_promoted = pcmk__itoa(PCMK_OCF_RUNNING_PROMOTED);
	}

	CRM_CHECK(node != NULL, return FALSE);
	if (!force && !pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
	pe_rsc_trace(rsc, "Skipping active resource detection for %s", rsc->id);
	return FALSE;
	}

	if (pe__is_guest_or_remote_node(node)) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because Pacemaker Remote nodes cannot run stonith agents",
	rsc->id, node->details->id);
	return FALSE;
	} else if (pe__is_guest_node(node)
	&& pe__resource_contains_guest_node(data_set, rsc)) {
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because guest nodes cannot run resources containing guest nodes",
	rsc->id, node->details->id);
	return FALSE;
	} else if (rsc->is_remote_node) {
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because Pacemaker Remote nodes cannot host remote connections",
	rsc->id, node->details->id);
	return FALSE;
	}
	}

	if (rsc->children) {
	GList *gIter = NULL;
	gboolean any_created = FALSE;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	any_created = child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set)
	\|\| any_created;
	}

	return any_created;

	} else if ((rsc->container) && (!rsc->is_remote_node)) {
	pe_rsc_trace(rsc, "Skipping %s: it is within container %s", rsc->id, rsc->container->id);
	return FALSE;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_rsc_trace(rsc, "Skipping orphan: %s", rsc->id);
	return FALSE;
	}

	// Check whether resource is already known on node
	if (!force && g_hash_table_lookup(rsc->known_on, node->details->id)) {
	pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname);
	return FALSE;
	}

	allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (rsc->exclusive_discover \|\| top->exclusive_discover) {
	if (allowed == NULL) {
	/* exclusive discover is enabled and this node is not in the allowed list. */
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, A", rsc->id, node->details->id);
	return FALSE;
	} else if (allowed->rsc_discover_mode != pe_discover_exclusive) {
	/* exclusive discover is enabled and this node is not marked
	* as a node this resource should be discovered on */
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, B", rsc->id, node->details->id);
	return FALSE;
	}
	}

	if(allowed == NULL && node->rsc_discover_mode == pe_discover_never) {
	/* If this node was allowed to host this resource it would
	* have been explicitly added to the 'allowed_nodes' list.
	* However it wasn't and the node has discovery disabled, so
	* no need to probe for this resource.
	*/
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, C", rsc->id, node->details->id);
	return FALSE;
	}

	if (allowed && allowed->rsc_discover_mode == pe_discover_never) {
	/* this resource is marked as not needing to be discovered on this node */
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, discovery mode", rsc->id, node->details->id);
	return FALSE;
	}

	if (pe__is_guest_node(node)) {
	pe_resource_t *remote = node->details->remote_rsc->container;

	if(remote->role == RSC_ROLE_STOPPED) {
	/* If the container is stopped, then we know anything that
	* might have been inside it is also stopped and there is
	* no need to probe.
	*
	* If we don't know the container's state on the target
	* either:
	*
	* - the container is running, the transition will abort
	* and we'll end up in a different case next time, or
	*
	* - the container is stopped
	*
	* Either way there is no need to probe.
	*
	*/
	if(remote->allocated_to
	&& g_hash_table_lookup(remote->known_on, remote->allocated_to->details->id) == NULL) {
	/* For safety, we order the 'rsc' start after 'remote'
	* has been probed.
	*
	* Using 'top' helps for groups, but we may need to
	* follow the start's ordering chain backwards.
	*/
	pcmk__new_ordering(remote,
	pcmk__op_key(remote->id, RSC_STATUS, 0),
	NULL, top,
	pcmk__op_key(top->id, RSC_START, 0), NULL,
	pe_order_optional, data_set);
	}
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopped",
	rsc->id, node->details->id, remote->id);
	return FALSE;

	/* Here we really we want to check if remote->stop is required,
	* but that information doesn't exist yet
	*/
	} else if(node->details->remote_requires_reset
	\|\| node->details->unclean
	\|\| pcmk_is_set(remote->flags, pe_rsc_failed)
	\|\| remote->next_role == RSC_ROLE_STOPPED
	\|\| (remote->allocated_to
	&& pe_find_node(remote->running_on, remote->allocated_to->details->uname) == NULL)
	) {
	/* The container is stopping or restarting, don't start
	* 'rsc' until 'remote' stops as this also implies that
	* 'rsc' is stopped - avoiding the need to probe
	*/
	pcmk__new_ordering(remote, pcmk__op_key(remote->id, RSC_STOP, 0),
	NULL, top, pcmk__op_key(top->id, RSC_START, 0),
	NULL, pe_order_optional, data_set);
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopping, restarting or moving",
	rsc->id, node->details->id, remote->id);
	return FALSE;
	/* } else {
	* The container is running so there is no problem probing it
	*/
	}
	}

	key = pcmk__op_key(rsc->id, RSC_STATUS, 0);
	probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
	pe__clear_action_flags(probe, pe_action_optional);

	pcmk__order_vs_unfence(rsc, node, probe, pe_order_optional, data_set);

	/*
	* We need to know if it's running_on (not just known_on) this node
	* to correctly determine the target rc.
	*/
	running = pe_find_node_id(rsc->running_on, node->details->id);
	if (running == NULL) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);

	} else if (rsc->role == RSC_ROLE_PROMOTED) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_promoted);
	}

	crm_debug("Probing %s on %s (%s) %d %p", rsc->id, node->details->uname, role2text(rsc->role),
	pcmk_is_set(probe->flags, pe_action_runnable), rsc->running_on);

	if (pcmk__is_unfence_device(rsc, data_set) \|\| !pe_rsc_is_clone(top)) {
	top = rsc;
	} else {
	crm_trace("Probing %s on %s (%s) as %s", rsc->id, node->details->uname, role2text(rsc->role), top->id);
	}

	if (!pcmk_is_set(probe->flags, pe_action_runnable)
	&& (rsc->running_on == NULL)) {
	/* Prevent the start from occurring if rsc isn't active, but
	* don't cause it to stop if it was active already
	*/
	pe__set_order_flags(flags, pe_order_runnable_left);
	}

	pcmk__new_ordering(rsc, NULL, probe, top,
	pcmk__op_key(top->id, RSC_START, 0), NULL, flags,
	data_set);

	// Order the probe before any agent reload
	pcmk__new_ordering(rsc, NULL, probe, top, reload_key(rsc), NULL,
	pe_order_optional, data_set);

	return TRUE;
	}

	void
	native_append_meta(pe_resource_t * rsc, xmlNode * xml)
	{
	char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
	pe_resource_t *parent;

	if (value) {
	char *name = NULL;

	name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
	crm_xml_add(xml, name, value);
	free(name);
	}

	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
	if (value) {
	char *name = NULL;

	name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	for (parent = rsc; parent != NULL; parent = parent->parent) {
	if (parent->container) {
	crm_xml_add(xml, CRM_META"_"XML_RSC_ATTR_CONTAINER, parent->container->id);
	}
	}
	}

	// Primitive implementation of resource_alloc_functions_t:add_utilization()
	void
	pcmk__primitive_add_utilization(pe_resource_t rsc, pe_resource_t orig_rsc,
	GList all_rscs, GHashTable utilization)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return;
	}

	pe_rsc_trace(orig_rsc, "%s: Adding primitive %s as colocated utilization",
	orig_rsc->id, rsc->id);
	pcmk__release_node_capacity(utilization, rsc);
	}
	diff --git a/lib/pacemaker/pcmk_sched_nodes.c b/lib/pacemaker/pcmk_sched_nodes.c
	index b13afd848b..cceda569a6 100644
	--- a/lib/pacemaker/pcmk_sched_nodes.c
	+++ b/lib/pacemaker/pcmk_sched_nodes.c
	@@ -1,255 +1,323 @@
	/*
	- * Copyright 2004-2021 the Pacemaker project contributors
	+ * Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>
	#include <crm/lrmd.h> // lrmd_event_data_t
	#include <crm/common/xml_internal.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Check whether a node is available to run resources
	*
	* \param[in] node Node to check
	*
	* \return true if node is online and not shutting down, unclean, or in standby
	* or maintenance mode, otherwise false
	*/
	bool
	pcmk__node_available(const pe_node_t *node)
	{
	// @TODO Should we add (node->weight >= 0)?
	return (node != NULL) && (node->details != NULL) && node->details->online
	&& !node->details->shutdown && !node->details->unclean
	&& !node->details->standby && !node->details->maintenance;
	}

	/*!
	* \internal
	* \brief Copy a hash table of node objects
	*
	* \param[in] nodes Hash table to copy
	*
	* \return New copy of nodes (or NULL if nodes is NULL)
	*/
	GHashTable *
	pcmk__copy_node_table(GHashTable *nodes)
	{
	GHashTable *new_table = NULL;
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (nodes == NULL) {
	return NULL;
	}
	new_table = pcmk__strkey_table(NULL, free);
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	pe_node_t *new_node = pe__copy_node(node);

	g_hash_table_insert(new_table, (gpointer) new_node->details->id,
	new_node);
	}
	return new_table;
	}

	/*!
	* \internal
	* \brief Copy a list of node objects
	*
	* \param[in] list List to copy
	* \param[in] reset Set copies' scores to 0
	*
	* \return New list of shallow copies of nodes in original list
	*/
	GList *
	pcmk__copy_node_list(const GList *list, bool reset)
	{
	GList *result = NULL;

	for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pe_node_t *new_node = NULL;
	pe_node_t this_node = (pe_node_t ) gIter->data;

	new_node = pe__copy_node(this_node);
	if (reset) {
	new_node->weight = 0;
	}
	result = g_list_prepend(result, new_node);
	}
	return result;
	}

	struct node_weight_s {
	pe_node_t *active;
	pe_working_set_t *data_set;
	};

	/*!
	* \internal
	* \brief Compare two nodes for allocation desirability
	*
	* Given two nodes, check which one is more preferred by allocation criteria
	* such as node weight and utilization.
	*
	* \param[in] a First node to compare
	* \param[in] b Second node to compare
	* \param[in] data Sort data (as struct node_weight_s *)
	*
	* \return -1 if \p a is preferred, +1 if \p b is preferred, or 0 if they are
	* equally preferred
	*/
	static gint
	compare_nodes(gconstpointer a, gconstpointer b, gpointer data)
	{
	const pe_node_t node1 = (const pe_node_t ) a;
	const pe_node_t node2 = (const pe_node_t ) b;
	struct node_weight_s *nw = data;

	int node1_weight = 0;
	int node2_weight = 0;

	int result = 0;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	// Compare node weights

	node1_weight = pcmk__node_available(node1)? node1->weight : -INFINITY;
	node2_weight = pcmk__node_available(node2)? node2->weight : -INFINITY;

	if (node1_weight > node2_weight) {
	crm_trace("%s (%d) > %s (%d) : weight",
	node1->details->uname, node1_weight, node2->details->uname,
	node2_weight);
	return -1;
	}

	if (node1_weight < node2_weight) {
	crm_trace("%s (%d) < %s (%d) : weight",
	node1->details->uname, node1_weight, node2->details->uname,
	node2_weight);
	return 1;
	}

	crm_trace("%s (%d) == %s (%d) : weight",
	node1->details->uname, node1_weight, node2->details->uname,
	node2_weight);

	// If appropriate, compare node utilization

	if (pcmk__str_eq(nw->data_set->placement_strategy, "minimal",
	pcmk__str_casei)) {
	goto equal;
	}

	if (pcmk__str_eq(nw->data_set->placement_strategy, "balanced",
	pcmk__str_casei)) {
	result = pcmk__compare_node_capacities(node1, node2);
	if (result < 0) {
	crm_trace("%s > %s : capacity (%d)",
	node1->details->uname, node2->details->uname, result);
	return -1;
	} else if (result > 0) {
	crm_trace("%s < %s : capacity (%d)",
	node1->details->uname, node2->details->uname, result);
	return 1;
	}
	}

	// Compare number of allocated resources

	if (node1->details->num_resources < node2->details->num_resources) {
	crm_trace("%s (%d) > %s (%d) : resources",
	node1->details->uname, node1->details->num_resources,
	node2->details->uname, node2->details->num_resources);
	return -1;

	} else if (node1->details->num_resources > node2->details->num_resources) {
	crm_trace("%s (%d) < %s (%d) : resources",
	node1->details->uname, node1->details->num_resources,
	node2->details->uname, node2->details->num_resources);
	return 1;
	}

	// Check whether one node is already running desired resource

	if (nw->active != NULL) {
	if (nw->active->details == node1->details) {
	crm_trace("%s (%d) > %s (%d) : active",
	node1->details->uname, node1->details->num_resources,
	node2->details->uname, node2->details->num_resources);
	return -1;
	} else if (nw->active->details == node2->details) {
	crm_trace("%s (%d) < %s (%d) : active",
	node1->details->uname, node1->details->num_resources,
	node2->details->uname, node2->details->num_resources);
	return 1;
	}
	}

	// If all else is equal, prefer node with lowest-sorting name
	equal:
	crm_trace("%s = %s", node1->details->uname, node2->details->uname);
	return strcmp(node1->details->uname, node2->details->uname);
	}

	/*!
	* \internal
	* \brief Sort a list of nodes by allocation desirability
	*
	* \param[in] nodes Node list to sort
	* \param[in] active_node If not NULL, node currently running resource
	* \param[in] data_set Cluster working set
	*
	* \return New head of sorted list
	*/
	GList *
	pcmk__sort_nodes(GList nodes, pe_node_t active_node,
	pe_working_set_t *data_set)
	{
	struct node_weight_s nw = { active_node, data_set };

	return g_list_sort_with_data(nodes, compare_nodes, &nw);
	}

	/*!
	* \internal
	* \brief Check whether any node is available to run resources
	*
	* \param[in] nodes Nodes to check
	*
	* \return true if any node in \p nodes is available to run resources,
	* otherwise false
	*/
	bool
	pcmk__any_node_available(GHashTable *nodes)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (nodes == NULL) {
	return false;
	}
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if ((node->weight >= 0) && pcmk__node_available(node)) {
	return true;
	}
	}
	return false;
	}
	+
	+/*!
	+ * \internal
	+ * \brief Add node attribute value to an integer, if it is a health attribute
	+ *
	+ * \param[in] key Name of node attribute
	+ * \param[in] value String value of node attribute
	+ * \param[in] user_data Address of integer to which \p value should be added
	+ * if \p key is a node health attribute
	+ */
	+static void
	+add_node_health_value(gpointer key, gpointer value, gpointer user_data)
	+{
	+ if (pcmk__starts_with((const char *) key, "#health")) {
	+ int score = char2score((const char *) value);
	+ int health = (int ) user_data;
	+
	+ health = pcmk__add_scores(score, health);
	+ }
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Apply node health values for all nodes in cluster
	+ *
	+ * \param[in] data_set Cluster working set
	+ */
	+void
	+pcmk__apply_node_health(pe_working_set_t *data_set)
	+{
	+ const char *health_strategy = pe_pref(data_set->config_hash,
	+ "node-health-strategy");
	+ int base_health = 0;
	+
	+ if (pcmk__str_eq(health_strategy, "none",
	+ pcmk__str_null_matches\|pcmk__str_casei)) {
	+ return;
	+ }
	+ crm_info("Applying node health strategy '%s'", health_strategy);
	+
	+ // The progressive strategy can use a base health score
	+ if (pcmk__str_eq(health_strategy, "progressive", pcmk__str_casei)) {
	+ base_health = char2score(pe_pref(data_set->config_hash,
	+ "node-health-base"));
	+ }
	+
	+ for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	+ pe_node_t node = (pe_node_t ) iter->data;
	+ int health = base_health;
	+
	+ // Calculate overall node health score as sum of all health values
	+ g_hash_table_foreach(node->details->attrs, add_node_health_value,
	+ &health);
	+
	+ // A health score of 0 has no effect
	+ if (health == 0) {
	+ continue;
	+ }
	+ crm_info("Node %s overall system health is %d",
	+ node->details->uname, health);
	+
	+ // Use node health as a location score for each resource on the node
	+ for (GList *rsc = data_set->resources; rsc != NULL; rsc = rsc->next) {
	+ pcmk__new_location(health_strategy, (pe_resource_t *) rsc->data,
	+ health, NULL, node, data_set);
	+ }
	+ }
	+}
	diff --git a/lib/pacemaker/pcmk_sched_promotable.c b/lib/pacemaker/pcmk_sched_promotable.c
	index 75fdf8ada6..90a25f20e5 100644
	--- a/lib/pacemaker/pcmk_sched_promotable.c
	+++ b/lib/pacemaker/pcmk_sched_promotable.c
	@@ -1,1057 +1,1057 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	#define VARIANT_CLONE 1
	#include <lib/pengine/variant.h>

	extern gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set);

	extern bool pcmk__is_daemon;

	static void
	child_promoting_constraints(clone_variant_data_t * clone_data, enum pe_ordering type,
	pe_resource_t * rsc, pe_resource_t * child, pe_resource_t * last,
	pe_working_set_t * data_set)
	{
	if (child == NULL) {
	if (clone_data->ordered && last != NULL) {
	pe_rsc_trace(rsc, "Ordered version (last node)");
	/* last child promote before promoted started */
	pcmk__order_resource_actions(last, RSC_PROMOTE, rsc, RSC_PROMOTED,
	type, data_set);
	}
	return;
	}

	/* child promote before global promoted */
	pcmk__order_resource_actions(child, RSC_PROMOTE, rsc, RSC_PROMOTED, type,
	data_set);

	/* global promote before child promote */
	pcmk__order_resource_actions(rsc, RSC_PROMOTE, child, RSC_PROMOTE, type,
	data_set);

	if (clone_data->ordered) {
	pe_rsc_trace(rsc, "Ordered version");
	if (last == NULL) {
	/* global promote before first child promote */
	last = rsc;

	}
	/* else: child/child relative promote */
	pcmk__order_starts(last, child, type, data_set);
	pcmk__order_resource_actions(last, RSC_PROMOTE, child, RSC_PROMOTE,
	type, data_set);

	} else {
	pe_rsc_trace(rsc, "Un-ordered version");
	}
	}

	static void
	child_demoting_constraints(clone_variant_data_t * clone_data, enum pe_ordering type,
	pe_resource_t * rsc, pe_resource_t * child, pe_resource_t * last,
	pe_working_set_t * data_set)
	{
	if (child == NULL) {
	if (clone_data->ordered && last != NULL) {
	pe_rsc_trace(rsc, "Ordered version (last node)");
	/* global demote before first child demote */
	pcmk__order_resource_actions(rsc, RSC_DEMOTE, last, RSC_DEMOTE,
	pe_order_optional, data_set);
	}
	return;
	}

	/* child demote before global demoted */
	pcmk__order_resource_actions(child, RSC_DEMOTE, rsc, RSC_DEMOTED,
	pe_order_implies_then_printed, data_set);

	/* global demote before child demote */
	pcmk__order_resource_actions(rsc, RSC_DEMOTE, child, RSC_DEMOTE,
	pe_order_implies_first_printed, data_set);

	if (clone_data->ordered && last != NULL) {
	pe_rsc_trace(rsc, "Ordered version");

	/* child/child relative demote */
	pcmk__order_resource_actions(child, RSC_DEMOTE, last, RSC_DEMOTE, type,
	data_set);

	} else if (clone_data->ordered) {
	pe_rsc_trace(rsc, "Ordered version (1st node)");
	/* first child stop before global stopped */
	pcmk__order_resource_actions(child, RSC_DEMOTE, rsc, RSC_DEMOTED, type,
	data_set);

	} else {
	pe_rsc_trace(rsc, "Un-ordered version");
	}
	}

	static void
	check_promotable_actions(pe_resource_t rsc, gboolean demoting,
	gboolean *promoting)
	{
	GList *gIter = NULL;

	if (rsc->children) {
	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	check_promotable_actions(child, demoting, promoting);
	}
	return;
	}

	CRM_ASSERT(demoting != NULL);
	CRM_ASSERT(promoting != NULL);

	gIter = rsc->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (promoting && demoting) {
	return;

	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	continue;

	} else if (pcmk__str_eq(RSC_DEMOTE, action->task, pcmk__str_casei)) {
	*demoting = TRUE;

	} else if (pcmk__str_eq(RSC_PROMOTE, action->task, pcmk__str_casei)) {
	*promoting = TRUE;
	}
	}
	}

	static void
	apply_promoted_location(pe_resource_t child, GList location_constraints,
	pe_node_t *chosen)
	{
	CRM_CHECK(child && chosen, return);
	for (GList *gIter = location_constraints; gIter; gIter = gIter->next) {
	pe_node_t *cons_node = NULL;
	pe__location_t *cons = gIter->data;

	if (cons->role_filter == RSC_ROLE_PROMOTED) {
	pe_rsc_trace(child, "Applying %s to %s", cons->id, child->id);
	cons_node = pe_find_node_id(cons->node_list_rh, chosen->details->id);
	}
	if (cons_node != NULL) {
	- int new_priority = pe__add_scores(child->priority,
	- cons_node->weight);
	+ int new_priority = pcmk__add_scores(child->priority,
	+ cons_node->weight);

	pe_rsc_trace(child, "\t%s[%s]: %d -> %d (%d)",
	child->id, cons_node->details->uname, child->priority,
	new_priority, cons_node->weight);
	child->priority = new_priority;
	}
	}
	}

	static pe_node_t *
	guest_location(pe_node_t *guest_node)
	{
	pe_resource_t *guest = guest_node->details->remote_rsc->container;

	return guest->fns->location(guest, NULL, FALSE);
	}

	static pe_node_t *
	node_to_be_promoted_on(pe_resource_t *rsc)
	{
	pe_node_t *node = NULL;
	pe_node_t *local_node = NULL;
	pe_resource_t *parent = uber_parent(rsc);
	clone_variant_data_t *clone_data = NULL;

	#if 0
	enum rsc_role_e role = RSC_ROLE_UNKNOWN;

	role = rsc->fns->state(rsc, FALSE);
	crm_info("%s role: %s", rsc->id, role2text(role));
	#endif

	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	if (node_to_be_promoted_on(child) == NULL) {
	pe_rsc_trace(rsc, "Child %s of %s can't be promoted", child->id, rsc->id);
	return NULL;
	}
	}
	}

	node = rsc->fns->location(rsc, NULL, FALSE);
	if (node == NULL) {
	pe_rsc_trace(rsc, "%s cannot be promoted: not allocated", rsc->id);
	return NULL;

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	if (rsc->fns->state(rsc, TRUE) == RSC_ROLE_PROMOTED) {
	crm_notice("Forcing unmanaged instance %s to remain promoted on %s",
	rsc->id, node->details->uname);

	} else {
	return NULL;
	}

	} else if (rsc->priority < 0) {
	pe_rsc_trace(rsc, "%s cannot be promoted: preference: %d",
	rsc->id, rsc->priority);
	return NULL;

	} else if (!pcmk__node_available(node)) {
	crm_trace("Node can't run any resources: %s", node->details->uname);
	return NULL;

	/* @TODO It's possible this check should be done in pcmk__node_available()
	* instead. We should investigate all its callers to figure out whether that
	* would be a good idea.
	*/
	} else if (pe__is_guest_node(node) && (guest_location(node) == NULL)) {
	pe_rsc_trace(rsc, "%s cannot be promoted: guest %s not allocated",
	rsc->id, node->details->remote_rsc->container->id);
	return NULL;
	}

	get_clone_variant_data(clone_data, parent);
	local_node = pe_hash_table_lookup(parent->allowed_nodes, node->details->id);

	if (local_node == NULL) {
	crm_err("%s cannot run on %s: node not allowed", rsc->id, node->details->uname);
	return NULL;

	} else if ((local_node->count < clone_data->promoted_node_max)
	\|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	return local_node;

	} else {
	pe_rsc_trace(rsc, "%s cannot be promoted on %s: node full",
	rsc->id, node->details->uname);
	}

	return NULL;
	}

	static gint
	sort_promotable_instance(gconstpointer a, gconstpointer b, gpointer data_set)
	{
	int rc;
	enum rsc_role_e role1 = RSC_ROLE_UNKNOWN;
	enum rsc_role_e role2 = RSC_ROLE_UNKNOWN;

	const pe_resource_t resource1 = (const pe_resource_t )a;
	const pe_resource_t resource2 = (const pe_resource_t )b;

	CRM_ASSERT(resource1 != NULL);
	CRM_ASSERT(resource2 != NULL);

	role1 = resource1->fns->state(resource1, TRUE);
	role2 = resource2->fns->state(resource2, TRUE);

	rc = sort_rsc_index(a, b);
	if (rc != 0) {
	crm_trace("%s %c %s (index)", resource1->id, rc < 0 ? '<' : '>', resource2->id);
	return rc;
	}

	if (role1 > role2) {
	crm_trace("%s %c %s (role)", resource1->id, '<', resource2->id);
	return -1;

	} else if (role1 < role2) {
	crm_trace("%s %c %s (role)", resource1->id, '>', resource2->id);
	return 1;
	}

	return sort_clone_instance(a, b, data_set);
	}

	static void
	promotion_order(pe_resource_t rsc, pe_working_set_t data_set)
	{
	GList *gIter = NULL;
	pe_node_t *node = NULL;
	pe_node_t *chosen = NULL;
	clone_variant_data_t *clone_data = NULL;
	char score[33];
	size_t len = sizeof(score);

	get_clone_variant_data(clone_data, rsc);

	if (clone_data->added_promoted_constraints) {
	return;
	}
	clone_data->added_promoted_constraints = true;
	pe_rsc_trace(rsc, "Merging weights for %s", rsc->id);
	pe__set_resource_flags(rsc, pe_rsc_merging);

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	pe_rsc_trace(rsc, "Sort index: %s = %d", child->id, child->sort_index);
	}
	pe__show_node_weights(true, rsc, "Before", rsc->allowed_nodes, data_set);

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	chosen = child->fns->location(child, NULL, FALSE);
	if (chosen == NULL \|\| child->sort_index < 0) {
	pe_rsc_trace(rsc, "Skipping %s", child->id);
	continue;
	}

	node = (pe_node_t *) pe_hash_table_lookup(rsc->allowed_nodes, chosen->details->id);
	CRM_ASSERT(node != NULL);
	// Add promotion preferences and rsc_location scores when role=Promoted
	score2char_stack(child->sort_index, score, len);
	pe_rsc_trace(rsc, "Adding %s to %s from %s", score,
	node->details->uname, child->id);
	- node->weight = pe__add_scores(child->sort_index, node->weight);
	+ node->weight = pcmk__add_scores(child->sort_index, node->weight);
	}

	pe__show_node_weights(true, rsc, "Middle", rsc->allowed_nodes, data_set);

	gIter = rsc->rsc_cons;
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	/* (Re-)add location preferences of resources that a promoted instance
	* should/must be colocated with.
	*/
	if (constraint->dependent_role == RSC_ROLE_PROMOTED) {
	enum pe_weights flags = constraint->score == INFINITY ? 0 : pe_weights_rollback;

	pe_rsc_trace(rsc, "RHS: %s with %s: %d",
	constraint->dependent->id, constraint->primary->id,
	constraint->score);
	rsc->allowed_nodes = constraint->primary->cmds->merge_weights(
	constraint->primary, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute,
	constraint->score / (float) INFINITY, flags);
	}
	}

	gIter = rsc->rsc_cons_lhs;
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	if (!pcmk__colocation_has_influence(constraint, NULL)) {
	continue;
	}

	/* (Re-)add location preferences of resources that wish to be colocated
	* with a promoted instance.
	*/
	if (constraint->primary_role == RSC_ROLE_PROMOTED) {
	pe_rsc_trace(rsc, "LHS: %s with %s: %d",
	constraint->dependent->id, constraint->primary->id,
	constraint->score);
	rsc->allowed_nodes = constraint->dependent->cmds->merge_weights(
	constraint->dependent, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute,
	constraint->score / (float) INFINITY,
	pe_weights_rollback\|pe_weights_positive);
	}
	}

	gIter = rsc->rsc_tickets;
	for (; gIter != NULL; gIter = gIter->next) {
	rsc_ticket_t rsc_ticket = (rsc_ticket_t ) gIter->data;

	if ((rsc_ticket->role_lh == RSC_ROLE_PROMOTED)
	&& (rsc_ticket->ticket->granted == FALSE \|\| rsc_ticket->ticket->standby)) {
	resource_location(rsc, NULL, -INFINITY, "__stateful_without_ticket__", data_set);
	}
	}

	pe__show_node_weights(true, rsc, "After", rsc->allowed_nodes, data_set);

	/* write them back and sort */

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	chosen = child->fns->location(child, NULL, FALSE);
	if (!pcmk_is_set(child->flags, pe_rsc_managed)
	&& (child->next_role == RSC_ROLE_PROMOTED)) {
	child->sort_index = INFINITY;

	} else if (chosen == NULL \|\| child->sort_index < 0) {
	pe_rsc_trace(rsc, "%s: %d", child->id, child->sort_index);

	} else {
	node = (pe_node_t *) pe_hash_table_lookup(rsc->allowed_nodes, chosen->details->id);
	CRM_ASSERT(node != NULL);

	child->sort_index = node->weight;
	}
	pe_rsc_trace(rsc, "Set sort index: %s = %d", child->id, child->sort_index);
	}

	rsc->children = g_list_sort_with_data(rsc->children,
	sort_promotable_instance, data_set);
	pe__clear_resource_flags(rsc, pe_rsc_merging);
	}

	static gboolean
	filter_anonymous_instance(pe_resource_t rsc, const pe_node_t node)
	{
	GList *rIter = NULL;
	char *key = clone_strip(rsc->id);
	pe_resource_t *parent = uber_parent(rsc);

	for (rIter = parent->children; rIter; rIter = rIter->next) {
	/* If there is an active instance on the node, only it receives the
	* promotion score. Use ->find_rsc() in case this is a cloned group.
	*/
	pe_resource_t *child = rIter->data;
	pe_resource_t *active = parent->fns->find_rsc(child, key, node, pe_find_clone\|pe_find_current);

	if(rsc == active) {
	pe_rsc_trace(rsc, "Found %s for %s active on %s: done", active->id, key, node->details->uname);
	free(key);
	return TRUE;
	} else if(active) {
	pe_rsc_trace(rsc, "Found %s for %s on %s: not %s", active->id, key, node->details->uname, rsc->id);
	free(key);
	return FALSE;
	} else {
	pe_rsc_trace(rsc, "%s on %s: not active", key, node->details->uname);
	}
	}

	for (rIter = parent->children; rIter; rIter = rIter->next) {
	pe_resource_t *child = rIter->data;

	/*
	* We know it's not running, but any score will still count if
	* the instance has been probed on $node
	*
	* Again use ->find_rsc() because we might be a cloned group
	* and knowing that other members of the group are known here
	* implies nothing
	*/
	rsc = parent->fns->find_rsc(child, key, NULL, pe_find_clone);
	CRM_LOG_ASSERT(rsc);
	if(rsc) {
	pe_rsc_trace(rsc, "Checking %s for %s on %s", rsc->id, key, node->details->uname);
	if (g_hash_table_lookup(rsc->known_on, node->details->id)) {
	free(key);
	return TRUE;
	}
	}
	}
	free(key);
	return FALSE;
	}

	static const char *
	lookup_promotion_score(pe_resource_t rsc, const pe_node_t node, const char *name)
	{
	const char *attr_value = NULL;

	if (node && name) {
	char *attr_name = pcmk_promotion_score_name(name);

	attr_value = pe_node_attribute_calculated(node, attr_name, rsc);
	free(attr_name);
	}
	return attr_value;
	}

	static int
	promotion_score(pe_resource_t rsc, const pe_node_t node, int not_set_value)
	{
	char *name = rsc->id;
	const char *attr_value = NULL;
	int score = not_set_value;
	pe_node_t *match = NULL;

	CRM_CHECK(node != NULL, return not_set_value);

	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;
	int c_score = promotion_score(child, node, not_set_value);

	if (score == not_set_value) {
	score = c_score;
	} else {
	score += c_score;
	}
	}
	return score;
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)
	&& filter_anonymous_instance(rsc, node)) {

	pe_rsc_trace(rsc, "Anonymous clone %s is allowed on %s", rsc->id, node->details->uname);

	} else if (rsc->running_on \|\| g_hash_table_size(rsc->known_on)) {
	/* If we've probed and/or started the resource anywhere, consider
	* promotion scores only from nodes where we know the status. However,
	* if the status of all nodes is unknown (e.g. cluster startup),
	* skip this code, to make sure we take into account any permanent
	* promotion scores set previously.
	*/
	pe_node_t *known = pe_hash_table_lookup(rsc->known_on, node->details->id);

	match = pe_find_node_id(rsc->running_on, node->details->id);
	if ((match == NULL) && (known == NULL)) {
	pe_rsc_trace(rsc, "skipping %s (aka. %s) promotion score on %s because inactive",
	rsc->id, rsc->clone_name, node->details->uname);
	return score;
	}
	}

	match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match == NULL) {
	return score;

	} else if (match->weight < 0) {
	pe_rsc_trace(rsc, "%s on %s has score: %d - ignoring",
	rsc->id, match->details->uname, match->weight);
	return score;
	}

	if (rsc->clone_name) {
	/* Use the name the lrm knows this resource as,
	* since that's what crm_attribute --promotion would have used
	*/
	name = rsc->clone_name;
	}

	attr_value = lookup_promotion_score(rsc, node, name);
	pe_rsc_trace(rsc, "promotion score for %s on %s = %s",
	name, node->details->uname, crm_str(attr_value));

	if ((attr_value == NULL) && !pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	/* If we don't have any LRM history yet, we won't have clone_name -- in
	* that case, for anonymous clones, try the resource name without any
	* instance number.
	*/
	name = clone_strip(rsc->id);
	if (strcmp(rsc->id, name)) {
	attr_value = lookup_promotion_score(rsc, node, name);
	pe_rsc_trace(rsc, "stripped promotion score for %s on %s = %s",
	name, node->details->uname, crm_str(attr_value));
	}
	free(name);
	}

	if (attr_value != NULL) {
	score = char2score(attr_value);
	}

	return score;
	}

	void
	pcmk__add_promotion_scores(pe_resource_t *rsc)
	{
	int score, new_score;
	GList *gIter = rsc->children;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	if (clone_data->added_promotion_scores) {
	/* Make sure we only do this once */
	return;
	}

	clone_data->added_promotion_scores = true;

	for (; gIter != NULL; gIter = gIter->next) {
	GHashTableIter iter;
	pe_node_t *node = NULL;
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	g_hash_table_iter_init(&iter, child_rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (!pcmk__node_available(node)) {
	/* This node will never be promoted, so don't apply the
	* promotion score, as that may lead to clone shuffling.
	*/
	continue;
	}

	score = promotion_score(child_rsc, node, 0);
	if (score > 0) {
	- new_score = pe__add_scores(node->weight, score);
	+ new_score = pcmk__add_scores(node->weight, score);
	if (new_score != node->weight) {
	pe_rsc_trace(rsc, "\t%s: Updating preference for %s (%d->%d)",
	child_rsc->id, node->details->uname, node->weight, new_score);
	node->weight = new_score;
	}
	}

	new_score = QB_MAX(child_rsc->priority, score);
	if (new_score != child_rsc->priority) {
	pe_rsc_trace(rsc, "\t%s: Updating priority (%d->%d)",
	child_rsc->id, child_rsc->priority, new_score);
	child_rsc->priority = new_score;
	}
	}
	}
	}

	static void
	set_role_unpromoted(pe_resource_t *rsc, bool current)
	{
	GList *gIter = rsc->children;

	if (current) {
	if (rsc->role == RSC_ROLE_STARTED) {
	rsc->role = RSC_ROLE_UNPROMOTED;
	}

	} else {
	GList *allocated = NULL;

	rsc->fns->location(rsc, &allocated, FALSE);
	pe__set_next_role(rsc, (allocated? RSC_ROLE_UNPROMOTED : RSC_ROLE_STOPPED),
	"unpromoted instance");
	g_list_free(allocated);
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	set_role_unpromoted(child_rsc, current);
	}
	}

	static void
	set_role_promoted(pe_resource_t *rsc, gpointer user_data)
	{
	if (rsc->next_role == RSC_ROLE_UNKNOWN) {
	pe__set_next_role(rsc, RSC_ROLE_PROMOTED, "promoted instance");
	}

	g_list_foreach(rsc->children, (GFunc) set_role_promoted, NULL);
	}

	pe_node_t *
	pcmk__set_instance_roles(pe_resource_t rsc, pe_working_set_t data_set)
	{
	int promoted = 0;
	GList *gIter = NULL;
	GList *gIter2 = NULL;
	GHashTableIter iter;
	pe_node_t *node = NULL;
	pe_node_t *chosen = NULL;
	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;
	char score[33];
	size_t len = sizeof(score);
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	// Repurpose count to track the number of promoted instances allocated
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	node->count = 0;
	}

	/*
	* assign priority
	*/
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	GList *list = NULL;
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	pe_rsc_trace(rsc, "Assigning priority for %s: %s", child_rsc->id,
	role2text(child_rsc->next_role));

	if (child_rsc->fns->state(child_rsc, TRUE) == RSC_ROLE_STARTED) {
	set_role_unpromoted(child_rsc, true);
	}

	chosen = child_rsc->fns->location(child_rsc, &list, FALSE);
	if (pcmk__list_of_multiple(list)) {
	pcmk__config_err("Cannot promote non-colocated child %s",
	child_rsc->id);
	}

	g_list_free(list);
	if (chosen == NULL) {
	continue;
	}

	next_role = child_rsc->fns->state(child_rsc, FALSE);
	switch (next_role) {
	case RSC_ROLE_STARTED:
	case RSC_ROLE_UNKNOWN:
	/*
	* Default to -1 if no value is set
	*
	* This allows instances eligible for promotion to be specified
	* based solely on rsc_location constraints,
	* but prevents anyone from being promoted if
	* neither a constraint nor a promotion score is present
	*/
	child_rsc->priority = promotion_score(child_rsc, chosen, -1);
	break;

	case RSC_ROLE_UNPROMOTED:
	case RSC_ROLE_STOPPED:
	child_rsc->priority = -INFINITY;
	break;
	case RSC_ROLE_PROMOTED:
	/* We will arrive here if we're re-creating actions after a stonith
	*/
	break;
	default:
	CRM_CHECK(FALSE /* unhandled */ ,
	crm_err("Unknown resource role: %d for %s", next_role, child_rsc->id));
	}

	apply_promoted_location(child_rsc, child_rsc->rsc_location, chosen);
	apply_promoted_location(child_rsc, rsc->rsc_location, chosen);

	for (gIter2 = child_rsc->rsc_cons; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) gIter2->data;

	child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->primary, cons,
	data_set);
	}

	child_rsc->sort_index = child_rsc->priority;
	pe_rsc_trace(rsc, "Assigning priority for %s: %d", child_rsc->id, child_rsc->priority);

	if (next_role == RSC_ROLE_PROMOTED) {
	child_rsc->sort_index = INFINITY;
	}
	}

	pe__show_node_weights(true, rsc, "Pre merge", rsc->allowed_nodes, data_set);
	promotion_order(rsc, data_set);

	// Choose the first N eligible instances to be promoted
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	score2char_stack(child_rsc->sort_index, score, len);

	chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
	if (pcmk_is_set(data_set->flags, pe_flag_show_scores) && !pcmk__is_daemon) {
	if (data_set->priv != NULL) {
	pcmk__output_t *out = data_set->priv;
	out->message(out, "promotion-score", child_rsc, chosen, score);
	}

	} else {
	pe_rsc_trace(rsc, "%s promotion score on %s: %s", child_rsc->id,
	(chosen? chosen->details->uname : "none"), score);
	}

	chosen = NULL; /* nuke 'chosen' so that we don't promote more than the
	* required number of instances
	*/

	if (child_rsc->sort_index < 0) {
	pe_rsc_trace(rsc, "Not supposed to promote child: %s", child_rsc->id);

	} else if ((promoted < clone_data->promoted_max)
	\|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	chosen = node_to_be_promoted_on(child_rsc);
	}

	pe_rsc_debug(rsc, "%s promotion score: %d", child_rsc->id, child_rsc->priority);

	if (chosen == NULL) {
	set_role_unpromoted(child_rsc, false);
	continue;

	} else if ((child_rsc->role < RSC_ROLE_PROMOTED)
	&& !pcmk_is_set(data_set->flags, pe_flag_have_quorum)
	&& data_set->no_quorum_policy == no_quorum_freeze) {
	crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
	child_rsc->id, role2text(child_rsc->role), role2text(child_rsc->next_role));
	set_role_unpromoted(child_rsc, false);
	continue;
	}

	chosen->count++;
	pe_rsc_info(rsc, "Promoting %s (%s %s)",
	child_rsc->id, role2text(child_rsc->role), chosen->details->uname);
	set_role_promoted(child_rsc, NULL);
	promoted++;
	}

	pe_rsc_info(rsc, "%s: Promoted %d instances of a possible %d",
	rsc->id, promoted, clone_data->promoted_max);

	return NULL;
	}

	void
	create_promotable_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_action_t *action = NULL;
	GList *gIter = rsc->children;
	pe_action_t *action_complete = NULL;
	gboolean any_promoting = FALSE;
	gboolean any_demoting = FALSE;
	pe_resource_t *last_promote_rsc = NULL;
	pe_resource_t *last_demote_rsc = NULL;

	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pe_rsc_debug(rsc, "Creating actions for %s", rsc->id);

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean child_promoting = FALSE;
	gboolean child_demoting = FALSE;
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	pe_rsc_trace(rsc, "Creating actions for %s", child_rsc->id);
	child_rsc->cmds->create_actions(child_rsc, data_set);
	check_promotable_actions(child_rsc, &child_demoting, &child_promoting);

	any_demoting = any_demoting \|\| child_demoting;
	any_promoting = any_promoting \|\| child_promoting;
	pe_rsc_trace(rsc, "Created actions for %s: %d %d", child_rsc->id, child_promoting,
	child_demoting);
	}

	/* promote */
	action = pcmk__new_rsc_pseudo_action(rsc, RSC_PROMOTE, !any_promoting,
	true);
	action_complete = pcmk__new_rsc_pseudo_action(rsc, RSC_PROMOTED,
	!any_promoting, true);
	action_complete->priority = INFINITY;

	child_promoting_constraints(clone_data, pe_order_optional,
	rsc, NULL, last_promote_rsc, data_set);

	if (clone_data->promote_notify == NULL) {
	clone_data->promote_notify = pcmk__clone_notif_pseudo_ops(rsc,
	RSC_PROMOTE,
	action,
	action_complete);
	}

	/* demote */
	action = pcmk__new_rsc_pseudo_action(rsc, RSC_DEMOTE, !any_demoting, true);
	action_complete = pcmk__new_rsc_pseudo_action(rsc, RSC_DEMOTED,
	!any_demoting, true);
	action_complete->priority = INFINITY;

	child_demoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_demote_rsc, data_set);

	if (clone_data->demote_notify == NULL) {
	clone_data->demote_notify = pcmk__clone_notif_pseudo_ops(rsc,
	RSC_DEMOTE,
	action,
	action_complete);

	if (clone_data->promote_notify) {
	/* If we ever wanted groups to have notifications we'd need to move this to native_internal_constraints() one day
	* Requires exposing *_notify
	*/
	order_actions(clone_data->stop_notify->post_done, clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->start_notify->post_done, clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done, clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done, clone_data->start_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done, clone_data->stop_notify->pre,
	pe_order_optional);
	}
	}

	/* restore the correct priority */

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->priority = rsc->priority;
	}
	}

	void
	promote_demote_constraints(pe_resource_t rsc, pe_working_set_t data_set)
	{
	/* global stopped before start */
	pcmk__order_resource_actions(rsc, RSC_STOPPED, rsc, RSC_START,
	pe_order_optional, data_set);

	/* global stopped before promote */
	pcmk__order_resource_actions(rsc, RSC_STOPPED, rsc, RSC_PROMOTE,
	pe_order_optional, data_set);

	/* global demoted before start */
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_START,
	pe_order_optional, data_set);

	/* global started before promote */
	pcmk__order_resource_actions(rsc, RSC_STARTED, rsc, RSC_PROMOTE,
	pe_order_optional, data_set);

	/* global demoted before stop */
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_STOP,
	pe_order_optional, data_set);

	/* global demote before demoted */
	pcmk__order_resource_actions(rsc, RSC_DEMOTE, rsc, RSC_DEMOTED,
	pe_order_optional, data_set);

	/* global demoted before promote */
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE,
	pe_order_optional, data_set);
	}


	void
	promotable_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = rsc->children;
	pe_resource_t *last_rsc = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	promote_demote_constraints(rsc, data_set);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	/* child demote before promote */
	pcmk__order_resource_actions(child_rsc, RSC_DEMOTE, child_rsc,
	RSC_PROMOTE, pe_order_optional, data_set);

	child_promoting_constraints(clone_data, pe_order_optional,
	rsc, child_rsc, last_rsc, data_set);

	child_demoting_constraints(clone_data, pe_order_optional,
	rsc, child_rsc, last_rsc, data_set);

	last_rsc = child_rsc;
	}
	}

	static void
	node_hash_update_one(GHashTable * hash, pe_node_t * other, const char *attr, int score)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	const char *value = NULL;

	if (other == NULL) {
	return;

	} else if (attr == NULL) {
	attr = CRM_ATTR_UNAME;
	}

	value = pe_node_attribute_raw(other, attr);
	g_hash_table_iter_init(&iter, hash);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	const char *tmp = pe_node_attribute_raw(node, attr);

	if (pcmk__str_eq(value, tmp, pcmk__str_casei)) {
	crm_trace("%s: %d + %d", node->details->uname, node->weight, other->weight);
	- node->weight = pe__add_scores(node->weight, score);
	+ node->weight = pcmk__add_scores(node->weight, score);
	}
	}
	}

	void
	promotable_colocation_rh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;

	if (pcmk_is_set(dependent->flags, pe_rsc_provisional)) {
	GList *affected_nodes = NULL;

	for (gIter = primary->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	pe_node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
	enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, FALSE);

	pe_rsc_trace(primary, "Processing: %s", child_rsc->id);
	if ((chosen != NULL) && (next_role == constraint->primary_role)) {
	pe_rsc_trace(primary, "Applying: %s %s %s %d", child_rsc->id,
	role2text(next_role), chosen->details->uname, constraint->score);
	if (constraint->score < INFINITY) {
	node_hash_update_one(dependent->allowed_nodes, chosen,
	constraint->node_attribute, constraint->score);
	}
	affected_nodes = g_list_prepend(affected_nodes, chosen);
	}
	}

	/* Only do this if it's not a promoted-with-promoted colocation. Doing
	* this unconditionally would prevent unpromoted instances from being
	* started.
	*/
	if ((constraint->dependent_role != RSC_ROLE_PROMOTED)
	\|\| (constraint->primary_role != RSC_ROLE_PROMOTED)) {

	if (constraint->score >= INFINITY) {
	node_list_exclude(dependent->allowed_nodes, affected_nodes,
	TRUE);
	}
	}
	g_list_free(affected_nodes);

	} else if (constraint->dependent_role == RSC_ROLE_PROMOTED) {
	pe_resource_t *primary_instance;

	primary_instance = find_compatible_child(dependent, primary,
	constraint->primary_role,
	FALSE, data_set);
	if ((primary_instance == NULL) && (constraint->score >= INFINITY)) {
	pe_rsc_trace(dependent, "%s can't be promoted %s",
	dependent->id, constraint->id);
	dependent->priority = -INFINITY;

	} else if (primary_instance != NULL) {
	- int new_priority = pe__add_scores(dependent->priority,
	- constraint->score);
	+ int new_priority = pcmk__add_scores(dependent->priority,
	+ constraint->score);

	pe_rsc_debug(dependent, "Applying %s to %s",
	constraint->id, dependent->id);
	pe_rsc_debug(dependent, "\t%s: %d->%d",
	dependent->id, dependent->priority, new_priority);
	dependent->priority = new_priority;
	}
	}

	return;
	}
	diff --git a/lib/pengine/common.c b/lib/pengine/common.c
	index bc341eb060..455b985e51 100644
	--- a/lib/pengine/common.c
	+++ b/lib/pengine/common.c
	@@ -1,634 +1,564 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>

	#include <glib.h>

	#include <crm/pengine/internal.h>

	gboolean was_processing_error = FALSE;
	gboolean was_processing_warning = FALSE;

	static bool
	check_health(const char *value)
	{
	return pcmk__strcase_any_of(value, "none", "custom", "only-green", "progressive",
	"migrate-on-red", NULL);
	}

	static bool
	check_placement_strategy(const char *value)
	{
	return pcmk__strcase_any_of(value, "default", "utilization", "minimal",
	"balanced", NULL);
	}

	static pcmk__cluster_option_t pe_opts[] = {
	/* name, old name, type, allowed values,
	* default value, validator,
	* short description,
	* long description
	*/
	{
	"no-quorum-policy", NULL, "select", "stop, freeze, ignore, demote, suicide",
	"stop", pcmk__valid_quorum,
	"What to do when the cluster does not have quorum",
	NULL
	},
	{
	"symmetric-cluster", NULL, "boolean", NULL,
	"true", pcmk__valid_boolean,
	"Whether resources can run on any node by default",
	NULL
	},
	{
	"maintenance-mode", NULL, "boolean", NULL,
	"false", pcmk__valid_boolean,
	"Whether the cluster should refrain from monitoring, starting, "
	"and stopping resources",
	NULL
	},
	{
	"start-failure-is-fatal", NULL, "boolean", NULL,
	"true", pcmk__valid_boolean,
	"Whether a start failure should prevent a resource from being "
	"recovered on the same node",
	"When true, the cluster will immediately ban a resource from a node "
	"if it fails to start there. When false, the cluster will instead "
	"check the resource's fail count against its migration-threshold."
	},
	{
	"enable-startup-probes", NULL, "boolean", NULL,
	"true", pcmk__valid_boolean,
	"Whether the cluster should check for active resources during start-up",
	NULL
	},
	{
	XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL,
	"false", pcmk__valid_boolean,
	"Whether to lock resources to a cleanly shut down node",
	"When true, resources active on a node when it is cleanly shut down "
	"are kept \"locked\" to that node (not allowed to run elsewhere) "
	"until they start again on that node after it rejoins (or for at "
	"most shutdown-lock-limit, if set). Stonith resources and "
	"Pacemaker Remote connections are never locked. Clone and bundle "
	"instances and the promoted role of promotable clones are currently"
	" never locked, though support could be added in a future release."
	},
	{
	XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	"Do not lock resources to a cleanly shut down node longer than this",
	"If shutdown-lock is true and this is set to a nonzero time duration, "
	"shutdown locks will expire after this much time has passed since "
	"the shutdown was initiated, even if the node has not rejoined."
	},

	// Fencing-related options
	{
	"stonith-enabled", NULL, "boolean", NULL,
	"true", pcmk__valid_boolean,
	"* Advanced Use Only * "
	"Whether nodes may be fenced as part of recovery",
	"If false, unresponsive nodes are immediately assumed to be harmless, "
	"and resources that were active on them may be recovered "
	"elsewhere. This can result in a \"split-brain\" situation, "
	"potentially leading to data loss and/or service unavailability."
	},
	{
	"stonith-action", NULL, "select", "reboot, off, poweroff",
	"reboot", pcmk__is_fencing_action,
	"Action to send to fence device when a node needs to be fenced "
	"(\"poweroff\" is a deprecated alias for \"off\")",
	NULL
	},
	{
	"stonith-timeout", NULL, "time", NULL,
	"60s", pcmk__valid_interval_spec,
	"* Advanced Use Only * Unused by Pacemaker",
	"This value is not used by Pacemaker, but is kept for backward "
	"compatibility, and certain legacy fence agents might use it."
	},
	{
	XML_ATTR_HAVE_WATCHDOG, NULL, "boolean", NULL,
	"false", pcmk__valid_boolean,
	N_("Whether watchdog integration is enabled"),
	"This is set automatically by the cluster according to whether SBD "
	"is detected to be in use. User-configured values are ignored. "
	"The value `true` is meaningful if diskless SBD is used and "
	"`stonith-watchdog-timeout` is nonzero. In that case, if fencing "
	"is required, watchdog-based self-fencing will be performed via "
	"SBD without requiring a fencing resource explicitly configured."
	},
	{
	"concurrent-fencing", NULL, "boolean", NULL,
	PCMK__CONCURRENT_FENCING_DEFAULT, pcmk__valid_boolean,
	"Allow performing fencing operations in parallel",
	NULL
	},
	{
	"startup-fencing", NULL, "boolean", NULL,
	"true", pcmk__valid_boolean,
	"* Advanced Use Only * Whether to fence unseen nodes at start-up",
	"Setting this to false may lead to a \"split-brain\" situation,"
	"potentially leading to data loss and/or service unavailability."
	},
	{
	XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	"Apply fencing delay targeting the lost nodes with the highest total resource priority",
	"Apply specified delay for the fencings that are targeting the lost "
	"nodes with the highest total resource priority in case we don't "
	"have the majority of the nodes in our cluster partition, so that "
	"the more significant nodes potentially win any fencing match, "
	"which is especially meaningful under split-brain of 2-node "
	"cluster. A promoted resource instance takes the base priority + 1 "
	"on calculation if the base priority is not 0. Any static/random "
	"delays that are introduced by `pcmk_delay_base/max` configured "
	"for the corresponding fencing resources will be added to this "
	"delay. This delay should be significantly greater than, safely "
	"twice, the maximum `pcmk_delay_base/max`. By default, priority "
	"fencing delay is disabled."
	},

	{
	"cluster-delay", NULL, "time", NULL,
	"60s", pcmk__valid_interval_spec,
	"Maximum time for node-to-node communication",
	"The node elected Designated Controller (DC) will consider an action "
	"failed if it does not get a response from the node executing the "
	"action within this time (after considering the action's own "
	"timeout). The \"correct\" value will depend on the speed and "
	"load of your network and cluster nodes."
	},
	{
	"batch-limit", NULL, "integer", NULL,
	"0", pcmk__valid_number,
	"Maximum number of jobs that the cluster may execute in parallel "
	"across all nodes",
	"The \"correct\" value will depend on the speed and load of your "
	"network and cluster nodes. If set to 0, the cluster will "
	"impose a dynamically calculated limit when any node has a "
	"high load."
	},
	{
	"migration-limit", NULL, "integer", NULL,
	"-1", pcmk__valid_number,
	"The number of live migration actions that the cluster is allowed "
	"to execute in parallel on a node (-1 means no limit)"
	},

	/* Orphans and stopping */
	{
	"stop-all-resources", NULL, "boolean", NULL,
	"false", pcmk__valid_boolean,
	"Whether the cluster should stop all active resources",
	NULL
	},
	{
	"stop-orphan-resources", NULL, "boolean", NULL,
	"true", pcmk__valid_boolean,
	"Whether to stop resources that were removed from the configuration",
	NULL
	},
	{
	"stop-orphan-actions", NULL, "boolean", NULL,
	"true", pcmk__valid_boolean,
	"Whether to cancel recurring actions removed from the configuration",
	NULL
	},
	{
	"remove-after-stop", NULL, "boolean", NULL,
	"false", pcmk__valid_boolean,
	"* Deprecated * Whether to remove stopped resources from "
	"the executor",
	"Values other than default are poorly tested and potentially dangerous."
	" This option will be removed in a future release."
	},

	/* Storing inputs */
	{
	"pe-error-series-max", NULL, "integer", NULL,
	"-1", pcmk__valid_number,
	"The number of scheduler inputs resulting in errors to save",
	"Zero to disable, -1 to store unlimited."
	},
	{
	"pe-warn-series-max", NULL, "integer", NULL,
	"5000", pcmk__valid_number,
	"The number of scheduler inputs resulting in warnings to save",
	"Zero to disable, -1 to store unlimited."
	},
	{
	"pe-input-series-max", NULL, "integer", NULL,
	"4000", pcmk__valid_number,
	"The number of scheduler inputs without errors or warnings to save",
	"Zero to disable, -1 to store unlimited."
	},

	/* Node health */
	{
	"node-health-strategy", NULL, "select",
	"none, migrate-on-red, only-green, progressive, custom",
	"none", check_health,
	"How cluster should react to node health attributes",
	"Requires external entities to create node attributes (named with "
	"the prefix \"#health\") with values \"red\", \"yellow\" or "
	"\"green\"."
	},
	{
	"node-health-base", NULL, "integer", NULL,
	"0", pcmk__valid_number,
	"Base health score assigned to a node",
	"Only used when node-health-strategy is set to progressive."
	},
	{
	"node-health-green", NULL, "integer", NULL,
	"0", pcmk__valid_number,
	"The score to use for a node health attribute whose value is \"green\"",
	"Only used when node-health-strategy is set to custom or progressive."
	},
	{
	"node-health-yellow", NULL, "integer", NULL,
	"0", pcmk__valid_number,
	"The score to use for a node health attribute whose value is \"yellow\"",
	"Only used when node-health-strategy is set to custom or progressive."
	},
	{
	"node-health-red", NULL, "integer", NULL,
	"-INFINITY", pcmk__valid_number,
	"The score to use for a node health attribute whose value is \"red\"",
	"Only used when node-health-strategy is set to custom or progressive."
	},

	/Placement Strategy/
	{
	"placement-strategy", NULL, "select",
	"default, utilization, minimal, balanced",
	"default", check_placement_strategy,
	"How the cluster should allocate resources to nodes",
	NULL
	},
	};

	void
	pe_metadata(pcmk__output_t *out)
	{
	char *s = pcmk__format_option_metadata("pacemaker-schedulerd",
	"Pacemaker scheduler options",
	"Cluster options used by Pacemaker's scheduler",
	pe_opts, PCMK__NELEM(pe_opts));
	out->output_xml(out, "metadata", s);
	free(s);
	}

	void
	verify_pe_options(GHashTable * options)
	{
	pcmk__validate_cluster_options(options, pe_opts, PCMK__NELEM(pe_opts));
	}

	const char *
	pe_pref(GHashTable * options, const char *name)
	{
	return pcmk__cluster_option(options, pe_opts, PCMK__NELEM(pe_opts), name);
	}

	const char *
	fail2text(enum action_fail_response fail)
	{
	const char *result = "<unknown>";

	switch (fail) {
	case action_fail_ignore:
	result = "ignore";
	break;
	case action_fail_demote:
	result = "demote";
	break;
	case action_fail_block:
	result = "block";
	break;
	case action_fail_recover:
	result = "recover";
	break;
	case action_fail_migrate:
	result = "migrate";
	break;
	case action_fail_stop:
	result = "stop";
	break;
	case action_fail_fence:
	result = "fence";
	break;
	case action_fail_standby:
	result = "standby";
	break;
	case action_fail_restart_container:
	result = "restart-container";
	break;
	case action_fail_reset_remote:
	result = "reset-remote";
	break;
	}
	return result;
	}

	enum action_tasks
	text2task(const char *task)
	{
	if (pcmk__str_eq(task, CRMD_ACTION_STOP, pcmk__str_casei)) {
	return stop_rsc;
	} else if (pcmk__str_eq(task, CRMD_ACTION_STOPPED, pcmk__str_casei)) {
	return stopped_rsc;
	} else if (pcmk__str_eq(task, CRMD_ACTION_START, pcmk__str_casei)) {
	return start_rsc;
	} else if (pcmk__str_eq(task, CRMD_ACTION_STARTED, pcmk__str_casei)) {
	return started_rsc;
	} else if (pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	return shutdown_crm;
	} else if (pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)) {
	return stonith_node;
	} else if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
	return monitor_rsc;
	} else if (pcmk__str_eq(task, CRMD_ACTION_NOTIFY, pcmk__str_casei)) {
	return action_notify;
	} else if (pcmk__str_eq(task, CRMD_ACTION_NOTIFIED, pcmk__str_casei)) {
	return action_notified;
	} else if (pcmk__str_eq(task, CRMD_ACTION_PROMOTE, pcmk__str_casei)) {
	return action_promote;
	} else if (pcmk__str_eq(task, CRMD_ACTION_DEMOTE, pcmk__str_casei)) {
	return action_demote;
	} else if (pcmk__str_eq(task, CRMD_ACTION_PROMOTED, pcmk__str_casei)) {
	return action_promoted;
	} else if (pcmk__str_eq(task, CRMD_ACTION_DEMOTED, pcmk__str_casei)) {
	return action_demoted;
	}
	#if SUPPORT_TRACING
	if (pcmk__str_eq(task, CRMD_ACTION_CANCEL, pcmk__str_casei)) {
	return no_action;
	} else if (pcmk__str_eq(task, CRMD_ACTION_DELETE, pcmk__str_casei)) {
	return no_action;
	} else if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
	return no_action;
	} else if (pcmk__str_eq(task, CRM_OP_LRM_REFRESH, pcmk__str_casei)) {
	return no_action;
	} else if (pcmk__str_eq(task, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
	return no_action;
	} else if (pcmk__str_eq(task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
	return no_action;
	}
	crm_trace("Unsupported action: %s", task);
	#endif

	return no_action;
	}

	const char *
	task2text(enum action_tasks task)
	{
	const char *result = "<unknown>";

	switch (task) {
	case no_action:
	result = "no_action";
	break;
	case stop_rsc:
	result = CRMD_ACTION_STOP;
	break;
	case stopped_rsc:
	result = CRMD_ACTION_STOPPED;
	break;
	case start_rsc:
	result = CRMD_ACTION_START;
	break;
	case started_rsc:
	result = CRMD_ACTION_STARTED;
	break;
	case shutdown_crm:
	result = CRM_OP_SHUTDOWN;
	break;
	case stonith_node:
	result = CRM_OP_FENCE;
	break;
	case monitor_rsc:
	result = CRMD_ACTION_STATUS;
	break;
	case action_notify:
	result = CRMD_ACTION_NOTIFY;
	break;
	case action_notified:
	result = CRMD_ACTION_NOTIFIED;
	break;
	case action_promote:
	result = CRMD_ACTION_PROMOTE;
	break;
	case action_promoted:
	result = CRMD_ACTION_PROMOTED;
	break;
	case action_demote:
	result = CRMD_ACTION_DEMOTE;
	break;
	case action_demoted:
	result = CRMD_ACTION_DEMOTED;
	break;
	}

	return result;
	}

	const char *
	role2text(enum rsc_role_e role)
	{
	switch (role) {
	case RSC_ROLE_UNKNOWN:
	return RSC_ROLE_UNKNOWN_S;
	case RSC_ROLE_STOPPED:
	return RSC_ROLE_STOPPED_S;
	case RSC_ROLE_STARTED:
	return RSC_ROLE_STARTED_S;
	case RSC_ROLE_UNPROMOTED:
	#ifdef PCMK__COMPAT_2_0
	return RSC_ROLE_UNPROMOTED_LEGACY_S;
	#else
	return RSC_ROLE_UNPROMOTED_S;
	#endif
	case RSC_ROLE_PROMOTED:
	#ifdef PCMK__COMPAT_2_0
	return RSC_ROLE_PROMOTED_LEGACY_S;
	#else
	return RSC_ROLE_PROMOTED_S;
	#endif
	}
	CRM_CHECK(role >= RSC_ROLE_UNKNOWN, return RSC_ROLE_UNKNOWN_S);
	CRM_CHECK(role < RSC_ROLE_MAX, return RSC_ROLE_UNKNOWN_S);
	// coverity[dead_error_line]
	return RSC_ROLE_UNKNOWN_S;
	}

	enum rsc_role_e
	text2role(const char *role)
	{
	CRM_ASSERT(role != NULL);
	if (pcmk__str_eq(role, RSC_ROLE_STOPPED_S, pcmk__str_casei)) {
	return RSC_ROLE_STOPPED;
	} else if (pcmk__str_eq(role, RSC_ROLE_STARTED_S, pcmk__str_casei)) {
	return RSC_ROLE_STARTED;
	} else if (pcmk__strcase_any_of(role, RSC_ROLE_UNPROMOTED_S,
	RSC_ROLE_UNPROMOTED_LEGACY_S, NULL)) {
	return RSC_ROLE_UNPROMOTED;
	} else if (pcmk__strcase_any_of(role, RSC_ROLE_PROMOTED_S,
	RSC_ROLE_PROMOTED_LEGACY_S, NULL)) {
	return RSC_ROLE_PROMOTED;
	} else if (pcmk__str_eq(role, RSC_ROLE_UNKNOWN_S, pcmk__str_casei)) {
	return RSC_ROLE_UNKNOWN;
	}
	crm_err("Unknown role: %s", role);
	return RSC_ROLE_UNKNOWN;
	}

	-/*!
	- * \internal
	- * \brief Add two scores (bounding to +/- INFINITY)
	- *
	- * \param[in] score1 First score to add
	- * \param[in] score2 Second score to add
	- */
	-int
	-pe__add_scores(int score1, int score2)
	-{
	- int result = score1 + score2;
	-
	- // First handle the cases where one or both is infinite
	-
	- if (score1 <= -CRM_SCORE_INFINITY) {
	-
	- if (score2 <= -CRM_SCORE_INFINITY) {
	- crm_trace("-INFINITY + -INFINITY = -INFINITY");
	- } else if (score2 >= CRM_SCORE_INFINITY) {
	- crm_trace("-INFINITY + +INFINITY = -INFINITY");
	- } else {
	- crm_trace("-INFINITY + %d = -INFINITY", score2);
	- }
	-
	- return -CRM_SCORE_INFINITY;
	-
	- } else if (score2 <= -CRM_SCORE_INFINITY) {
	-
	- if (score1 >= CRM_SCORE_INFINITY) {
	- crm_trace("+INFINITY + -INFINITY = -INFINITY");
	- } else {
	- crm_trace("%d + -INFINITY = -INFINITY", score1);
	- }
	-
	- return -CRM_SCORE_INFINITY;
	-
	- } else if (score1 >= CRM_SCORE_INFINITY) {
	-
	- if (score2 >= CRM_SCORE_INFINITY) {
	- crm_trace("+INFINITY + +INFINITY = +INFINITY");
	- } else {
	- crm_trace("+INFINITY + %d = +INFINITY", score2);
	- }
	-
	- return CRM_SCORE_INFINITY;
	-
	- } else if (score2 >= CRM_SCORE_INFINITY) {
	- crm_trace("%d + +INFINITY = +INFINITY", score1);
	- return CRM_SCORE_INFINITY;
	- }
	-
	- /* As long as CRM_SCORE_INFINITY is less than half of the maximum integer,
	- * we can ignore the possibility of integer overflow
	- */
	-
	- // Bound result to infinity
	-
	- if (result >= CRM_SCORE_INFINITY) {
	- crm_trace("%d + %d = +INFINITY", score1, score2);
	- return CRM_SCORE_INFINITY;
	-
	- } else if (result <= -CRM_SCORE_INFINITY) {
	- crm_trace("%d + %d = -INFINITY", score1, score2);
	- return -CRM_SCORE_INFINITY;
	- }
	-
	- crm_trace("%d + %d = %d", score1, score2, result);
	- return result;
	-}
	-
	void
	add_hash_param(GHashTable * hash, const char name, const char value)
	{
	CRM_CHECK(hash != NULL, return);

	crm_trace("adding: name=%s value=%s", crm_str(name), crm_str(value));
	if (name == NULL \|\| value == NULL) {
	return;

	} else if (pcmk__str_eq(value, "#default", pcmk__str_casei)) {
	return;

	} else if (g_hash_table_lookup(hash, name) == NULL) {
	g_hash_table_insert(hash, strdup(name), strdup(value));
	}
	}

	const char *
	pe_node_attribute_calculated(const pe_node_t node, const char name,
	const pe_resource_t *rsc)
	{
	const char *source;

	if(node == NULL) {
	return NULL;

	} else if(rsc == NULL) {
	return g_hash_table_lookup(node->details->attrs, name);
	}

	source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET);
	if(source == NULL \|\| !pcmk__str_eq("host", source, pcmk__str_casei)) {
	return g_hash_table_lookup(node->details->attrs, name);
	}

	/* Use attributes set for the containers location
	* instead of for the container itself
	*
	* Useful when the container is using the host's local
	* storage
	*/

	CRM_ASSERT(node->details->remote_rsc);
	CRM_ASSERT(node->details->remote_rsc->container);

	if(node->details->remote_rsc->container->running_on) {
	pe_node_t *host = node->details->remote_rsc->container->running_on->data;
	pe_rsc_trace(rsc, "%s: Looking for %s on the container host %s", rsc->id, name, host->details->uname);
	return g_hash_table_lookup(host->details->attrs, name);
	}

	pe_rsc_trace(rsc, "%s: Not looking for %s on the container host: %s is inactive",
	rsc->id, name, node->details->remote_rsc->container->id);
	return NULL;
	}

	const char *
	pe_node_attribute_raw(pe_node_t node, const char name)
	{
	if(node == NULL) {
	return NULL;
	}
	return g_hash_table_lookup(node->details->attrs, name);
	}
	diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
	index 5b4bf40316..e82af2aae8 100644
	--- a/lib/pengine/complex.c
	+++ b/lib/pengine/complex.c
	@@ -1,1124 +1,1124 @@
	/*
	- * Copyright 2004-2021 the Pacemaker project contributors
	+ * Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>

	void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);

	resource_object_functions_t resource_class_functions[] = {
	{
	native_unpack,
	native_find_rsc,
	native_parameter,
	native_print,
	native_active,
	native_resource_state,
	native_location,
	native_free,
	pe__count_common,
	pe__native_is_filtered,
	},
	{
	group_unpack,
	native_find_rsc,
	native_parameter,
	group_print,
	group_active,
	group_resource_state,
	native_location,
	group_free,
	pe__count_common,
	pe__group_is_filtered,
	},
	{
	clone_unpack,
	native_find_rsc,
	native_parameter,
	clone_print,
	clone_active,
	clone_resource_state,
	native_location,
	clone_free,
	pe__count_common,
	pe__clone_is_filtered,
	},
	{
	pe__unpack_bundle,
	native_find_rsc,
	native_parameter,
	pe__print_bundle,
	pe__bundle_active,
	pe__bundle_resource_state,
	native_location,
	pe__free_bundle,
	pe__count_bundle,
	pe__bundle_is_filtered,
	}
	};

	static enum pe_obj_types
	get_resource_type(const char *name)
	{
	if (pcmk__str_eq(name, XML_CIB_TAG_RESOURCE, pcmk__str_casei)) {
	return pe_native;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_GROUP, pcmk__str_casei)) {
	return pe_group;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_INCARNATION, pcmk__str_casei)) {
	return pe_clone;

	} else if (pcmk__str_eq(name, PCMK_XE_PROMOTABLE_LEGACY, pcmk__str_casei)) {
	// @COMPAT deprecated since 2.0.0
	return pe_clone;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_CONTAINER, pcmk__str_casei)) {
	return pe_container;
	}

	return pe_unknown;
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	add_hash_param(user_data, key, value);
	}

	static void
	expand_parents_fixed_nvpairs(pe_resource_t * rsc, pe_rule_eval_data_t * rule_data, GHashTable * meta_hash, pe_working_set_t * data_set)
	{
	GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
	pe_resource_t *p = rsc->parent;

	if (p == NULL) {
	return ;
	}

	/* Search all parent resources, get the fixed value of "meta_attributes" set only in the original xml, and stack it in the hash table. */
	/* The fixed value of the lower parent resource takes precedence and is not overwritten. */
	while(p != NULL) {
	/* A hash table for comparison is generated, including the id-ref. */
	pe__unpack_dataset_nvpairs(p->xml, XML_TAG_META_SETS,
	rule_data, parent_orig_meta, NULL, FALSE, data_set);
	p = p->parent;
	}

	/* If there is a fixed value of "meta_attributes" of the parent resource, it will be processed. */
	if (parent_orig_meta != NULL) {
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, parent_orig_meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	/* Parameters set in the original xml of the parent resource will also try to overwrite the child resource. */
	/* Attributes that already exist in the child lease are not updated. */
	dup_attr(key, value, meta_hash);
	}
	}

	if (parent_orig_meta != NULL) {
	g_hash_table_destroy(parent_orig_meta);
	}

	return ;

	}
	void
	get_meta_attributes(GHashTable * meta_hash, pe_resource_t * rsc,
	pe_node_t * node, pe_working_set_t * data_set)
	{
	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE)
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->xml); a != NULL; a = a->next) {
	const char prop_name = (const char ) a->name;
	const char *prop_value = crm_element_value(rsc->xml, prop_name);

	add_hash_param(meta_hash, prop_name, prop_value);
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, &rule_data,
	meta_hash, NULL, FALSE, data_set);

	/* Set the "meta_attributes" explicitly set in the parent resource to the hash table of the child resource. */
	/* If it is already explicitly set as a child, it will not be overwritten. */
	if (rsc->parent != NULL) {
	expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, data_set);
	}

	/* check the defaults */
	pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_META_SETS,
	&rule_data, meta_hash, NULL, FALSE, data_set);

	/* If there is "meta_attributes" that the parent resource has not explicitly set, set a value that is not set from rsc_default either. */
	/* The values already set up to this point will not be overwritten. */
	if (rsc->parent) {
	g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash);
	}
	}

	void
	get_rsc_attributes(GHashTable * meta_hash, pe_resource_t * rsc,
	pe_node_t * node, pe_working_set_t * data_set)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, &rule_data,
	meta_hash, NULL, FALSE, data_set);

	/* set anything else based on the parent */
	if (rsc->parent != NULL) {
	get_rsc_attributes(meta_hash, rsc->parent, node, data_set);

	} else {
	/* and finally check the defaults */
	pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_ATTR_SETS,
	&rule_data, meta_hash, NULL, FALSE, data_set);
	}
	}

	#if ENABLE_VERSIONED_ATTRS
	void
	pe_get_versioned_attributes(xmlNode * meta_hash, pe_resource_t * rsc,
	pe_node_t * node, pe_working_set_t * data_set)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = (node == NULL)? NULL : node->details->attrs,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe_eval_versioned_attributes(data_set->input, rsc->xml, XML_TAG_ATTR_SETS,
	&rule_data, meta_hash, NULL);

	/* set anything else based on the parent */
	if (rsc->parent != NULL) {
	pe_get_versioned_attributes(meta_hash, rsc->parent, node, data_set);

	} else {
	/* and finally check the defaults */
	pe_eval_versioned_attributes(data_set->input, data_set->rsc_defaults,
	XML_TAG_ATTR_SETS, &rule_data, meta_hash,
	NULL);
	}
	}
	#endif

	static char *
	template_op_key(xmlNode * op)
	{
	const char *name = crm_element_value(op, "name");
	const char *role = crm_element_value(op, "role");
	char *key = NULL;

	if ((role == NULL)
	\|\| pcmk__strcase_any_of(role, RSC_ROLE_STARTED_S, RSC_ROLE_UNPROMOTED_S,
	RSC_ROLE_UNPROMOTED_LEGACY_S, NULL)) {
	role = RSC_ROLE_UNKNOWN_S;
	}

	key = crm_strdup_printf("%s-%s", name, role);
	return key;
	}

	static gboolean
	unpack_template(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
	{
	xmlNode *cib_resources = NULL;
	xmlNode *template = NULL;
	xmlNode *new_xml = NULL;
	xmlNode *child_xml = NULL;
	xmlNode *rsc_ops = NULL;
	xmlNode *template_ops = NULL;
	const char *template_ref = NULL;
	const char *clone = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pe_err("No resource object for template unpacking");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pe_err("'%s' object must have a id", crm_element_name(xml_obj));
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pe_err("The resource object '%s' should not reference itself", id);
	return FALSE;
	}

	cib_resources = get_xpath_object("//"XML_CIB_TAG_RESOURCES, data_set->input, LOG_TRACE);
	if (cib_resources == NULL) {
	pe_err("No resources configured");
	return FALSE;
	}

	template = pcmk__xe_match(cib_resources, XML_CIB_TAG_RSC_TEMPLATE,
	XML_ATTR_ID, template_ref);
	if (template == NULL) {
	pe_err("No template named '%s'", template_ref);
	return FALSE;
	}

	new_xml = copy_xml(template);
	xmlNodeSetName(new_xml, xml_obj->name);
	crm_xml_replace(new_xml, XML_ATTR_ID, id);

	clone = crm_element_value(xml_obj, XML_RSC_ATTR_INCARNATION);
	if(clone) {
	crm_xml_add(new_xml, XML_RSC_ATTR_INCARNATION, clone);
	}

	template_ops = find_xml_node(new_xml, "operations", FALSE);

	for (child_xml = pcmk__xe_first_child(xml_obj); child_xml != NULL;
	child_xml = pcmk__xe_next(child_xml)) {
	xmlNode *new_child = NULL;

	new_child = add_node_copy(new_xml, child_xml);

	if (pcmk__str_eq((const char *)new_child->name, "operations", pcmk__str_none)) {
	rsc_ops = new_child;
	}
	}

	if (template_ops && rsc_ops) {
	xmlNode *op = NULL;
	GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);

	for (op = pcmk__xe_first_child(rsc_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	g_hash_table_insert(rsc_ops_hash, key, op);
	}

	for (op = pcmk__xe_first_child(template_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
	add_node_copy(rsc_ops, op);
	}

	free(key);
	}

	if (rsc_ops_hash) {
	g_hash_table_destroy(rsc_ops_hash);
	}

	free_xml(template_ops);
	}

	/free_xml(expanded_xml); */
	*expanded_xml = new_xml;

	/* Disable multi-level templates for now */
	/*if(unpack_template(new_xml, expanded_xml, data_set) == FALSE) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;

	return FALSE;
	} */

	return TRUE;
	}

	static gboolean
	add_template_rsc(xmlNode * xml_obj, pe_working_set_t * data_set)
	{
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pe_err("No resource object for processing resource list of template");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pe_err("'%s' object must have a id", crm_element_name(xml_obj));
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pe_err("The resource object '%s' should not reference itself", id);
	return FALSE;
	}

	if (add_tag_ref(data_set->template_rsc_sets, template_ref, id) == FALSE) {
	return FALSE;
	}

	return TRUE;
	}

	static bool
	detect_promotable(pe_resource_t *rsc)
	{
	const char *promotable = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_PROMOTABLE);

	if (crm_is_true(promotable)) {
	return TRUE;
	}

	// @COMPAT deprecated since 2.0.0
	if (pcmk__str_eq(crm_element_name(rsc->xml), PCMK_XE_PROMOTABLE_LEGACY,
	pcmk__str_casei)) {
	/* @TODO in some future version, pe_warn_once() here,
	* then drop support in even later version
	*/
	g_hash_table_insert(rsc->meta, strdup(XML_RSC_ATTR_PROMOTABLE),
	strdup(XML_BOOLEAN_TRUE));
	return TRUE;
	}
	return FALSE;
	}

	static void
	free_params_table(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \brief Get a table of resource parameters
	*
	* \param[in] rsc Resource to query
	* \param[in] node Node for evaluating rules (NULL for defaults)
	* \param[in] data_set Cluster working set
	*
	* \return Hash table containing resource parameter names and values
	* (or NULL if \p rsc or \p data_set is NULL)
	* \note The returned table will be destroyed when the resource is freed, so
	* callers should not destroy it.
	*/
	GHashTable *
	pe_rsc_params(pe_resource_t rsc, pe_node_t node, pe_working_set_t *data_set)
	{
	GHashTable *params_on_node = NULL;

	/* A NULL node is used to request the resource's default parameters
	* (not evaluated for node), but we always want something non-NULL
	* as a hash table key.
	*/
	const char *node_name = "";

	// Sanity check
	if ((rsc == NULL) \|\| (data_set == NULL)) {
	return NULL;
	}
	if ((node != NULL) && (node->details->uname != NULL)) {
	node_name = node->details->uname;
	}

	// Find the parameter table for given node
	if (rsc->parameter_cache == NULL) {
	rsc->parameter_cache = pcmk__strikey_table(free, free_params_table);
	} else {
	params_on_node = g_hash_table_lookup(rsc->parameter_cache, node_name);
	}

	// If none exists yet, create one with parameters evaluated for node
	if (params_on_node == NULL) {
	params_on_node = pcmk__strkey_table(free, free);
	get_rsc_attributes(params_on_node, rsc, node, data_set);
	g_hash_table_insert(rsc->parameter_cache, strdup(node_name),
	params_on_node);
	}
	return params_on_node;
	}

	gboolean
	common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc,
	pe_resource_t * parent, pe_working_set_t * data_set)
	{
	bool isdefault = FALSE;
	xmlNode *expanded_xml = NULL;
	xmlNode *ops = NULL;
	const char *value = NULL;
	const char rclass = NULL; / Look for this after any templates have been expanded */
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	bool guest_node = FALSE;
	bool remote_node = FALSE;
	bool has_versioned_params = FALSE;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	crm_log_xml_trace(xml_obj, "Processing resource input...");

	if (id == NULL) {
	pe_err("Must specify id tag in <resource>");
	return FALSE;

	} else if (rsc == NULL) {
	pe_err("Nowhere to unpack resource into");
	return FALSE;

	}

	if (unpack_template(xml_obj, &expanded_xml, data_set) == FALSE) {
	return FALSE;
	}

	*rsc = calloc(1, sizeof(pe_resource_t));
	(*rsc)->cluster = data_set;

	if (expanded_xml) {
	crm_log_xml_trace(expanded_xml, "Expanded resource...");
	(*rsc)->xml = expanded_xml;
	(*rsc)->orig_xml = xml_obj;

	} else {
	(*rsc)->xml = xml_obj;
	(*rsc)->orig_xml = NULL;
	}

	/* Do not use xml_obj from here on, use (rsc)->xml in case templates are involved /
	rclass = crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS);
	(*rsc)->parent = parent;

	ops = find_xml_node((*rsc)->xml, "operations", FALSE);
	(*rsc)->ops_xml = expand_idref(ops, data_set->input);

	(rsc)->variant = get_resource_type(crm_element_name((rsc)->xml));
	if ((*rsc)->variant == pe_unknown) {
	pe_err("Unknown resource type: %s", crm_element_name((*rsc)->xml));
	free(*rsc);
	return FALSE;
	}

	#if ENABLE_VERSIONED_ATTRS
	(*rsc)->versioned_parameters = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);
	#endif

	(*rsc)->meta = pcmk__strkey_table(free, free);
	(*rsc)->allowed_nodes = pcmk__strkey_table(NULL, free);
	(*rsc)->known_on = pcmk__strkey_table(NULL, free);

	value = crm_element_value((*rsc)->xml, XML_RSC_ATTR_INCARNATION);
	if (value) {
	(*rsc)->id = crm_strdup_printf("%s:%s", id, value);
	add_hash_param((*rsc)->meta, XML_RSC_ATTR_INCARNATION, value);

	} else {
	(*rsc)->id = strdup(id);
	}

	(rsc)->fns = &resource_class_functions[(rsc)->variant];
	pe_rsc_trace((*rsc), "Unpacking resource...");

	get_meta_attributes((rsc)->meta, rsc, NULL, data_set);
	(rsc)->parameters = pe_rsc_params(rsc, NULL, data_set); // \deprecated
	#if ENABLE_VERSIONED_ATTRS
	pe_get_versioned_attributes((rsc)->versioned_parameters, rsc, NULL, data_set);
	#endif

	(*rsc)->flags = 0;
	pe__set_resource_flags(*rsc, pe_rsc_runnable\|pe_rsc_provisional);

	if (!pcmk_is_set(data_set->flags, pe_flag_maintenance_mode)) {
	pe__set_resource_flags(*rsc, pe_rsc_managed);
	}

	(*rsc)->rsc_cons = NULL;
	(*rsc)->rsc_tickets = NULL;
	(*rsc)->actions = NULL;
	(*rsc)->role = RSC_ROLE_STOPPED;
	(*rsc)->next_role = RSC_ROLE_UNKNOWN;

	(*rsc)->recovery_type = recovery_stop_start;
	(*rsc)->stickiness = 0;
	(*rsc)->migration_threshold = INFINITY;
	(*rsc)->failure_timeout = 0;

	value = g_hash_table_lookup((*rsc)->meta, XML_CIB_ATTR_PRIORITY);
	(*rsc)->priority = char2score(value);

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CRITICAL);
	if ((value == NULL) \|\| crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_critical);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_NOTIFY);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_notify);
	}

	if (xml_contains_remote_node((*rsc)->xml)) {
	(*rsc)->is_remote_node = TRUE;
	if (g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CONTAINER)) {
	guest_node = TRUE;
	} else {
	remote_node = TRUE;
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_OP_ATTR_ALLOW_MIGRATE);
	#if ENABLE_VERSIONED_ATTRS
	has_versioned_params = xml_has_children((*rsc)->versioned_parameters);
	#endif
	if (crm_is_true(value) && has_versioned_params) {
	pe_rsc_trace((*rsc), "Migration is disabled for resources with versioned parameters");
	} else if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_allow_migrate);
	} else if ((value == NULL) && remote_node && !has_versioned_params) {
	/* By default, we want remote nodes to be able
	* to float around the cluster without having to stop all the
	* resources within the remote-node before moving. Allowing
	* migration support enables this feature. If this ever causes
	* problems, migration support can be explicitly turned off with
	* allow-migrate=false.
	* We don't support migration for versioned resources, though. */
	pe__set_resource_flags(*rsc, pe_rsc_allow_migrate);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MANAGED);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_managed);
	} else {
	pe__clear_resource_flags(*rsc, pe_rsc_managed);
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MAINTENANCE);
	if (crm_is_true(value)) {
	pe__clear_resource_flags(*rsc, pe_rsc_managed);
	pe__set_resource_flags(*rsc, pe_rsc_maintenance);
	}
	if (pcmk_is_set(data_set->flags, pe_flag_maintenance_mode)) {
	pe__clear_resource_flags(*rsc, pe_rsc_managed);
	pe__set_resource_flags(*rsc, pe_rsc_maintenance);
	}

	if (pe_rsc_is_clone(uber_parent(*rsc))) {
	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_UNIQUE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_unique);
	}
	if (detect_promotable(*rsc)) {
	pe__set_resource_flags(*rsc, pe_rsc_promotable);
	}
	} else {
	pe__set_resource_flags(*rsc, pe_rsc_unique);
	}

	pe_rsc_trace((rsc), "Options for %s", (rsc)->id);

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_RESTART);
	if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	(*rsc)->restart_type = pe_restart_restart;
	pe_rsc_trace((*rsc), "\tDependency restart handling: restart");
	pe_warn_once(pe_wo_restart_type,
	"Support for restart-type is deprecated and will be removed in a future release");

	} else {
	(*rsc)->restart_type = pe_restart_ignore;
	pe_rsc_trace((*rsc), "\tDependency restart handling: ignore");
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MULTIPLE);
	if (pcmk__str_eq(value, "stop_only", pcmk__str_casei)) {
	(*rsc)->recovery_type = recovery_stop_only;
	pe_rsc_trace((*rsc), "\tMultiple running resource recovery: stop only");

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	(*rsc)->recovery_type = recovery_block;
	pe_rsc_trace((*rsc), "\tMultiple running resource recovery: block");

	} else {
	(*rsc)->recovery_type = recovery_stop_start;
	pe_rsc_trace((*rsc), "\tMultiple running resource recovery: stop/start");
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_STICKINESS);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->stickiness = char2score(value);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_FAIL_STICKINESS);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->migration_threshold = char2score(value);
	if ((*rsc)->migration_threshold < 0) {
	/* @TODO We use 1 here to preserve previous behavior, but this
	* should probably use the default (INFINITY) or 0 (to disable)
	* instead.
	*/
	pe_warn_once(pe_wo_neg_threshold,
	XML_RSC_ATTR_FAIL_STICKINESS
	" must be non-negative, using 1 instead");
	(*rsc)->migration_threshold = 1;
	}
	}

	if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pe__set_working_set_flags(data_set, pe_flag_have_stonith_resource);
	pe__set_resource_flags(*rsc, pe_rsc_fence_device);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_REQUIRES);

	handle_requires_pref:
	if (pcmk__str_eq(value, "nothing", pcmk__str_casei)) {

	} else if (pcmk__str_eq(value, "quorum", pcmk__str_casei)) {
	pe__set_resource_flags(*rsc, pe_rsc_needs_quorum);

	} else if (pcmk__str_eq(value, "unfencing", pcmk__str_casei)) {
	if (pcmk_is_set((*rsc)->flags, pe_rsc_fence_device)) {
	pcmk__config_warn("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
	"to 'quorum' because fencing devices cannot "
	"require unfencing", (*rsc)->id);
	value = "quorum";
	isdefault = TRUE;
	goto handle_requires_pref;

	} else if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pcmk__config_warn("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
	"to 'quorum' because fencing is disabled",
	(*rsc)->id);
	value = "quorum";
	isdefault = TRUE;
	goto handle_requires_pref;

	} else {
	pe__set_resource_flags(*rsc, pe_rsc_needs_fencing
	\|pe_rsc_needs_unfencing);
	}

	} else if (pcmk__str_eq(value, "fencing", pcmk__str_casei)) {
	pe__set_resource_flags(*rsc, pe_rsc_needs_fencing);
	if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pcmk__config_warn("%s requires fencing but fencing is disabled",
	(*rsc)->id);
	}

	} else {
	const char *orig_value = value;

	isdefault = TRUE;
	if (pcmk_is_set((*rsc)->flags, pe_rsc_fence_device)) {
	value = "quorum";

	} else if (((*rsc)->variant == pe_native)
	&& pcmk__str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS), PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)
	&& pcmk__str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_PROVIDER), "pacemaker", pcmk__str_casei)
	&& pcmk__str_eq(crm_element_value((*rsc)->xml, XML_ATTR_TYPE), "remote", pcmk__str_casei)
	) {
	value = "quorum";

	} else if (pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)) {
	value = "unfencing";

	} else if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	value = "fencing";

	} else if (data_set->no_quorum_policy == no_quorum_ignore) {
	value = "nothing";

	} else {
	value = "quorum";
	}

	if (orig_value != NULL) {
	pcmk__config_err("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
	"to '%s' because '%s' is not valid",
	(*rsc)->id, value, orig_value);
	}

	goto handle_requires_pref;
	}

	pe_rsc_trace((*rsc), "\tRequired to start: %s%s", value, isdefault?" (default)":"");
	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_FAIL_TIMEOUT);
	if (value != NULL) {
	// Stored as seconds
	(*rsc)->failure_timeout = (int) (crm_parse_interval_spec(value) / 1000);
	}

	if (remote_node) {
	GHashTable params = pe_rsc_params(rsc, NULL, data_set);

	/* Grabbing the value now means that any rules based on node attributes
	* will evaluate to false, so such rules should not be used with
	* reconnect_interval.
	*
	* @TODO Evaluate per node before using
	*/
	value = g_hash_table_lookup(params, XML_REMOTE_ATTR_RECONNECT_INTERVAL);
	if (value) {
	/* reconnect delay works by setting failure_timeout and preventing the
	* connection from starting until the failure is cleared. */
	(*rsc)->remote_reconnect_ms = crm_parse_interval_spec(value);
	/* we want to override any default failure_timeout in use when remote
	* reconnect_interval is in use. */
	(rsc)->failure_timeout = (rsc)->remote_reconnect_ms / 1000;
	}
	}

	get_target_role(rsc, &((rsc)->next_role));
	pe_rsc_trace((*rsc), "\tDesired next state: %s",
	(rsc)->next_role != RSC_ROLE_UNKNOWN ? role2text((rsc)->next_role) : "default");

	if ((rsc)->fns->unpack(rsc, data_set) == FALSE) {
	return FALSE;
	}

	if (pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	// This tag must stay exactly the same because it is tested elsewhere
	resource_location(*rsc, NULL, 0, "symmetric_default", data_set);
	} else if (guest_node) {
	/* remote resources tied to a container resource must always be allowed
	* to opt-in to the cluster. Whether the connection resource is actually
	* allowed to be placed on a node is dependent on the container resource */
	resource_location(*rsc, NULL, 0, "remote_connection_default", data_set);
	}

	pe_rsc_trace((*rsc), "\tAction notification: %s",
	pcmk_is_set((*rsc)->flags, pe_rsc_notify)? "required" : "not required");

	(*rsc)->utilization = pcmk__strkey_table(free, free);

	pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, &rule_data,
	(*rsc)->utilization, NULL, FALSE, data_set);

	/* data_set->resources = g_list_append(data_set->resources, (rsc)); /

	if (expanded_xml) {
	if (add_template_rsc(xml_obj, data_set) == FALSE) {
	return FALSE;
	}
	}
	return TRUE;
	}

	void
	common_update_score(pe_resource_t * rsc, const char *id, int score)
	{
	pe_node_t *node = NULL;

	node = pe_hash_table_lookup(rsc->allowed_nodes, id);
	if (node != NULL) {
	pe_rsc_trace(rsc, "Updating score for %s on %s: %d + %d", rsc->id, id, node->weight, score);
	- node->weight = pe__add_scores(node->weight, score);
	+ node->weight = pcmk__add_scores(node->weight, score);
	}

	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	common_update_score(child_rsc, id, score);
	}
	}
	}

	gboolean
	is_parent(pe_resource_t child, pe_resource_t rsc)
	{
	pe_resource_t *parent = child;

	if (parent == NULL \|\| rsc == NULL) {
	return FALSE;
	}
	while (parent->parent != NULL) {
	if (parent->parent == rsc) {
	return TRUE;
	}
	parent = parent->parent;
	}
	return FALSE;
	}

	pe_resource_t *
	uber_parent(pe_resource_t * rsc)
	{
	pe_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while (parent->parent != NULL && parent->parent->variant != pe_container) {
	parent = parent->parent;
	}
	return parent;
	}

	void
	common_free(pe_resource_t * rsc)
	{
	if (rsc == NULL) {
	return;
	}

	pe_rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant);

	g_list_free(rsc->rsc_cons);
	g_list_free(rsc->rsc_cons_lhs);
	g_list_free(rsc->rsc_tickets);
	g_list_free(rsc->dangling_migrations);

	if (rsc->parameter_cache != NULL) {
	g_hash_table_destroy(rsc->parameter_cache);
	}
	#if ENABLE_VERSIONED_ATTRS
	if (rsc->versioned_parameters != NULL) {
	free_xml(rsc->versioned_parameters);
	}
	#endif
	if (rsc->meta != NULL) {
	g_hash_table_destroy(rsc->meta);
	}
	if (rsc->utilization != NULL) {
	g_hash_table_destroy(rsc->utilization);
	}

	if ((rsc->parent == NULL) && pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	free_xml(rsc->xml);
	rsc->xml = NULL;
	free_xml(rsc->orig_xml);
	rsc->orig_xml = NULL;

	/* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */
	} else if (rsc->orig_xml) {
	free_xml(rsc->xml);
	rsc->xml = NULL;
	}
	if (rsc->running_on) {
	g_list_free(rsc->running_on);
	rsc->running_on = NULL;
	}
	if (rsc->known_on) {
	g_hash_table_destroy(rsc->known_on);
	rsc->known_on = NULL;
	}
	if (rsc->actions) {
	g_list_free(rsc->actions);
	rsc->actions = NULL;
	}
	if (rsc->allowed_nodes) {
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = NULL;
	}
	g_list_free(rsc->fillers);
	g_list_free(rsc->rsc_location);
	pe_rsc_trace(rsc, "Resource freed");
	free(rsc->id);
	free(rsc->clone_name);
	free(rsc->allocated_to);
	free(rsc->variant_opaque);
	free(rsc->pending_task);
	free(rsc);
	}

	/*!
	* \brief
	* \internal Find a node (and optionally count all) where resource is active
	*
	* \param[in] rsc Resource to check
	* \param[out] count_all If not NULL, will be set to count of active nodes
	* \param[out] count_clean If not NULL, will be set to count of clean nodes
	*
	* \return An active node (or NULL if resource is not active anywhere)
	*
	* \note The order of preference is: an active node that is the resource's
	* partial migration source; if the resource's "requires" is "quorum" or
	* "nothing", the first active node in the list that is clean and online;
	* the first active node in the list.
	*/
	pe_node_t *
	pe__find_active_on(const pe_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pe_node_t *active = NULL;
	pe_node_t *node = NULL;
	bool keep_looking = FALSE;
	bool is_happy = FALSE;

	if (count_all) {
	*count_all = 0;
	}
	if (count_clean) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}

	for (GList *node_iter = rsc->running_on; node_iter != NULL;
	node_iter = node_iter->next) {

	node = node_iter->data;
	keep_looking = FALSE;

	is_happy = node->details->online && !node->details->unclean;

	if (count_all) {
	++*count_all;
	}
	if (count_clean && is_happy) {
	++*count_clean;
	}
	if (count_all \|\| count_clean) {
	// If we're counting, we need to go through entire list
	keep_looking = TRUE;
	}

	if (rsc->partial_migration_source != NULL) {
	if (node->details == rsc->partial_migration_source->details) {
	// This is the migration source
	active = node;
	} else {
	keep_looking = TRUE;
	}
	} else if (!pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	if (is_happy && (!active \|\| !active->details->online
	\|\| active->details->unclean)) {
	// This is the first clean node
	active = node;
	} else {
	keep_looking = TRUE;
	}
	}
	if (active == NULL) {
	// This is first node in list
	active = node;
	}

	if (keep_looking == FALSE) {
	// Don't waste time iterating if we don't have to
	break;
	}
	}
	return active;
	}

	/*!
	* \brief
	* \internal Find and count active nodes according to "requires"
	*
	* \param[in] rsc Resource to check
	* \param[out] count If not NULL, will be set to count of active nodes
	*
	* \return An active node (or NULL if resource is not active anywhere)
	*
	* \note This is a convenience wrapper for pe__find_active_on() where the count
	* of all active nodes or only clean active nodes is desired according to
	* the "requires" meta-attribute.
	*/
	pe_node_t *
	pe__find_active_requires(const pe_resource_t rsc, unsigned int count)
	{
	if (rsc && !pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	return pe__find_active_on(rsc, NULL, count);
	}
	return pe__find_active_on(rsc, count, NULL);
	}

	void
	pe__count_common(pe_resource_t *rsc)
	{
	if (rsc->children != NULL) {
	for (GList *item = rsc->children; item != NULL; item = item->next) {
	((pe_resource_t *) item->data)->fns->count(item->data);
	}

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)
	\|\| (rsc->role > RSC_ROLE_STOPPED)) {
	rsc->cluster->ninstances++;
	if (pe__resource_is_disabled(rsc)) {
	rsc->cluster->disabled_resources++;
	}
	if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	rsc->cluster->blocked_resources++;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource's next role
	*
	* \param[in,out] rsc Resource to be updated
	* \param[in] role Resource's new next role
	* \param[in] why Human-friendly reason why role is changing (for logs)
	*/
	void
	pe__set_next_role(pe_resource_t rsc, enum rsc_role_e role, const char why)
	{
	CRM_ASSERT((rsc != NULL) && (why != NULL));
	if (rsc->next_role != role) {
	pe_rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
	rsc->id, role2text(rsc->next_role), role2text(role), why);
	rsc->next_role = role;
	}
	}
	diff --git a/lib/pengine/failcounts.c b/lib/pengine/failcounts.c
	index 0e9d58465d..7d410083b3 100644
	--- a/lib/pengine/failcounts.c
	+++ b/lib/pengine/failcounts.c
	@@ -1,381 +1,381 @@
	/*
	* Copyright 2008-2022 the Pacemaker project contributors
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/types.h>
	#include <regex.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/pengine/internal.h>

	static gboolean
	is_matched_failure(const char rsc_id, xmlNode conf_op_xml,
	xmlNode *lrm_op_xml)
	{
	gboolean matched = FALSE;
	const char *conf_op_name = NULL;
	const char *lrm_op_task = NULL;
	const char *conf_op_interval_spec = NULL;
	guint conf_op_interval_ms = 0;
	guint lrm_op_interval_ms = 0;
	const char *lrm_op_id = NULL;
	char *last_failure_key = NULL;

	if (rsc_id == NULL \|\| conf_op_xml == NULL \|\| lrm_op_xml == NULL) {
	return FALSE;
	}

	// Get name and interval from configured op
	conf_op_name = crm_element_value(conf_op_xml, "name");
	conf_op_interval_spec = crm_element_value(conf_op_xml,
	XML_LRM_ATTR_INTERVAL);
	conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);

	// Get name and interval from op history entry
	lrm_op_task = crm_element_value(lrm_op_xml, XML_LRM_ATTR_TASK);
	crm_element_value_ms(lrm_op_xml, XML_LRM_ATTR_INTERVAL_MS,
	&lrm_op_interval_ms);

	if ((conf_op_interval_ms != lrm_op_interval_ms)
	\|\| !pcmk__str_eq(conf_op_name, lrm_op_task, pcmk__str_casei)) {
	return FALSE;
	}

	lrm_op_id = ID(lrm_op_xml);
	last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);

	if (pcmk__str_eq(last_failure_key, lrm_op_id, pcmk__str_casei)) {
	matched = TRUE;

	} else {
	char *expected_op_key = pcmk__op_key(rsc_id, conf_op_name,
	conf_op_interval_ms);

	if (pcmk__str_eq(expected_op_key, lrm_op_id, pcmk__str_casei)) {
	int rc = 0;
	int target_rc = pe__target_rc_from_xml(lrm_op_xml);

	crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc);
	if (rc != target_rc) {
	matched = TRUE;
	}
	}
	free(expected_op_key);
	}

	free(last_failure_key);
	return matched;
	}

	static gboolean
	block_failure(pe_node_t node, pe_resource_t rsc, xmlNode *xml_op,
	pe_working_set_t *data_set)
	{
	char *xml_name = clone_strip(rsc->id);

	/* @TODO This xpath search occurs after template expansion, but it is unable
	* to properly detect on-fail in id-ref, operation meta-attributes, or
	* op_defaults, or evaluate rules.
	*
	* Also, on-fail defaults to block (in unpack_operation()) for stop actions
	* when stonith is disabled.
	*
	* Ideally, we'd unpack the operation before this point, and pass in a
	* meta-attributes table that takes all that into consideration.
	*/
	char *xpath = crm_strdup_printf("//primitive[@id='%s']//op[@on-fail='block']",
	xml_name);

	xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
	gboolean should_block = FALSE;

	free(xpath);

	if (xpathObj) {
	int max = numXpathResults(xpathObj);
	int lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *pref = getXpathResult(xpathObj, lpc);

	if (xml_op) {
	should_block = is_matched_failure(xml_name, pref, xml_op);
	if (should_block) {
	break;
	}

	} else {
	const char *conf_op_name = NULL;
	const char *conf_op_interval_spec = NULL;
	guint conf_op_interval_ms = 0;
	char *lrm_op_xpath = NULL;
	xmlXPathObject *lrm_op_xpathObj = NULL;

	// Get name and interval from configured op
	conf_op_name = crm_element_value(pref, "name");
	conf_op_interval_spec = crm_element_value(pref, XML_LRM_ATTR_INTERVAL);
	conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);

	lrm_op_xpath = crm_strdup_printf("//node_state[@uname='%s']"
	"//lrm_resource[@id='%s']"
	"/lrm_rsc_op[@operation='%s'][@interval='%u']",
	node->details->uname, xml_name,
	conf_op_name, conf_op_interval_ms);
	lrm_op_xpathObj = xpath_search(data_set->input, lrm_op_xpath);

	free(lrm_op_xpath);

	if (lrm_op_xpathObj) {
	int max2 = numXpathResults(lrm_op_xpathObj);
	int lpc2 = 0;

	for (lpc2 = 0; lpc2 < max2; lpc2++) {
	xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
	lpc2);

	should_block = is_matched_failure(xml_name, pref,
	lrm_op_xml);
	if (should_block) {
	break;
	}
	}
	}
	freeXpathObject(lrm_op_xpathObj);

	if (should_block) {
	break;
	}
	}
	}
	}

	free(xml_name);
	freeXpathObject(xpathObj);

	return should_block;
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(pe_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
	}

	/*!
	* \internal
	* \brief Compile regular expression to match a failure-related node attribute
	*
	* \param[in] prefix Attribute prefix to match
	* \param[in] rsc_name Resource name to match as used in failure attributes
	* \param[in] is_legacy Whether DC uses per-resource fail counts
	* \param[in] is_unique Whether the resource is a globally unique clone
	* \param[out] re Where to store resulting regular expression
	*
	* \note Fail attributes are named like PREFIX-RESOURCE#OP_INTERVAL.
	* The caller is responsible for freeing re with regfree().
	*/
	static void
	generate_fail_regex(const char prefix, const char rsc_name,
	gboolean is_legacy, gboolean is_unique, regex_t *re)
	{
	char *pattern;

	/* @COMPAT DC < 1.1.17: Fail counts used to be per-resource rather than
	* per-operation.
	*/
	const char *op_pattern = (is_legacy? "" : "#.+_[0-9]+");

	/* Ignore instance numbers for anything other than globally unique clones.
	* Anonymous clone fail counts could contain an instance number if the
	* clone was initially unique, failed, then was converted to anonymous.
	* @COMPAT Also, before 1.1.8, anonymous clone fail counts always contained
	* clone instance numbers.
	*/
	const char *instance_pattern = (is_unique? "" : "(:[0-9]+)?");

	pattern = crm_strdup_printf("^%s-%s%s%s$", prefix, rsc_name,
	instance_pattern, op_pattern);
	CRM_LOG_ASSERT(regcomp(re, pattern, REG_EXTENDED\|REG_NOSUB) == 0);
	free(pattern);
	}

	/*!
	* \internal
	* \brief Compile regular expressions to match failure-related node attributes
	*
	* \param[in] rsc Resource being checked for failures
	* \param[in] data_set Data set (for CRM feature set version)
	* \param[out] failcount_re Storage for regular expression for fail count
	* \param[out] lastfailure_re Storage for regular expression for last failure
	*
	* \note The caller is responsible for freeing the expressions with regfree().
	*/
	static void
	generate_fail_regexes(pe_resource_t rsc, pe_working_set_t data_set,
	regex_t failcount_re, regex_t lastfailure_re)
	{
	char *rsc_name = rsc_fail_name(rsc);
	const char *version = crm_element_value(data_set->input, XML_ATTR_CRM_VERSION);
	gboolean is_legacy = (compare_version(version, "3.0.13") < 0);

	generate_fail_regex(PCMK__FAIL_COUNT_PREFIX, rsc_name, is_legacy,
	pcmk_is_set(rsc->flags, pe_rsc_unique), failcount_re);

	generate_fail_regex(PCMK__LAST_FAILURE_PREFIX, rsc_name, is_legacy,
	pcmk_is_set(rsc->flags, pe_rsc_unique), lastfailure_re);

	free(rsc_name);
	}

	int
	pe_get_failcount(pe_node_t node, pe_resource_t rsc, time_t *last_failure,
	uint32_t flags, xmlNode xml_op, pe_working_set_t data_set)
	{
	char *key = NULL;
	const char *value = NULL;
	regex_t failcount_re, lastfailure_re;
	int failcount = 0;
	time_t last = 0;
	GHashTableIter iter;

	generate_fail_regexes(rsc, data_set, &failcount_re, &lastfailure_re);

	/* Resource fail count is sum of all matching operation fail counts */
	g_hash_table_iter_init(&iter, node->details->attrs);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	if (regexec(&failcount_re, key, 0, NULL, 0) == 0) {
	- failcount = pe__add_scores(failcount, char2score(value));
	+ failcount = pcmk__add_scores(failcount, char2score(value));
	} else if (regexec(&lastfailure_re, key, 0, NULL, 0) == 0) {
	long long last_ll;

	if (pcmk__scan_ll(value, &last_ll, 0LL) == pcmk_rc_ok) {
	last = (time_t) QB_MAX(last, last_ll);
	}
	}
	}

	regfree(&failcount_re);
	regfree(&lastfailure_re);

	if ((failcount > 0) && (last > 0) && (last_failure != NULL)) {
	*last_failure = last;
	}

	/* If failure blocks the resource, disregard any failure timeout */
	if ((failcount > 0) && rsc->failure_timeout
	&& block_failure(node, rsc, xml_op, data_set)) {

	pe_warn("Ignoring failure timeout %d for %s because it conflicts with on-fail=block",
	rsc->failure_timeout, rsc->id);
	rsc->failure_timeout = 0;
	}

	/* If all failures have expired, ignore fail count */
	if (pcmk_is_set(flags, pe_fc_effective) && (failcount > 0) && (last > 0)
	&& rsc->failure_timeout) {

	time_t now = get_effective_time(data_set);

	if (now > (last + rsc->failure_timeout)) {
	crm_debug("Failcount for %s on %s expired after %ds",
	rsc->id, node->details->uname, rsc->failure_timeout);
	failcount = 0;
	}
	}

	/* We never want the fail counts of a bundle container's fillers to
	* count towards the container's fail count.
	*
	* Most importantly, a Pacemaker Remote connection to a bundle container
	* is a filler of the container, but can reside on a different node than the
	* container itself. Counting its fail count on its node towards the
	* container's fail count on that node could lead to attempting to stop the
	* container on the wrong node.
	*/

	if (pcmk_is_set(flags, pe_fc_fillers) && rsc->fillers
	&& !pe_rsc_is_bundled(rsc)) {

	GList *gIter = NULL;

	for (gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
	pe_resource_t filler = (pe_resource_t ) gIter->data;
	time_t filler_last_failure = 0;

	failcount += pe_get_failcount(node, filler, &filler_last_failure,
	flags, xml_op, data_set);

	if (last_failure && filler_last_failure > *last_failure) {
	*last_failure = filler_last_failure;
	}
	}

	if (failcount > 0) {
	char *score = score2char(failcount);

	crm_info("Container %s and the resources within it "
	"have failed %s time%s on %s",
	rsc->id, score, pcmk__plural_s(failcount),
	node->details->uname);
	free(score);
	}

	} else if (failcount > 0) {
	char *score = score2char(failcount);

	crm_info("%s has failed %s time%s on %s",
	rsc->id, score, pcmk__plural_s(failcount),
	node->details->uname);
	free(score);
	}

	return failcount;
	}

	/*!
	* \brief Schedule a controller operation to clear a fail count
	*
	* \param[in] rsc Resource with failure
	* \param[in] node Node failure occurred on
	* \param[in] reason Readable description why needed (for logging)
	* \param[in] data_set Working set for cluster
	*
	* \return Scheduled action
	*/
	pe_action_t *
	pe__clear_failcount(pe_resource_t rsc, pe_node_t node,
	const char reason, pe_working_set_t data_set)
	{
	char *key = NULL;
	pe_action_t *clear = NULL;

	CRM_CHECK(rsc && node && reason && data_set, return NULL);

	key = pcmk__op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
	clear = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE,
	data_set);
	add_hash_param(clear->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
	rsc->id, node->details->uname, reason, clear->uuid);
	return clear;
	}
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index 3fe13739ab..805b8e1bc4 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,4129 +1,4167 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <glib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	CRM_TRACE_INIT_DATA(pe_status);

	/* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
	* use pe__set_working_set_flags()/pe__clear_working_set_flags() so that the
	* flag is stringified more readably in log messages.
	*/
	#define set_config_flag(data_set, option, flag) do { \
	const char *scf_value = pe_pref((data_set)->config_hash, (option)); \
	if (scf_value != NULL) { \
	if (crm_is_true(scf_value)) { \
	(data_set)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", \
	crm_system_name, (data_set)->flags, \
	(flag), #flag); \
	} else { \
	(data_set)->flags = pcmk__clear_flags_as(__func__, __LINE__,\
	LOG_TRACE, "Working set", \
	crm_system_name, (data_set)->flags, \
	(flag), #flag); \
	} \
	} \
	} while(0)

	static void unpack_rsc_op(pe_resource_t rsc, pe_node_t node, xmlNode *xml_op,
	xmlNode **last_failure,
	enum action_fail_response *failed,
	pe_working_set_t *data_set);
	static void determine_remote_online_status(pe_working_set_t *data_set,
	pe_node_t *this_node);
	static void add_node_attrs(xmlNode attrs, pe_node_t node, bool overwrite,
	pe_working_set_t *data_set);
	static void determine_online_status(xmlNode node_state, pe_node_t this_node,
	pe_working_set_t *data_set);

	static void unpack_node_lrm(pe_node_t node, xmlNode xml,
	pe_working_set_t *data_set);


	// Bitmask for warnings we only want to print once
	uint32_t pe_wo = 0;

	static gboolean
	is_dangling_guest_node(pe_node_t *node)
	{
	/* we are looking for a remote-node that was supposed to be mapped to a
	* container resource, but all traces of that container have disappeared
	* from both the config and the status section. */
	if (pe__is_guest_or_remote_node(node) &&
	node->details->remote_rsc &&
	node->details->remote_rsc->container == NULL &&
	pcmk_is_set(node->details->remote_rsc->flags,
	pe_rsc_orphan_container_filler)) {
	return TRUE;
	}

	return FALSE;
	}

	/*!
	* \brief Schedule a fence action for a node
	*
	* \param[in,out] data_set Current working set of cluster
	* \param[in,out] node Node to fence
	* \param[in] reason Text description of why fencing is needed
	* \param[in] priority_delay Whether to consider `priority-fencing-delay`
	*/
	void
	pe_fence_node(pe_working_set_t * data_set, pe_node_t * node,
	const char *reason, bool priority_delay)
	{
	CRM_CHECK(node, return);

	/* A guest node is fenced by marking its container as failed */
	if (pe__is_guest_node(node)) {
	pe_resource_t *rsc = node->details->remote_rsc->container;

	if (!pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	crm_notice("Not fencing guest node %s "
	"(otherwise would because %s): "
	"its guest resource %s is unmanaged",
	node->details->uname, reason, rsc->id);
	} else {
	crm_warn("Guest node %s will be fenced "
	"(by recovering its guest resource %s): %s",
	node->details->uname, rsc->id, reason);

	/* We don't mark the node as unclean because that would prevent the
	* node from running resources. We want to allow it to run resources
	* in this transition if the recovery succeeds.
	*/
	node->details->remote_requires_reset = TRUE;
	pe__set_resource_flags(rsc, pe_rsc_failed\|pe_rsc_stop);
	}
	}

	} else if (is_dangling_guest_node(node)) {
	crm_info("Cleaning up dangling connection for guest node %s: "
	"fencing was already done because %s, "
	"and guest resource no longer exists",
	node->details->uname, reason);
	pe__set_resource_flags(node->details->remote_rsc,
	pe_rsc_failed\|pe_rsc_stop);

	} else if (pe__is_remote_node(node)) {
	pe_resource_t *rsc = node->details->remote_rsc;

	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	crm_notice("Not fencing remote node %s "
	"(otherwise would because %s): connection is unmanaged",
	node->details->uname, reason);
	} else if(node->details->remote_requires_reset == FALSE) {
	node->details->remote_requires_reset = TRUE;
	crm_warn("Remote node %s %s: %s",
	node->details->uname,
	pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
	reason);
	}
	node->details->unclean = TRUE;
	// No need to apply `priority-fencing-delay` for remote nodes
	pe_fence_op(node, NULL, TRUE, reason, FALSE, data_set);

	} else if (node->details->unclean) {
	crm_trace("Cluster node %s %s because %s",
	node->details->uname,
	pe_can_fence(data_set, node)? "would also be fenced" : "also is unclean",
	reason);

	} else {
	crm_warn("Cluster node %s %s: %s",
	node->details->uname,
	pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
	reason);
	node->details->unclean = TRUE;
	pe_fence_op(node, NULL, TRUE, reason, priority_delay, data_set);
	}
	}

	// @TODO xpaths can't handle templates, rules, or id-refs

	// nvpair with provides or requires set to unfencing
	#define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR \
	"[(@" XML_NVPAIR_ATTR_NAME "='" PCMK_STONITH_PROVIDES "'" \
	"or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
	"and @" XML_NVPAIR_ATTR_VALUE "='unfencing']"

	// unfencing in rsc_defaults or any resource
	#define XPATH_ENABLE_UNFENCING \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES \
	"//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR \
	"\|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG \
	"/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR

	static void
	set_if_xpath(uint64_t flag, const char xpath, pe_working_set_t data_set)
	{
	xmlXPathObjectPtr result = NULL;

	if (!pcmk_is_set(data_set->flags, flag)) {
	result = xpath_search(data_set->input, xpath);
	if (result && (numXpathResults(result) > 0)) {
	pe__set_working_set_flags(data_set, flag);
	}
	freeXpathObject(result);
	}
	}

	+/*!
	+ * \internal
	+ * \brief Set the node health values to use for "red", "yellow", and "green"
	+ *
	+ * \param[in] data_set Cluster working set
	+ */
	+static void
	+unpack_node_health_values(pe_working_set_t *data_set)
	+{
	+ const char *health_strategy = pe_pref(data_set->config_hash,
	+ "node-health-strategy");
	+
	+ if (pcmk__str_eq(health_strategy, "none",
	+ pcmk__str_null_matches\|pcmk__str_casei)) {
	+ pcmk__score_red = 0;
	+ pcmk__score_yellow = 0;
	+ pcmk__score_green = 0;
	+
	+ } else if (pcmk__str_eq(health_strategy, "migrate-on-red",
	+ pcmk__str_casei)) {
	+ pcmk__score_red = -INFINITY;
	+ pcmk__score_yellow = 0;
	+ pcmk__score_green = 0;
	+
	+ } else if (pcmk__str_eq(health_strategy, "only-green", pcmk__str_casei)) {
	+ pcmk__score_red = -INFINITY;
	+ pcmk__score_yellow = -INFINITY;
	+ pcmk__score_green = 0;
	+
	+ } else { // "progressive" or "custom"
	+ pcmk__score_red = char2score(pe_pref(data_set->config_hash,
	+ "node-health-red"));
	+ pcmk__score_green = char2score(pe_pref(data_set->config_hash,
	+ "node-health-green"));
	+ pcmk__score_yellow = char2score(pe_pref(data_set->config_hash,
	+ "node-health-yellow"));
	+ }
	+
	+ if ((pcmk__score_red != 0) \|\| (pcmk__score_yellow != 0)
	+ \|\| (pcmk__score_green != 0)) {
	+ crm_debug("Values of node health scores: red=%d yellow=%d green=%d",
	+ pcmk__score_red, pcmk__score_yellow, pcmk__score_green);
	+ }
	+}
	+
	gboolean
	unpack_config(xmlNode * config, pe_working_set_t * data_set)
	{
	const char *value = NULL;
	GHashTable *config_hash = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	data_set->config_hash = config_hash;

	pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
	CIB_OPTIONS_FIRST, FALSE, data_set);

	verify_pe_options(data_set->config_hash);

	set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
	if (!pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_info("Watchdog-based self-fencing will be performed via SBD if "
	"fencing is required and stonith-watchdog-timeout is nonzero");
	pe__set_working_set_flags(data_set, pe_flag_have_stonith_resource);
	}

	/* Set certain flags via xpath here, so they can be used before the relevant
	* configuration sections are unpacked.
	*/
	set_if_xpath(pe_flag_enable_unfencing, XPATH_ENABLE_UNFENCING, data_set);

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	data_set->stonith_timeout = (int) crm_parse_interval_spec(value);
	crm_debug("STONITH timeout: %d", data_set->stonith_timeout);

	set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
	crm_debug("STONITH of failed nodes is %s",
	pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)? "enabled" : "disabled");

	data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
	if (!strcmp(data_set->stonith_action, "poweroff")) {
	pe_warn_once(pe_wo_poweroff,
	"Support for stonith-action of 'poweroff' is deprecated "
	"and will be removed in a future release (use 'off' instead)");
	data_set->stonith_action = "off";
	}
	crm_trace("STONITH will %s nodes", data_set->stonith_action);

	set_config_flag(data_set, "concurrent-fencing", pe_flag_concurrent_fencing);
	crm_debug("Concurrent fencing is %s",
	pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)? "enabled" : "disabled");

	value = pe_pref(data_set->config_hash,
	XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
	if (value) {
	data_set->priority_fencing_delay = crm_parse_interval_spec(value) / 1000;
	crm_trace("Priority fencing delay is %ds", data_set->priority_fencing_delay);
	}

	set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
	crm_debug("Stop all active resources: %s",
	pcmk__btoa(pcmk_is_set(data_set->flags, pe_flag_stop_everything)));

	set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
	if (pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pe_pref(data_set->config_hash, "no-quorum-policy");

	if (pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
	data_set->no_quorum_policy = no_quorum_ignore;

	} else if (pcmk__str_eq(value, "freeze", pcmk__str_casei)) {
	data_set->no_quorum_policy = no_quorum_freeze;

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	data_set->no_quorum_policy = no_quorum_demote;

	} else if (pcmk__str_eq(value, "suicide", pcmk__str_casei)) {
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	int do_panic = 0;

	crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC,
	&do_panic);
	if (do_panic \|\| pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	data_set->no_quorum_policy = no_quorum_suicide;
	} else {
	crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
	data_set->no_quorum_policy = no_quorum_stop;
	}
	} else {
	pcmk__config_err("Resetting no-quorum-policy to 'stop' because "
	"fencing is disabled");
	data_set->no_quorum_policy = no_quorum_stop;
	}

	} else {
	data_set->no_quorum_policy = no_quorum_stop;
	}

	switch (data_set->no_quorum_policy) {
	case no_quorum_freeze:
	crm_debug("On loss of quorum: Freeze resources");
	break;
	case no_quorum_stop:
	crm_debug("On loss of quorum: Stop ALL resources");
	break;
	case no_quorum_demote:
	crm_debug("On loss of quorum: "
	"Demote promotable resources and stop other resources");
	break;
	case no_quorum_suicide:
	crm_notice("On loss of quorum: Fence all remaining nodes");
	break;
	case no_quorum_ignore:
	crm_notice("On loss of quorum: Ignore");
	break;
	}

	set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
	crm_trace("Orphan resources are %s",
	pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)? "stopped" : "ignored");

	set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
	crm_trace("Orphan resource actions are %s",
	pcmk_is_set(data_set->flags, pe_flag_stop_action_orphans)? "stopped" : "ignored");

	value = pe_pref(data_set->config_hash, "remove-after-stop");
	if (value != NULL) {
	if (crm_is_true(value)) {
	pe__set_working_set_flags(data_set, pe_flag_remove_after_stop);
	#ifndef PCMK__COMPAT_2_0
	pe_warn_once(pe_wo_remove_after,
	"Support for the remove-after-stop cluster property is"
	" deprecated and will be removed in a future release");
	#endif
	} else {
	pe__clear_working_set_flags(data_set, pe_flag_remove_after_stop);
	}
	}

	set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
	crm_trace("Maintenance mode: %s",
	pcmk__btoa(pcmk_is_set(data_set->flags, pe_flag_maintenance_mode)));

	set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
	crm_trace("Start failures are %s",
	pcmk_is_set(data_set->flags, pe_flag_start_failure_fatal)? "always fatal" : "handled by failcount");

	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	set_config_flag(data_set, "startup-fencing", pe_flag_startup_fencing);
	}
	if (pcmk_is_set(data_set->flags, pe_flag_startup_fencing)) {
	crm_trace("Unseen nodes will be fenced");
	} else {
	pe_warn_once(pe_wo_blind, "Blind faith: not fencing unseen nodes");
	}

	- pcmk__score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
	- pcmk__score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
	- pcmk__score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));
	-
	- crm_debug("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
	- pe_pref(data_set->config_hash, "node-health-red"),
	- pe_pref(data_set->config_hash, "node-health-yellow"),
	- pe_pref(data_set->config_hash, "node-health-green"));
	+ unpack_node_health_values(data_set);

	data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
	crm_trace("Placement strategy: %s", data_set->placement_strategy);

	set_config_flag(data_set, "shutdown-lock", pe_flag_shutdown_lock);
	crm_trace("Resources will%s be locked to cleanly shut down nodes",
	(pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)? "" : " not"));
	if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
	value = pe_pref(data_set->config_hash,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT);
	data_set->shutdown_lock = crm_parse_interval_spec(value) / 1000;
	crm_trace("Shutdown locks expire after %us", data_set->shutdown_lock);
	}

	return TRUE;
	}

	pe_node_t *
	pe_create_node(const char id, const char uname, const char *type,
	const char score, pe_working_set_t data_set)
	{
	pe_node_t *new_node = NULL;

	if (pe_find_node(data_set->nodes, uname) != NULL) {
	pcmk__config_warn("More than one node entry has name '%s'", uname);
	}

	new_node = calloc(1, sizeof(pe_node_t));
	if (new_node == NULL) {
	return NULL;
	}

	new_node->weight = char2score(score);
	new_node->fixed = FALSE;
	new_node->details = calloc(1, sizeof(struct pe_node_shared_s));

	if (new_node->details == NULL) {
	free(new_node);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->rsc_discovery_enabled = TRUE;
	new_node->details->running_rsc = NULL;
	new_node->details->type = node_ping;

	if (pcmk__str_eq(type, "remote", pcmk__str_casei)) {
	new_node->details->type = node_remote;
	pe__set_working_set_flags(data_set, pe_flag_have_remote_nodes);
	} else if (pcmk__str_eq(type, "member", pcmk__str_null_matches \| pcmk__str_casei)) {
	new_node->details->type = node_member;
	}

	new_node->details->attrs = pcmk__strkey_table(free, free);

	if (pe__is_guest_or_remote_node(new_node)) {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("remote"));
	} else {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("cluster"));
	}

	new_node->details->utilization = pcmk__strkey_table(free, free);
	new_node->details->digest_cache = pcmk__strkey_table(free,
	pe__free_digests);

	data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node, sort_node_uname);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, pe_working_set_t *data)
	{
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = ID(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	const char *is_managed = NULL;

	for (attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (!pcmk__str_eq((const char *)attr_set->name, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	continue;
	}

	for (attr = pcmk__xe_first_child(attr_set); attr != NULL;
	attr = pcmk__xe_next(attr)) {
	const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
	const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);

	if (pcmk__str_eq(name, XML_RSC_ATTR_REMOTE_NODE, pcmk__str_casei)) {
	remote_name = value;
	} else if (pcmk__str_eq(name, "remote-addr", pcmk__str_casei)) {
	remote_server = value;
	} else if (pcmk__str_eq(name, "remote-port", pcmk__str_casei)) {
	remote_port = value;
	} else if (pcmk__str_eq(name, "remote-connect-timeout", pcmk__str_casei)) {
	connect_timeout = value;
	} else if (pcmk__str_eq(name, "remote-allow-migrate", pcmk__str_casei)) {
	remote_allow_migrate=value;
	} else if (pcmk__str_eq(name, XML_RSC_ATTR_MANAGED, pcmk__str_casei)) {
	is_managed = value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (pe_find_resource(data->resources, remote_name) != NULL) {
	return NULL;
	}

	pe_create_remote_xml(parent, remote_name, container_id,
	remote_allow_migrate, is_managed,
	connect_timeout, remote_server, remote_port);
	return remote_name;
	}

	static void
	handle_startup_fencing(pe_working_set_t data_set, pe_node_t new_node)
	{
	if ((new_node->details->type == node_remote) && (new_node->details->remote_rsc == NULL)) {
	/* Ignore fencing for remote nodes that don't have a connection resource
	* associated with them. This happens when remote node entries get left
	* in the nodes section after the connection resource is removed.
	*/
	return;
	}

	if (pcmk_is_set(data_set->flags, pe_flag_startup_fencing)) {
	// All nodes are unclean until we've seen their status entry
	new_node->details->unclean = TRUE;

	} else {
	// Blind faith ...
	new_node->details->unclean = FALSE;
	}

	/* We need to be able to determine if a node's status section
	* exists or not separate from whether the node is unclean. */
	new_node->details->unseen = TRUE;
	}

	gboolean
	unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	pe_node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	for (xml_obj = pcmk__xe_first_child(xml_nodes); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, pcmk__str_none)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
	type = crm_element_value(xml_obj, XML_ATTR_TYPE);
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
	"> entry in configuration without id");
	continue;
	}
	new_node = pe_create_node(id, uname, type, score, data_set);

	if (new_node == NULL) {
	return FALSE;
	}

	/* if(data_set->have_quorum == FALSE */
	/* && data_set->no_quorum_policy == no_quorum_stop) { */
	/* /\* start shutting resources down \/ /
	/* new_node->weight = -INFINITY; */
	/* } */

	handle_startup_fencing(data_set, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, data_set);
	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
	new_node->details->utilization, NULL,
	FALSE, data_set);

	crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
	}
	}

	if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
	crm_info("Creating a fake local node");
	pe_create_node(data_set->localhost, data_set->localhost, NULL, 0,
	data_set);
	}

	return TRUE;
	}

	static void
	setup_container(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	const char *container_id = NULL;

	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) setup_container, data_set);
	return;
	}

	container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
	if (container_id && !pcmk__str_eq(container_id, rsc->id, pcmk__str_casei)) {
	pe_resource_t *container = pe_find_resource(data_set->resources, container_id);

	if (container) {
	rsc->container = container;
	pe__set_resource_flags(container, pe_rsc_is_container);
	container->fillers = g_list_append(container->fillers, rsc);
	pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
	} else {
	pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;

	/* Create remote nodes and guest nodes from the resource configuration
	* before unpacking resources.
	*/
	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	const char *new_node_id = NULL;

	/* Check for remote nodes, which are defined by ocf:pacemaker:remote
	* primitives.
	*/
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = ID(xml_obj);
	/* The "pe_find_node" check is here to make sure we don't iterate over
	* an expanded node that has already been added to the node list. */
	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found remote node %s defined by resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	data_set);
	}
	continue;
	}

	/* Check for guest nodes, which are defined by special meta-attributes
	* of a primitive of any type (for example, VirtualDomain or Xen).
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, pcmk__str_none)) {
	/* This will add an ocf:pacemaker:remote primitive to the
	* configuration for the guest node's connection, to be unpacked
	* later.
	*/
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, data_set);
	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found guest node %s in resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	data_set);
	}
	continue;
	}

	/* Check for guest nodes inside a group. Clones are currently not
	* supported as guest nodes.
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, pcmk__str_none)) {
	xmlNode *xml_obj2 = NULL;
	for (xml_obj2 = pcmk__xe_first_child(xml_obj); xml_obj2 != NULL;
	xml_obj2 = pcmk__xe_next(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, data_set);

	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found guest node %s in resource %s inside group %s",
	new_node_id, ID(xml_obj2), ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	data_set);
	}
	}
	}
	}
	return TRUE;
	}

	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the scheduler calculations.
	*/
	static void
	link_rsc2remotenode(pe_working_set_t data_set, pe_resource_t new_rsc)
	{
	pe_node_t *remote_node = NULL;

	if (new_rsc->is_remote_node == FALSE) {
	return;
	}

	if (pcmk_is_set(data_set->flags, pe_flag_quick_location)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	remote_node = pe_find_node(data_set->nodes, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return);

	pe_rsc_trace(new_rsc, "Linking remote connection resource %s to node %s",
	new_rsc->id, remote_node->details->uname);
	remote_node->details->remote_rsc = new_rsc;

	if (new_rsc->container == NULL) {
	/* Handle start-up fencing for remote nodes (as opposed to guest nodes)
	* the same as is done for cluster nodes.
	*/
	handle_startup_fencing(data_set, remote_node);

	} else {
	/* pe_create_node() marks the new node as "remote" or "cluster"; now
	* that we know the node is a guest node, update it correctly.
	*/
	g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("container"));
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	pe_tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	/*!
	* \internal
	* \brief Parse configuration XML for resource information
	*
	* \param[in] xml_resources Top of resource configuration XML
	* \param[in,out] data_set Where to put resource information
	*
	* \return TRUE
	*
	* \note unpack_remote_nodes() MUST be called before this, so that the nodes can
	* be used when common_unpack() calls resource_location()
	*/
	gboolean
	unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	GList *gIter = NULL;

	data_set->template_rsc_sets = pcmk__strkey_table(free, destroy_tag);

	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	pe_resource_t *new_rsc = NULL;

	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, pcmk__str_none)) {
	const char *template_id = ID(xml_obj);

	if (template_id && g_hash_table_lookup_extended(data_set->template_rsc_sets,
	template_id, NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	g_hash_table_insert(data_set->template_rsc_sets, strdup(template_id), NULL);
	}
	continue;
	}

	crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
	if (common_unpack(xml_obj, &new_rsc, NULL, data_set) && (new_rsc != NULL)) {
	data_set->resources = g_list_append(data_set->resources, new_rsc);
	pe_rsc_trace(new_rsc, "Added resource %s", new_rsc->id);

	} else {
	pcmk__config_err("Ignoring <%s> resource '%s' "
	"because configuration is invalid",
	crm_element_name(xml_obj), crm_str(ID(xml_obj)));
	if (new_rsc != NULL && new_rsc->fns != NULL) {
	new_rsc->fns->free(new_rsc);
	}
	}
	}

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	setup_container(rsc, data_set);
	link_rsc2remotenode(data_set, rsc);
	}

	data_set->resources = g_list_sort(data_set->resources, sort_rsc_priority);
	if (pcmk_is_set(data_set->flags, pe_flag_quick_location)) {
	/* Ignore */

	} else if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)
	&& !pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {

	pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
	pcmk__config_err("Either configure some or disable STONITH with the stonith-enabled option");
	pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
	{
	xmlNode *xml_tag = NULL;

	data_set->tags = pcmk__strkey_table(free, destroy_tag);

	for (xml_tag = pcmk__xe_first_child(xml_tags); xml_tag != NULL;
	xml_tag = pcmk__xe_next(xml_tag)) {

	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = ID(xml_tag);

	if (!pcmk__str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, pcmk__str_none)) {
	continue;
	}

	if (tag_id == NULL) {
	pcmk__config_err("Ignoring <%s> without " XML_ATTR_ID,
	crm_element_name(xml_tag));
	continue;
	}

	for (xml_obj_ref = pcmk__xe_first_child(xml_tag); xml_obj_ref != NULL;
	xml_obj_ref = pcmk__xe_next(xml_obj_ref)) {

	const char *obj_ref = ID(xml_obj_ref);

	if (!pcmk__str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, pcmk__str_none)) {
	continue;
	}

	if (obj_ref == NULL) {
	pcmk__config_err("Ignoring <%s> for tag '%s' without " XML_ATTR_ID,
	crm_element_name(xml_obj_ref), tag_id);
	continue;
	}

	if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	pe_ticket_t *ticket = NULL;

	ticket_id = ID(xml_ticket);
	if (pcmk__str_empty(ticket_id)) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, data_set);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = crm_element_value(xml_ticket, prop_name);

	if (pcmk__str_eq(prop_name, XML_ATTR_ID, pcmk__str_none)) {
	continue;
	}
	g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
	}

	granted = g_hash_table_lookup(ticket->state, "granted");
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, "last-granted");
	if (last_granted) {
	long long last_granted_ll;

	pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
	ticket->last_granted = (time_t) last_granted_ll;
	}

	standby = g_hash_table_lookup(ticket->state, "standby");
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_tickets); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (!pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, pcmk__str_none)) {
	continue;
	}
	unpack_ticket_state(xml_obj, data_set);
	}

	return TRUE;
	}

	static void
	unpack_handle_remote_attrs(pe_node_t this_node, xmlNode state, pe_working_set_t * data_set)
	{
	const char *resource_discovery_enabled = NULL;
	xmlNode *attrs = NULL;
	pe_resource_t *rsc = NULL;

	if (!pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	return;
	}

	if ((this_node == NULL) \|\| !pe__is_guest_or_remote_node(this_node)) {
	return;
	}
	crm_trace("Processing remote node id=%s, uname=%s", this_node->details->id, this_node->details->uname);

	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_MAINTENANCE),
	&(this_node->details->remote_maintenance), 0);

	rsc = this_node->details->remote_rsc;
	if (this_node->details->remote_requires_reset == FALSE) {
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	}
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, data_set);

	if (pe__shutdown_requested(this_node)) {
	crm_info("Node %s is shutting down", this_node->details->uname);
	this_node->details->shutdown = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
	crm_info("Node %s is in standby-mode", this_node->details->uname);
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) \|\|
	((rsc != NULL) && !pcmk_is_set(rsc->flags, pe_rsc_managed))) {
	crm_info("Node %s is in maintenance-mode", this_node->details->uname);
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	if (pe__is_remote_node(this_node)
	&& !pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	crm_warn("Ignoring %s attribute on remote node %s because stonith is disabled",
	XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
	} else {
	/* This is either a remote node with fencing enabled, or a guest
	* node. We don't care whether fencing is enabled when fencing guest
	* nodes, because they are "fenced" by recovering their containing
	* resource.
	*/
	crm_info("Node %s has resource discovery disabled", this_node->details->uname);
	this_node->details->rsc_discovery_enabled = FALSE;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack a cluster node's transient attributes
	*
	* \param[in] state CIB node state XML
	* \param[in] node Cluster node whose attributes are being unpacked
	* \param[in] data_set Cluster working set
	*/
	static void
	unpack_transient_attributes(xmlNode state, pe_node_t node,
	pe_working_set_t *data_set)
	{
	const char *discovery = NULL;
	xmlNode *attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);

	add_node_attrs(attrs, node, TRUE, data_set);

	if (crm_is_true(pe_node_attribute_raw(node, "standby"))) {
	crm_info("Node %s is in standby-mode", node->details->uname);
	node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(node, "maintenance"))) {
	crm_info("Node %s is in maintenance-mode", node->details->uname);
	node->details->maintenance = TRUE;
	}

	discovery = pe_node_attribute_raw(node, XML_NODE_ATTR_RSC_DISCOVERY);
	if ((discovery != NULL) && !crm_is_true(discovery)) {
	crm_warn("Ignoring %s attribute for node %s because disabling "
	"resource discovery is not allowed for cluster nodes",
	XML_NODE_ATTR_RSC_DISCOVERY, node->details->uname);
	}
	}

	/*!
	* \internal
	* \brief Unpack a node state entry (first pass)
	*
	* Unpack one node state entry from status. This unpacks information from the
	* node_state element itself and node attributes inside it, but not the
	* resource history inside it. Multiple passes through the status are needed to
	* fully unpack everything.
	*
	* \param[in] state CIB node state XML
	* \param[in] data_set Cluster working set
	*/
	static void
	unpack_node_state(xmlNode state, pe_working_set_t data_set)
	{
	const char *id = NULL;
	const char *uname = NULL;
	pe_node_t *this_node = NULL;

	id = crm_element_value(state, XML_ATTR_ID);
	if (id == NULL) {
	crm_warn("Ignoring malformed " XML_CIB_TAG_STATE " entry without "
	XML_ATTR_ID);
	return;
	}

	uname = crm_element_value(state, XML_ATTR_UNAME);
	if (uname == NULL) {
	crm_warn("Ignoring malformed " XML_CIB_TAG_STATE " entry without "
	XML_ATTR_UNAME);
	return;
	}

	this_node = pe_find_node_any(data_set->nodes, id, uname);
	if (this_node == NULL) {
	pcmk__config_warn("Ignoring recorded node state for '%s' because "
	"it is no longer in the configuration", uname);
	return;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	/* We can't determine the online status of Pacemaker Remote nodes until
	* after all resource history has been unpacked. In this first pass, we
	* do need to mark whether the node has been fenced, as this plays a
	* role during unpacking cluster node resource state.
	*/
	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_FENCED),
	&(this_node->details->remote_was_fenced), 0);
	return;
	}

	unpack_transient_attributes(state, this_node, data_set);

	/* Provisionally mark this cluster node as clean. We have at least seen it
	* in the current cluster's lifetime.
	*/
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;

	crm_trace("Determining online status of cluster node %s (id %s)",
	this_node->details->uname, id);
	determine_online_status(state, this_node, data_set);

	if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)
	&& this_node->details->online
	&& (data_set->no_quorum_policy == no_quorum_suicide)) {
	/* Everything else should flow from this automatically
	* (at least until the scheduler becomes able to migrate off
	* healthy resources)
	*/
	pe_fence_node(data_set, this_node, "cluster does not have quorum",
	FALSE);
	}
	}

	/*!
	* \internal
	* \brief Unpack nodes' resource history as much as possible
	*
	* Unpack as many nodes' resource history as possible in one pass through the
	* status. We need to process Pacemaker Remote nodes' connections/containers
	* before unpacking their history; the connection/container history will be
	* in another node's history, so it might take multiple passes to unpack
	* everything.
	*
	* \param[in] status CIB XML status section
	* \param[in] fence If true, treat any not-yet-unpacked nodes as unseen
	* \param[in] data_set Cluster working set
	*
	* \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
	* or EAGAIN if more unpacking remains to be done)
	*/
	static int
	unpack_node_history(xmlNode status, bool fence, pe_working_set_t data_set)
	{
	int rc = pcmk_rc_ok;

	// Loop through all node_state entries in CIB status
	for (xmlNode *state = first_named_child(status, XML_CIB_TAG_STATE);
	state != NULL; state = crm_next_same_xml(state)) {

	const char *id = ID(state);
	const char *uname = crm_element_value(state, XML_ATTR_UNAME);
	pe_node_t *this_node = NULL;

	if ((id == NULL) \|\| (uname == NULL)) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history from malformed "
	XML_CIB_TAG_STATE " without id and/or uname");
	continue;
	}

	this_node = pe_find_node_any(data_set->nodes, id, uname);
	if (this_node == NULL) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history for node %s because "
	"no longer in configuration", id);
	continue;
	}

	if (this_node->details->unpacked) {
	crm_trace("Not unpacking resource history for node %s because "
	"already unpacked", id);
	continue;
	}

	if (fence) {
	// We're processing all remaining nodes

	} else if (pe__is_guest_node(this_node)) {
	/* We can unpack a guest node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection and containing resource are both up.
	*/
	pe_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL) \|\| (rsc->role != RSC_ROLE_STARTED)
	\|\| (rsc->container->role != RSC_ROLE_STARTED)) {
	crm_trace("Not unpacking resource history for guest node %s "
	"because container and connection are not known to "
	"be up", id);
	continue;
	}

	} else if (pe__is_remote_node(this_node)) {
	/* We can unpack a remote node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection is up, with the exception of when shutdown locks are
	* in use.
	*/
	pe_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL)
	\|\| (!pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)
	&& (rsc->role != RSC_ROLE_STARTED))) {
	crm_trace("Not unpacking resource history for remote node %s "
	"because connection is not known to be up", id);
	continue;
	}

	/* If fencing and shutdown locks are disabled and we're not processing
	* unseen nodes, then we don't want to unpack offline nodes until online
	* nodes have been unpacked. This allows us to number active clone
	* instances first.
	*/
	} else if (!pcmk_any_flags_set(data_set->flags, pe_flag_stonith_enabled
	\|pe_flag_shutdown_lock)
	&& !this_node->details->online) {
	crm_trace("Not unpacking resource history for offline "
	"cluster node %s", id);
	continue;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(data_set, this_node);
	unpack_handle_remote_attrs(this_node, state, data_set);
	}

	crm_trace("Unpacking resource history for %snode %s",
	(fence? "unseen " : ""), id);

	this_node->details->unpacked = TRUE;
	unpack_node_lrm(this_node, state, data_set);

	rc = EAGAIN; // Other node histories might depend on this one
	}
	return rc;
	}

	/* remove nodes that are down, stopping */
	/* create positive rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode * status, pe_working_set_t * data_set)
	{
	xmlNode *state = NULL;

	crm_trace("Beginning unpack");

	if (data_set->tickets == NULL) {
	data_set->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	for (state = pcmk__xe_first_child(status); state != NULL;
	state = pcmk__xe_next(state)) {

	if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
	unpack_tickets_state((xmlNode *) state, data_set);

	} else if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	unpack_node_state(state, data_set);
	}
	}

	while (unpack_node_history(status, FALSE, data_set) == EAGAIN) {
	crm_trace("Another pass through node resource histories is needed");
	}

	// Now catch any nodes we didn't see
	unpack_node_history(status,
	pcmk_is_set(data_set->flags, pe_flag_stonith_enabled),
	data_set);

	/* Now that we know where resources are, we can schedule stops of containers
	* with failed bundle connections
	*/
	if (data_set->stop_needed != NULL) {
	for (GList *item = data_set->stop_needed; item; item = item->next) {
	pe_resource_t *container = item->data;
	pe_node_t *node = pe__current_node(container);

	if (node) {
	stop_action(container, node, FALSE);
	}
	}
	g_list_free(data_set->stop_needed);
	data_set->stop_needed = NULL;
	}

	/* Now that we know status of all Pacemaker Remote connections and nodes,
	* we can stop connections for node shutdowns, and check the online status
	* of remote/guest nodes that didn't have any node history to unpack.
	*/
	for (GList *gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t *this_node = gIter->data;

	if (!pe__is_guest_or_remote_node(this_node)) {
	continue;
	}
	if (this_node->details->shutdown
	&& (this_node->details->remote_rsc != NULL)) {
	pe__set_next_role(this_node->details->remote_rsc, RSC_ROLE_STOPPED,
	"remote shutdown");
	}
	if (!this_node->details->unpacked) {
	determine_remote_online_status(data_set, this_node);
	}
	}

	return TRUE;
	}

	static gboolean
	determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_state,
	pe_node_t * this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);

	if (!crm_is_true(in_cluster)) {
	crm_trace("Node is down: in_cluster=%s", crm_str(in_cluster));

	} else if (pcmk__str_eq(is_peer, ONLINESTATUS, pcmk__str_casei)) {
	if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	online = TRUE;
	} else {
	crm_debug("Node is not ready to run resources: %s", join);
	}

	} else if (this_node->details->expected_up == FALSE) {
	crm_trace("Controller is down: in_cluster=%s", crm_str(in_cluster));
	crm_trace("\tis_peer=%s, join=%s, expected=%s",
	crm_str(is_peer), crm_str(join), crm_str(exp_state));

	} else {
	/* mark it unclean */
	pe_fence_node(data_set, this_node, "peer is unexpectedly down", FALSE);
	crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",
	crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));
	}
	return online;
	}

	static gboolean
	determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_state,
	pe_node_t * this_node)
	{
	gboolean online = FALSE;
	gboolean do_terminate = FALSE;
	bool crmd_online = FALSE;
	const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
	const char *terminate = pe_node_attribute_raw(this_node, "terminate");

	/*
	- XML_NODE_IN_CLUSTER ::= true\|false
	- XML_NODE_IS_PEER ::= online\|offline
	- XML_NODE_JOIN_STATE ::= member\|down\|pending\|banned
	- XML_NODE_EXPECTED ::= member\|down
	*/

	if (crm_is_true(terminate)) {
	do_terminate = TRUE;

	} else if (terminate != NULL && strlen(terminate) > 0) {
	/* could be a time() value */
	char t = terminate[0];

	if (t != '0' && isdigit(t)) {
	do_terminate = TRUE;
	}
	}

	crm_trace("%s: in_cluster=%s, is_peer=%s, join=%s, expected=%s, term=%d",
	this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
	crm_str(join), crm_str(exp_state), do_terminate);

	online = crm_is_true(in_cluster);
	crmd_online = pcmk__str_eq(is_peer, ONLINESTATUS, pcmk__str_casei);
	if (exp_state == NULL) {
	exp_state = CRMD_JOINSTATE_DOWN;
	}

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", this_node->details->uname);

	/* Slightly different criteria since we can't shut down a dead peer */
	online = crmd_online;

	} else if (in_cluster == NULL) {
	pe_fence_node(data_set, this_node, "peer has not been seen by the cluster", FALSE);

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_casei)) {
	pe_fence_node(data_set, this_node,
	"peer failed Pacemaker membership criteria", FALSE);

	} else if (do_terminate == FALSE && pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN, pcmk__str_casei)) {

	if (crm_is_true(in_cluster) \|\| crmd_online) {
	crm_info("- Node %s is not ready to run resources", this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up", this_node->details->uname);
	}

	} else if (do_terminate && pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_casei)
	&& crm_is_true(in_cluster) == FALSE && !crmd_online) {
	crm_info("Node %s was just shot", this_node->details->uname);
	online = FALSE;

	} else if (crm_is_true(in_cluster) == FALSE) {
	// Consider `priority-fencing-delay` for lost nodes
	pe_fence_node(data_set, this_node, "peer is no longer part of the cluster", TRUE);

	} else if (!crmd_online) {
	pe_fence_node(data_set, this_node, "peer process is no longer available", FALSE);

	/* Everything is running at this point, now check join state */
	} else if (do_terminate) {
	pe_fence_node(data_set, this_node, "termination was requested", FALSE);

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	crm_info("Node %s is active", this_node->details->uname);

	} else if (pcmk__strcase_any_of(join, CRMD_JOINSTATE_PENDING, CRMD_JOINSTATE_DOWN, NULL)) {
	crm_info("Node %s is not ready to run resources", this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(data_set, this_node, "peer was in an unknown state", FALSE);
	crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",
	this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
	crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);
	}

	return online;
	}

	static void
	determine_remote_online_status(pe_working_set_t * data_set, pe_node_t * this_node)
	{
	pe_resource_t *rsc = this_node->details->remote_rsc;
	pe_resource_t *container = NULL;
	pe_node_t *host = NULL;

	/* If there is a node state entry for a (former) Pacemaker Remote node
	* but no resource creating that node, the node's connection resource will
	* be NULL. Consider it an offline remote node in that case.
	*/
	if (rsc == NULL) {
	this_node->details->online = FALSE;
	goto remote_online_done;
	}

	container = rsc->container;

	if (container && pcmk__list_of_1(rsc->running_on)) {
	host = rsc->running_on->data;
	}

	/* If the resource is currently started, mark it online. */
	if (rsc->role == RSC_ROLE_STARTED) {
	crm_trace("%s node %s presumed ONLINE because connection resource is started",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) {
	crm_trace("%s node %s shutting down because connection resource is stopping",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if(container && pcmk_is_set(container->flags, pe_rsc_failed)) {
	crm_trace("Guest node %s UNCLEAN because guest resource failed",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	crm_trace("%s node %s OFFLINE because connection resource failed",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;

	} else if (rsc->role == RSC_ROLE_STOPPED
	\|\| (container && container->role == RSC_ROLE_STOPPED)) {

	crm_trace("%s node %s OFFLINE because its resource is stopped",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = FALSE;

	} else if (host && (host->details->online == FALSE)
	&& host->details->unclean) {
	crm_trace("Guest node %s UNCLEAN because host is unclean",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;
	}

	remote_online_done:
	crm_trace("Remote node %s online=%s",
	this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
	}

	static void
	determine_online_status(xmlNode * node_state, pe_node_t * this_node, pe_working_set_t * data_set)
	{
	gboolean online = FALSE;
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);

	CRM_CHECK(this_node != NULL, return);

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;

	if (pe__shutdown_requested(this_node)) {
	this_node->details->shutdown = TRUE;

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	this_node->details->expected_up = TRUE;
	}

	if (this_node->details->type == node_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	online = determine_online_status_no_fencing(data_set, node_state, this_node);

	} else {
	online = determine_online_status_fencing(data_set, node_state, this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* don't run resources here */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if (this_node->details->type == node_ping) {
	crm_info("Node %s is not a Pacemaker node", this_node->details->uname);

	} else if (this_node->details->unclean) {
	pe_proc_warn("Node %s is unclean", this_node->details->uname);

	} else if (this_node->details->online) {
	crm_info("Node %s is %s", this_node->details->uname,
	this_node->details->shutdown ? "shutting down" :
	this_node->details->pending ? "pending" :
	this_node->details->standby ? "standby" :
	this_node->details->maintenance ? "maintenance" : "online");

	} else {
	crm_trace("Node %s is offline", this_node->details->uname);
	}
	}

	/*!
	* \internal
	* \brief Find the end of a resource's name, excluding any clone suffix
	*
	* \param[in] id Resource ID to check
	*
	* \return Pointer to last character of resource's base name
	*/
	const char *
	pe_base_name_end(const char *id)
	{
	if (!pcmk__str_empty(id)) {
	const char *end = id + strlen(id) - 1;

	for (const char *s = end; s > id; --s) {
	switch (*s) {
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	return (s == end)? s : (s - 1);
	default:
	return end;
	}
	}
	return end;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get a resource name excluding any clone suffix
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_strip(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	char *basename = NULL;

	CRM_ASSERT(end);
	basename = strndup(last_rsc_id, end - last_rsc_id + 1);
	CRM_ASSERT(basename);
	return basename;
	}

	/*!
	* \internal
	* \brief Get the name of the first instance of a cloned resource
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name plus :0
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_zero(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	size_t base_name_len = end - last_rsc_id + 1;
	char *zero = NULL;

	CRM_ASSERT(end);
	zero = calloc(base_name_len + 3, sizeof(char));
	CRM_ASSERT(zero);
	memcpy(zero, last_rsc_id, base_name_len);
	zero[base_name_len] = ':';
	zero[base_name_len + 1] = '0';
	return zero;
	}

	static pe_resource_t *
	create_fake_resource(const char rsc_id, xmlNode rsc_entry, pe_working_set_t * data_set)
	{
	pe_resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (!common_unpack(xml_rsc, &rsc, NULL, data_set)) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	pe_node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	node = pe_find_node(data_set->nodes, rsc_id);
	if (node == NULL) {
	node = pe_create_node(rsc_id, rsc_id, "remote", NULL, data_set);
	}
	link_rsc2remotenode(data_set, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
	/* This orphaned rsc needs to be mapped to a container. */
	crm_trace("Detected orphaned container filler %s", rsc_id);
	pe__set_resource_flags(rsc, pe_rsc_orphan_container_filler);
	}
	pe__set_resource_flags(rsc, pe_rsc_orphan);
	data_set->resources = g_list_append(data_set->resources, rsc);
	return rsc;
	}

	/*!
	* \internal
	* \brief Create orphan instance for anonymous clone resource history
	*/
	static pe_resource_t *
	create_anonymous_orphan(pe_resource_t parent, const char rsc_id,
	pe_node_t node, pe_working_set_t data_set)
	{
	pe_resource_t *top = pe__create_clone_child(parent, data_set);

	// find_rsc() because we might be a cloned group
	pe_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL, pe_find_clone);

	pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s",
	top->id, parent->id, rsc_id, node->details->uname);
	return orphan;
	}

	/*!
	* \internal
	* \brief Check a node for an instance of an anonymous clone
	*
	* Return a child instance of the specified anonymous clone, in order of
	* preference: (1) the instance running on the specified node, if any;
	* (2) an inactive instance (i.e. within the total of clone-max instances);
	* (3) a newly created orphan (i.e. clone-max instances are already active).
	*
	* \param[in] data_set Cluster information
	* \param[in] node Node on which to check for instance
	* \param[in] parent Clone to check
	* \param[in] rsc_id Name of cloned resource in history (without instance)
	*/
	static pe_resource_t *
	find_anonymous_clone(pe_working_set_t * data_set, pe_node_t * node, pe_resource_t * parent,
	const char *rsc_id)
	{
	GList *rIter = NULL;
	pe_resource_t *rsc = NULL;
	pe_resource_t *inactive_instance = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(pe_rsc_is_clone(parent));
	CRM_ASSERT(!pcmk_is_set(parent->flags, pe_rsc_unique));

	// Check for active (or partially active, for cloned groups) instance
	pe_rsc_trace(parent, "Looking for %s on %s in %s", rsc_id, node->details->uname, parent->id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GList *locations = NULL;
	pe_resource_t *child = rIter->data;

	/* Check whether this instance is already known to be active or pending
	* anywhere, at this stage of unpacking. Because this function is called
	* for a resource before the resource's individual operation history
	* entries are unpacked, locations will generally not contain the
	* desired node.
	*
	* However, there are three exceptions:
	* (1) when child is a cloned group and we have already unpacked the
	* history of another member of the group on the same node;
	* (2) when we've already unpacked the history of another numbered
	* instance on the same node (which can happen if globally-unique
	* was flipped from true to false); and
	* (3) when we re-run calculations on the same data set as part of a
	* simulation.
	*/
	child->fns->location(child, &locations, 2);
	if (locations) {
	/* We should never associate the same numbered anonymous clone
	* instance with multiple nodes, and clone instances can't migrate,
	* so there must be only one location, regardless of history.
	*/
	CRM_LOG_ASSERT(locations->next == NULL);

	if (((pe_node_t *)locations->data)->details == node->details) {
	/* This child instance is active on the requested node, so check
	* for a corresponding configured resource. We use find_rsc()
	* instead of child because child may be a cloned group, and we
	* need the particular member corresponding to rsc_id.
	*
	* If the history entry is orphaned, rsc will be NULL.
	*/
	rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
	if (rsc) {
	/* If there are multiple instance history entries for an
	* anonymous clone in a single node's history (which can
	* happen if globally-unique is switched from true to
	* false), we want to consider the instances beyond the
	* first as orphans, even if there are inactive instance
	* numbers available.
	*/
	if (rsc->running_on) {
	crm_notice("Active (now-)anonymous clone %s has "
	"multiple (orphan) instance histories on %s",
	parent->id, node->details->uname);
	skip_inactive = TRUE;
	rsc = NULL;
	} else {
	pe_rsc_trace(parent, "Resource %s, active", rsc->id);
	}
	}
	}
	g_list_free(locations);

	} else {
	pe_rsc_trace(parent, "Resource %s, skip inactive", child->id);
	if (!skip_inactive && !inactive_instance
	&& !pcmk_is_set(child->flags, pe_rsc_block)) {
	// Remember one inactive instance in case we don't find active
	inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
	pe_find_clone);

	/* ... but don't use it if it was already associated with a
	* pending action on another node
	*/
	if (inactive_instance && inactive_instance->pending_node
	&& (inactive_instance->pending_node->details != node->details)) {
	inactive_instance = NULL;
	}
	}
	}
	}

	if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
	pe_rsc_trace(parent, "Resource %s, empty slot", inactive_instance->id);
	rsc = inactive_instance;
	}

	/* If the resource has "requires" set to "quorum" or "nothing", and we don't
	* have a clone instance for every node, we don't want to consume a valid
	* instance number for unclean nodes. Such instances may appear to be active
	* according to the history, but should be considered inactive, so we can
	* start an instance elsewhere. Treat such instances as orphans.
	*
	* An exception is instances running on guest nodes -- since guest node
	* "fencing" is actually just a resource stop, requires shouldn't apply.
	*
	* @TODO Ideally, we'd use an inactive instance number if it is not needed
	* for any clean instances. However, we don't know that at this point.
	*/
	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)
	&& (!node->details->online \|\| node->details->unclean)
	&& !pe__is_guest_node(node)
	&& !pe__is_universal_clone(parent, data_set)) {

	rsc = NULL;
	}

	if (rsc == NULL) {
	rsc = create_anonymous_orphan(parent, rsc_id, node, data_set);
	pe_rsc_trace(parent, "Resource %s, orphan", rsc->id);
	}
	return rsc;
	}

	static pe_resource_t *
	unpack_find_resource(pe_working_set_t * data_set, pe_node_t * node, const char *rsc_id,
	xmlNode * rsc_entry)
	{
	pe_resource_t *rsc = NULL;
	pe_resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(data_set->resources, rsc_id);

	if (rsc == NULL) {
	/* If we didn't find the resource by its name in the operation history,
	* check it again as a clone instance. Even when clone-max=0, we create
	* a single :0 orphan to match against here.
	*/
	char *clone0_id = clone_zero(rsc_id);
	pe_resource_t *clone0 = pe_find_resource(data_set->resources, clone0_id);

	if (clone0 && !pcmk_is_set(clone0->flags, pe_rsc_unique)) {
	rsc = clone0;
	parent = uber_parent(clone0);
	crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
	} else {
	crm_trace("%s is not known as %s either (orphan)",
	rsc_id, clone0_id);
	}
	free(clone0_id);

	} else if (rsc->variant > pe_native) {
	crm_trace("Resource history for %s is orphaned because it is no longer primitive",
	rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (pe_rsc_is_anon_clone(parent)) {

	if (pe_rsc_is_bundled(parent)) {
	rsc = pe__find_bundle_replica(parent->parent, node);
	} else {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(data_set, node, parent, base);
	free(base);
	CRM_ASSERT(rsc != NULL);
	}
	}

	if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_casei)
	&& !pcmk__str_eq(rsc_id, rsc->clone_name, pcmk__str_casei)) {

	pcmk__str_update(&rsc->clone_name, rsc_id);
	pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, node->details->uname, rsc->id,
	(pcmk_is_set(rsc->flags, pe_rsc_orphan)? " (ORPHAN)" : ""));
	}
	return rsc;
	}

	static pe_resource_t *
	process_orphan_resource(xmlNode * rsc_entry, pe_node_t * node, pe_working_set_t * data_set)
	{
	pe_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
	rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
	if (rsc == NULL) {
	return NULL;
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
	pe__clear_resource_flags(rsc, pe_rsc_managed);

	} else {
	CRM_CHECK(rsc != NULL, return NULL);
	pe_rsc_trace(rsc, "Added orphan %s", rsc->id);
	resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__", data_set);
	}
	return rsc;
	}

	static void
	process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
	enum action_fail_response on_fail,
	xmlNode * migrate_op, pe_working_set_t * data_set)
	{
	pe_node_t *tmpnode = NULL;
	char *reason = NULL;
	enum action_fail_response save_on_fail = action_fail_ignore;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail));

	/* process current state */
	if (rsc->role != RSC_ROLE_UNKNOWN) {
	pe_resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
	pe_node_t *n = pe__copy_node(node);

	pe_rsc_trace(rsc, "%s (aka. %s) known on %s", rsc->id, rsc->clone_name,
	n->details->uname);
	g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
	}
	if (pcmk_is_set(iter->flags, pe_rsc_unique)) {
	break;
	}
	iter = iter->parent;
	}
	}

	/* If a managed resource is believed to be running, but node is down ... */
	if (rsc->role > RSC_ROLE_STOPPED
	&& node->details->online == FALSE
	&& node->details->maintenance == FALSE
	&& pcmk_is_set(rsc->flags, pe_rsc_managed)) {

	gboolean should_fence = FALSE;

	/* If this is a guest node, fence it (regardless of whether fencing is
	* enabled, because guest node fencing is done by recovery of the
	* container resource rather than by the fencer). Mark the resource
	* we're processing as failed. When the guest comes back up, its
	* operation history in the CIB will be cleared, freeing the affected
	* resource to run again once we are sure we know its state.
	*/
	if (pe__is_guest_node(node)) {
	pe__set_resource_flags(rsc, pe_rsc_failed\|pe_rsc_stop);
	should_fence = TRUE;

	} else if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& !pcmk_is_set(node->details->remote_rsc->flags, pe_rsc_failed)) {

	/* Setting unseen means that fencing of the remote node will
	* occur only if the connection resource is not going to start
	* somewhere. This allows connection resources on a failed
	* cluster node to move to another node without requiring the
	* remote nodes to be fenced as well.
	*/
	node->details->unseen = TRUE;
	reason = crm_strdup_printf("%s is active there (fencing will be"
	" revoked if remote connection can "
	"be re-established elsewhere)",
	rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
	}
	pe_fence_node(data_set, node, reason, FALSE);
	}
	free(reason);
	}

	/* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
	save_on_fail = on_fail;

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = action_fail_ignore;
	}

	switch (on_fail) {
	case action_fail_ignore:
	/* nothing to do */
	break;

	case action_fail_demote:
	pe__set_resource_flags(rsc, pe_rsc_failed);
	demote_action(rsc, node, FALSE);
	break;

	case action_fail_fence:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	reason = crm_strdup_printf("%s failed there", rsc->id);
	pe_fence_node(data_set, node, reason, FALSE);
	free(reason);
	break;

	case action_fail_standby:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case action_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	pe__set_resource_flags(rsc, pe_rsc_block);
	break;

	case action_fail_migrate:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY, "__action_migration_auto__", data_set);
	break;

	case action_fail_stop:
	pe__set_next_role(rsc, RSC_ROLE_STOPPED, "on-fail=stop");
	break;

	case action_fail_recover:
	if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	pe__set_resource_flags(rsc, pe_rsc_failed\|pe_rsc_stop);
	stop_action(rsc, node, FALSE);
	}
	break;

	case action_fail_restart_container:
	pe__set_resource_flags(rsc, pe_rsc_failed\|pe_rsc_stop);
	if (rsc->container && pe_rsc_is_bundled(rsc)) {
	/* A bundle's remote connection can run on a different node than
	* the bundle's container. We don't necessarily know where the
	* container is running yet, so remember it and add a stop
	* action for it later.
	*/
	data_set->stop_needed = g_list_prepend(data_set->stop_needed,
	rsc->container);
	} else if (rsc->container) {
	stop_action(rsc->container, node, FALSE);
	} else if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	stop_action(rsc, node, FALSE);
	}
	break;

	case action_fail_reset_remote:
	pe__set_resource_flags(rsc, pe_rsc_failed\|pe_rsc_stop);
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	tmpnode = NULL;
	if (rsc->is_remote_node) {
	tmpnode = pe_find_node(data_set->nodes, rsc->id);
	}
	if (tmpnode &&
	pe__is_remote_node(tmpnode) &&
	tmpnode->details->remote_was_fenced == 0) {

	/* The remote connection resource failed in a way that
	* should result in fencing the remote node.
	*/
	pe_fence_node(data_set, tmpnode,
	"remote connection is unrecoverable", FALSE);
	}
	}

	/* require the stop action regardless if fencing is occurring or not. */
	if (rsc->role > RSC_ROLE_STOPPED) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->remote_reconnect_ms) {
	pe__set_next_role(rsc, RSC_ROLE_STOPPED, "remote reset");
	}
	break;
	}

	/* ensure a remote-node connection failure forces an unclean remote-node
	* to be fenced. By setting unseen = FALSE, the remote-node failure will
	* result in a fencing operation regardless if we're going to attempt to
	* reconnect to the remote-node in this transition or not. */
	if (pcmk_is_set(rsc->flags, pe_rsc_failed) && rsc->is_remote_node) {
	tmpnode = pe_find_node(data_set->nodes, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	tmpnode->details->unseen = FALSE;
	}
	}

	if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pcmk__config_warn("Detected active orphan %s running on %s",
	rsc->id, node->details->uname);
	} else {
	pcmk__config_warn("Resource '%s' must be stopped manually on "
	"%s because cluster is configured not to "
	"stop active orphans",
	rsc->id, node->details->uname);
	}
	}

	native_add_running(rsc, node, data_set, (save_on_fail != action_fail_ignore));
	switch (on_fail) {
	case action_fail_ignore:
	break;
	case action_fail_demote:
	case action_fail_block:
	pe__set_resource_flags(rsc, pe_rsc_failed);
	break;
	default:
	pe__set_resource_flags(rsc, pe_rsc_failed\|pe_rsc_stop);
	break;
	}

	} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
	free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	GList *possible_matches = pe__resource_actions(rsc, node, RSC_STOP,
	FALSE);
	GList *gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t stop = (pe_action_t ) gIter->data;

	pe__set_action_flags(stop, pe_action_optional);
	}

	g_list_free(possible_matches);
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(pe_node_t * node, pe_resource_t * rsc,
	int start_index, int stop_index,
	GList sorted_op_list, pe_working_set_t data_set)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GList *gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	guint interval_ms = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);

	counter++;

	if (node->details->online == FALSE) {
	pe_rsc_trace(rsc, "Skipping %s/%s: node is offline", rsc->id, node->details->uname);
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pe_rsc_trace(rsc, "Skipping %s/%s: resource is not active", id, node->details->uname);
	continue;

	} else if (counter < start_index) {
	pe_rsc_trace(rsc, "Skipping %s/%s: old %d", id, node->details->uname, counter);
	continue;
	}

	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	if (interval_ms == 0) {
	pe_rsc_trace(rsc, "Skipping %s/%s: non-recurring", id, node->details->uname);
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
	pe_rsc_trace(rsc, "Skipping %s/%s: status", id, node->details->uname);
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = pcmk__op_key(rsc->id, task, interval_ms);
	pe_rsc_trace(rsc, "Creating %s/%s", key, node->details->uname);
	custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
	}
	}

	void
	calculate_active_ops(GList sorted_op_list, int start_index, int *stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_clone_start = -1;
	const char *task = NULL;
	const char *status = NULL;
	GList *gIter = sorted_op_list;

	*stop_index = -1;
	*start_index = -1;

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if (pcmk__str_eq(task, CRMD_ACTION_STOP, pcmk__str_casei)
	&& pcmk__str_eq(status, "0", pcmk__str_casei)) {
	*stop_index = counter;

	} else if (pcmk__strcase_any_of(task, CRMD_ACTION_START, CRMD_ACTION_MIGRATED, NULL)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index) && pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);

	if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
	implied_monitor_start = counter;
	}
	} else if (pcmk__strcase_any_of(task, CRMD_ACTION_PROMOTE, CRMD_ACTION_DEMOTE, NULL)) {
	implied_clone_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_clone_start != -1) {
	*start_index = implied_clone_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	// If resource history entry has shutdown lock, remember lock node and time
	static void
	unpack_shutdown_lock(xmlNode rsc_entry, pe_resource_t rsc, pe_node_t *node,
	pe_working_set_t *data_set)
	{
	time_t lock_time = 0; // When lock started (i.e. node shutdown time)

	if ((crm_element_value_epoch(rsc_entry, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	&lock_time) == pcmk_ok) && (lock_time != 0)) {

	if ((data_set->shutdown_lock > 0)
	&& (get_effective_time(data_set)
	> (lock_time + data_set->shutdown_lock))) {
	pe_rsc_info(rsc, "Shutdown lock for %s on %s expired",
	rsc->id, node->details->uname);
	pe__clear_resource_history(rsc, node, data_set);
	} else {
	rsc->lock_node = node;
	rsc->lock_time = lock_time;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack one lrm_resource entry from a node's CIB status
	*
	* \param[in] node Node whose status is being unpacked
	* \param[in] rsc_entry lrm_resource XML being unpacked
	* \param[in] data_set Cluster working set
	*
	* \return Resource corresponding to the entry, or NULL if no operation history
	*/
	static pe_resource_t *
	unpack_lrm_resource(pe_node_t node, xmlNode lrm_resource,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;

	const char *task = NULL;
	const char *rsc_id = ID(lrm_resource);

	pe_resource_t *rsc = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	xmlNode *migrate_op = NULL;
	xmlNode *rsc_op = NULL;
	xmlNode *last_failure = NULL;

	enum action_fail_response on_fail = action_fail_ignore;
	enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;

	if (rsc_id == NULL) {
	crm_warn("Ignoring malformed " XML_LRM_TAG_RESOURCE
	" entry without id");
	return NULL;
	}
	crm_trace("Unpacking " XML_LRM_TAG_RESOURCE " for %s on %s",
	rsc_id, node->details->uname);

	// Build a list of individual lrm_rsc_op entries, so we can sort them
	for (rsc_op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {

	op_list = g_list_prepend(op_list, rsc_op);
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	}
	}

	/* find the resource */
	rsc = unpack_find_resource(data_set, node, rsc_id, lrm_resource);
	if (rsc == NULL) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	} else {
	rsc = process_orphan_resource(lrm_resource, node, data_set);
	}
	}
	CRM_ASSERT(rsc != NULL);

	// Check whether the resource is "shutdown-locked" to this node
	if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
	unpack_shutdown_lock(lrm_resource, rsc, node, data_set);
	}

	/* process operations */
	saved_role = rsc->role;
	rsc->role = RSC_ROLE_UNKNOWN;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	if (pcmk__str_eq(task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
	migrate_op = rsc_op;
	}

	unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail, data_set);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail, migrate_op, data_set);

	if (get_target_role(rsc, &req_role)) {
	if (rsc->next_role == RSC_ROLE_UNKNOWN \|\| req_role < rsc->next_role) {
	pe__set_next_role(rsc, req_role, XML_RSC_ATTR_TARGET_ROLE);

	} else if (req_role > rsc->next_role) {
	pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role), role2text(req_role));
	}
	}

	if (saved_role > rsc->role) {
	rsc->role = saved_role;
	}

	return rsc;
	}

	static void
	handle_orphaned_container_fillers(xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
	{
	xmlNode *rsc_entry = NULL;
	for (rsc_entry = pcmk__xe_first_child(lrm_rsc_list); rsc_entry != NULL;
	rsc_entry = pcmk__xe_next(rsc_entry)) {

	pe_resource_t *rsc;
	pe_resource_t *container;
	const char *rsc_id;
	const char *container_id;

	if (!pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_casei)) {
	continue;
	}

	container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	if (container_id == NULL \|\| rsc_id == NULL) {
	continue;
	}

	container = pe_find_resource(data_set->resources, container_id);
	if (container == NULL) {
	continue;
	}

	rsc = pe_find_resource(data_set->resources, rsc_id);
	if (rsc == NULL \|\|
	!pcmk_is_set(rsc->flags, pe_rsc_orphan_container_filler) \|\|
	rsc->container != NULL) {
	continue;
	}

	pe_rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
	rsc->id, container_id);
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	}
	}

	/*!
	* \internal
	* \brief Unpack one node's lrm status section
	*
	* \param[in] node Node whose status is being unpacked
	* \param[in] xml CIB node state XML
	* \param[in] data_set Cluster working set
	*/
	static void
	unpack_node_lrm(pe_node_t node, xmlNode xml, pe_working_set_t *data_set)
	{
	bool found_orphaned_container_filler = false;

	// Drill down to lrm_resources section
	xml = find_xml_node(xml, XML_CIB_TAG_LRM, FALSE);
	if (xml == NULL) {
	return;
	}
	xml = find_xml_node(xml, XML_LRM_TAG_RESOURCES, FALSE);
	if (xml == NULL) {
	return;
	}

	// Unpack each lrm_resource entry
	for (xmlNode *rsc_entry = first_named_child(xml, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	pe_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, data_set);

	if ((rsc != NULL)
	&& pcmk_is_set(rsc->flags, pe_rsc_orphan_container_filler)) {
	found_orphaned_container_filler = true;
	}
	}

	/* Now that all resource state has been unpacked for this node, map any
	* orphaned container fillers to their container resource.
	*/
	if (found_orphaned_container_filler) {
	handle_orphaned_container_fillers(xml, data_set);
	}
	}

	static void
	set_active(pe_resource_t * rsc)
	{
	pe_resource_t *top = uber_parent(rsc);

	if (top && pcmk_is_set(top->flags, pe_rsc_promotable)) {
	rsc->role = RSC_ROLE_UNPROMOTED;
	} else {
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	pe_node_t *node = value;
	int *score = user_data;

	node->weight = *score;
	}

	#define STATUS_PATH_MAX 1024
	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	bool success_only, pe_working_set_t *data_set)
	{
	int offset = 0;
	char xpath[STATUS_PATH_MAX];
	xmlNode *xml = NULL;

	offset += snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//node_state[@uname='%s']", node);
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//" XML_LRM_TAG_RESOURCE "[@id='%s']",
	resource);

	/* Need to check against transition_magic too? */
	if (source && pcmk__str_eq(op, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset,
	"/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_target='%s']", op,
	source);
	} else if (source && pcmk__str_eq(op, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset,
	"/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_source='%s']", op,
	source);
	} else {
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset,
	"/" XML_LRM_TAG_RSC_OP "[@operation='%s']", op);
	}

	CRM_LOG_ASSERT(offset > 0);
	xml = get_xpath_object(xpath, data_set->input, LOG_DEBUG);

	if (xml && success_only) {
	int rc = PCMK_OCF_UNKNOWN_ERROR;
	int status = PCMK_EXEC_ERROR;

	crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
	crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
	if ((rc != PCMK_OCF_OK) \|\| (status != PCMK_EXEC_DONE)) {
	return NULL;
	}
	}
	return xml;
	}

	static int
	pe__call_id(xmlNode *op_xml)
	{
	int id = 0;

	if (op_xml) {
	crm_element_value_int(op_xml, XML_LRM_ATTR_CALLID, &id);
	}
	return id;
	}

	/*!
	* \brief Check whether a stop happened on the same node after some event
	*
	* \param[in] rsc Resource being checked
	* \param[in] node Node being checked
	* \param[in] xml_op Event that stop is being compared to
	* \param[in] data_set Cluster working set
	*
	* \return TRUE if stop happened after event, FALSE otherwise
	*
	* \note This is really unnecessary, but kept as a safety mechanism. We
	* currently don't save more than one successful event in history, so this
	* only matters when processing really old CIB files that we don't
	* technically support anymore, or as preparation for logging an extended
	* history in the future.
	*/
	static bool
	stop_happened_after(pe_resource_t rsc, pe_node_t node, xmlNode *xml_op,
	pe_working_set_t *data_set)
	{
	xmlNode *stop_op = find_lrm_op(rsc->id, CRMD_ACTION_STOP,
	node->details->uname, NULL, TRUE, data_set);

	return (stop_op && (pe__call_id(stop_op) > pe__call_id(xml_op)));
	}

	static void
	unpack_migrate_to_success(pe_resource_t rsc, pe_node_t node, xmlNode *xml_op,
	pe_working_set_t *data_set)
	{
	/* A successful migration sequence is:
	* migrate_to on source node
	* migrate_from on target node
	* stop on source node
	*
	* If a migrate_to is followed by a stop, the entire migration (successful
	* or failed) is complete, and we don't care what happened on the target.
	*
	* If no migrate_from has happened, the migration is considered to be
	* "partial". If the migrate_from failed, make sure the resource gets
	* stopped on both source and target (if up).
	*
	* If the migrate_to and migrate_from both succeeded (which also implies the
	* resource is no longer running on the source), but there is no stop, the
	* migration is considered to be "dangling". Schedule a stop on the source
	* in this case.
	*/
	int from_rc = 0;
	int from_status = 0;
	pe_node_t *target_node = NULL;
	pe_node_t *source_node = NULL;
	xmlNode *migrate_from = NULL;
	const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	// Sanity check
	CRM_CHECK(source && target && !strcmp(source, node->details->uname), return);

	if (stop_happened_after(rsc, node, xml_op, data_set)) {
	return;
	}

	// Clones are not allowed to migrate, so role can't be promoted
	rsc->role = RSC_ROLE_STARTED;

	target_node = pe_find_node(data_set->nodes, target);
	source_node = pe_find_node(data_set->nodes, source);

	// Check whether there was a migrate_from action on the target
	migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, target,
	source, FALSE, data_set);
	if (migrate_from) {
	crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
	crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS, &from_status);
	pe_rsc_trace(rsc, "%s op on %s exited with status=%d, rc=%d",
	ID(migrate_from), target, from_status, from_rc);
	}

	if (migrate_from && from_rc == PCMK_OCF_OK
	&& (from_status == PCMK_EXEC_DONE)) {
	/* The migrate_to and migrate_from both succeeded, so mark the migration
	* as "dangling". This will be used to schedule a stop action on the
	* source without affecting the target.
	*/
	pe_rsc_trace(rsc, "Detected dangling migration op: %s on %s", ID(xml_op),
	source);
	rsc->role = RSC_ROLE_STOPPED;
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);

	} else if (migrate_from && (from_status != PCMK_EXEC_PENDING)) { // Failed
	if (target_node && target_node->details->online) {
	pe_rsc_trace(rsc, "Marking active on %s %p %d", target, target_node,
	target_node->details->online);
	native_add_running(rsc, target_node, data_set, TRUE);
	}

	} else { // Pending, or complete but erased
	if (target_node && target_node->details->online) {
	pe_rsc_trace(rsc, "Marking active on %s %p %d", target, target_node,
	target_node->details->online);

	native_add_running(rsc, target_node, data_set, FALSE);
	if (source_node && source_node->details->online) {
	/* This is a partial migration: the migrate_to completed
	* successfully on the source, but the migrate_from has not
	* completed. Remember the source and target; if the newly
	* chosen target remains the same when we schedule actions
	* later, we may continue with the migration.
	*/
	rsc->partial_migration_target = target_node;
	rsc->partial_migration_source = source_node;
	}
	} else {
	/* Consider it failed here - forces a restart, prevents migration */
	pe__set_resource_flags(rsc, pe_rsc_failed\|pe_rsc_stop);
	pe__clear_resource_flags(rsc, pe_rsc_allow_migrate);
	}
	}
	}

	// Is there an action_name in node_name's rsc history newer than call_id?
	static bool
	newer_op(pe_resource_t rsc, const char action_name, const char *node_name,
	int call_id, pe_working_set_t *data_set)
	{
	xmlNode *action = find_lrm_op(rsc->id, action_name, node_name, NULL, TRUE,
	data_set);

	return pe__call_id(action) > call_id;
	}

	static void
	unpack_migrate_to_failure(pe_resource_t rsc, pe_node_t node, xmlNode *xml_op,
	pe_working_set_t *data_set)
	{
	int target_stop_id = 0;
	int target_migrate_from_id = 0;
	xmlNode *target_stop = NULL;
	xmlNode *target_migrate_from = NULL;
	const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	// Sanity check
	CRM_CHECK(source && target && !strcmp(source, node->details->uname), return);

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	rsc->role = RSC_ROLE_STARTED;

	// Check for stop on the target
	target_stop = find_lrm_op(rsc->id, CRMD_ACTION_STOP, target, NULL,
	TRUE, data_set);
	target_stop_id = pe__call_id(target_stop);

	// Check for migrate_from on the target
	target_migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, target,
	source, TRUE, data_set);
	target_migrate_from_id = pe__call_id(target_migrate_from);

	if ((target_stop == NULL) \|\| (target_stop_id < target_migrate_from_id)) {
	/* There was no stop on the target, or a stop that happened before a
	* migrate_from, so assume the resource is still active on the target
	* (if it is up).
	*/
	pe_node_t *target_node = pe_find_node(data_set->nodes, target);

	pe_rsc_trace(rsc, "stop (%d) + migrate_from (%d)",
	target_stop_id, target_migrate_from_id);
	if (target_node && target_node->details->online) {
	native_add_running(rsc, target_node, data_set, FALSE);
	}

	} else if (target_migrate_from == NULL) {
	/* We know there was a stop on the target, but there may not have been a
	* migrate_from (the stop could have happened before migrate_from was
	* scheduled or attempted).
	*
	* That means this could be a "dangling" migration. But first, check
	* whether there is a newer successful stop, start, or migrate_from on
	* the source node -- it's possible the failed migration was followed by
	* a successful stop, full restart, or migration in the reverse
	* direction, in which case we don't want to force a stop.
	*/
	int source_migrate_to_id = pe__call_id(xml_op);

	if (newer_op(rsc, CRMD_ACTION_MIGRATED, source, source_migrate_to_id,
	data_set)
	\|\| newer_op(rsc, CRMD_ACTION_START, source, source_migrate_to_id,
	data_set)
	\|\| newer_op(rsc, CRMD_ACTION_STOP, source, source_migrate_to_id,
	data_set)) {
	return;
	}

	// Mark node as having dangling migration so we can force a stop later
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
	}
	}

	static void
	unpack_migrate_from_failure(pe_resource_t rsc, pe_node_t node,
	xmlNode xml_op, pe_working_set_t data_set)
	{
	xmlNode *source_stop = NULL;
	xmlNode *source_migrate_to = NULL;
	const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	// Sanity check
	CRM_CHECK(source && target && !strcmp(target, node->details->uname), return);

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	rsc->role = RSC_ROLE_STARTED;

	// Check for a stop on the source
	source_stop = find_lrm_op(rsc->id, CRMD_ACTION_STOP, source, NULL,
	TRUE, data_set);

	// Check for a migrate_to on the source
	source_migrate_to = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATE,
	source, target, TRUE, data_set);

	if ((source_stop == NULL)
	\|\| (pe__call_id(source_stop) < pe__call_id(source_migrate_to))) {
	/* There was no stop on the source, or a stop that happened before
	* migrate_to, so assume the resource is still active on the source (if
	* it is up).
	*/
	pe_node_t *source_node = pe_find_node(data_set->nodes, source);

	if (source_node && source_node->details->online) {
	native_add_running(rsc, source_node, data_set, TRUE);
	}
	}
	}

	static void
	record_failed_op(xmlNode op, const pe_node_t node,
	const pe_resource_t rsc, pe_working_set_t data_set)
	{
	xmlNode *xIter = NULL;
	const char *op_key = crm_element_value(op, XML_LRM_ATTR_TASK_KEY);

	if (node->details->online == FALSE) {
	return;
	}

	for (xIter = data_set->failed->children; xIter; xIter = xIter->next) {
	const char *key = crm_element_value(xIter, XML_LRM_ATTR_TASK_KEY);
	const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);

	if(pcmk__str_eq(op_key, key, pcmk__str_casei) && pcmk__str_eq(uname, node->details->uname, pcmk__str_casei)) {
	crm_trace("Skipping duplicate entry %s on %s", op_key, node->details->uname);
	return;
	}
	}

	crm_trace("Adding entry %s on %s", op_key, node->details->uname);
	crm_xml_add(op, XML_ATTR_UNAME, node->details->uname);
	crm_xml_add(op, XML_LRM_ATTR_RSCID, rsc->id);
	add_node_copy(data_set->failed, op);
	}

	static const char get_op_key(xmlNode xml_op)
	{
	const char *key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
	if(key == NULL) {
	key = ID(xml_op);
	}
	return key;
	}

	static const char *
	last_change_str(xmlNode *xml_op)
	{
	time_t when;
	const char *when_s = NULL;

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&when) == pcmk_ok) {
	when_s = pcmk__epoch2str(&when);
	if (when_s) {
	// Skip day of week to make message shorter
	when_s = strchr(when_s, ' ');
	if (when_s) {
	++when_s;
	}
	}
	}
	return ((when_s && *when_s)? when_s : "unknown time");
	}

	/*!
	* \internal
	* \brief Compare two on-fail values
	*
	* \param[in] first One on-fail value to compare
	* \param[in] second The other on-fail value to compare
	*
	* \return A negative number if second is more severe than first, zero if they
	* are equal, or a positive number if first is more severe than second.
	* \note This is only needed until the action_fail_response values can be
	* renumbered at the next API compatibility break.
	*/
	static int
	cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
	{
	switch (first) {
	case action_fail_demote:
	switch (second) {
	case action_fail_ignore:
	return 1;
	case action_fail_demote:
	return 0;
	default:
	return -1;
	}
	break;

	case action_fail_reset_remote:
	switch (second) {
	case action_fail_ignore:
	case action_fail_demote:
	case action_fail_recover:
	return 1;
	case action_fail_reset_remote:
	return 0;
	default:
	return -1;
	}
	break;

	case action_fail_restart_container:
	switch (second) {
	case action_fail_ignore:
	case action_fail_demote:
	case action_fail_recover:
	case action_fail_reset_remote:
	return 1;
	case action_fail_restart_container:
	return 0;
	default:
	return -1;
	}
	break;

	default:
	break;
	}
	switch (second) {
	case action_fail_demote:
	return (first == action_fail_ignore)? -1 : 1;

	case action_fail_reset_remote:
	switch (first) {
	case action_fail_ignore:
	case action_fail_demote:
	case action_fail_recover:
	return -1;
	default:
	return 1;
	}
	break;

	case action_fail_restart_container:
	switch (first) {
	case action_fail_ignore:
	case action_fail_demote:
	case action_fail_recover:
	case action_fail_reset_remote:
	return -1;
	default:
	return 1;
	}
	break;

	default:
	break;
	}
	return first - second;
	}

	static void
	unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
	enum action_fail_response * on_fail, pe_working_set_t * data_set)
	{
	bool is_probe = false;
	pe_action_t *action = NULL;

	const char *key = get_op_key(xml_op);
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *exit_reason = crm_element_value(xml_op,
	XML_LRM_ATTR_EXIT_REASON);

	CRM_ASSERT(rsc);
	CRM_CHECK(task != NULL, return);

	*last_failure = xml_op;

	is_probe = pcmk_xe_is_probe(xml_op);

	if (exit_reason == NULL) {
	exit_reason = "";
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)
	&& (rc == PCMK_OCF_NOT_INSTALLED)) {
	crm_trace("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " rc=%d id=%s",
	services_ocf_exitcode_str(rc),
	(*exit_reason? ": " : ""), exit_reason,
	(is_probe? "probe" : task), rsc->id, node->details->uname,
	last_change_str(xml_op), rc, ID(xml_op));
	} else {
	crm_warn("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " rc=%d id=%s",
	services_ocf_exitcode_str(rc),
	(*exit_reason? ": " : ""), exit_reason,
	(is_probe? "probe" : task), rsc->id, node->details->uname,
	last_change_str(xml_op), rc, ID(xml_op));

	if (is_probe && (rc != PCMK_OCF_OK)
	&& (rc != PCMK_OCF_NOT_RUNNING)
	&& (rc != PCMK_OCF_RUNNING_PROMOTED)) {

	/* A failed (not just unexpected) probe result could mean the user
	* didn't know resources will be probed even where they can't run.
	*/
	crm_notice("If it is not possible for %s to run on %s, see "
	"the resource-discovery option for location constraints",
	rsc->id, node->details->uname);
	}

	record_failed_op(xml_op, node, rsc, data_set);
	}

	action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
	if (cmp_on_fail(*on_fail, action->on_fail) < 0) {
	pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail),
	fail2text(action->on_fail), action->uuid, key);
	*on_fail = action->on_fail;
	}

	if (!strcmp(task, CRMD_ACTION_STOP)) {
	resource_location(rsc, node, -INFINITY, "__stop_fail__", data_set);

	} else if (!strcmp(task, CRMD_ACTION_MIGRATE)) {
	unpack_migrate_to_failure(rsc, node, xml_op, data_set);

	} else if (!strcmp(task, CRMD_ACTION_MIGRATED)) {
	unpack_migrate_from_failure(rsc, node, xml_op, data_set);

	} else if (!strcmp(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_PROMOTED;

	} else if (!strcmp(task, CRMD_ACTION_DEMOTE)) {
	if (action->on_fail == action_fail_block) {
	rsc->role = RSC_ROLE_PROMOTED;
	pe__set_next_role(rsc, RSC_ROLE_STOPPED,
	"demote with on-fail=block");

	} else if(rc == PCMK_OCF_NOT_RUNNING) {
	rsc->role = RSC_ROLE_STOPPED;

	} else {
	/* Staying in the promoted role would put the scheduler and
	* controller into a loop. Setting the role to unpromoted is not
	* dangerous because the resource will be stopped as part of
	* recovery, and any promotion will be ordered after that stop.
	*/
	rsc->role = RSC_ROLE_UNPROMOTED;
	}
	}

	if(is_probe && rc == PCMK_OCF_NOT_INSTALLED) {
	/* leave stopped */
	pe_rsc_trace(rsc, "Leaving %s stopped", rsc->id);
	rsc->role = RSC_ROLE_STOPPED;

	} else if (rsc->role < RSC_ROLE_STARTED) {
	pe_rsc_trace(rsc, "Setting %s active", rsc->id);
	set_active(rsc);
	}

	pe_rsc_trace(rsc, "Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
	rsc->id, role2text(rsc->role),
	pcmk__btoa(node->details->unclean),
	fail2text(action->on_fail), role2text(action->fail_role));

	if (action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) {
	pe__set_next_role(rsc, action->fail_role, "failure");
	}

	if (action->fail_role == RSC_ROLE_STOPPED) {
	int score = -INFINITY;

	pe_resource_t *fail_rsc = rsc;

	if (fail_rsc->parent) {
	pe_resource_t *parent = uber_parent(fail_rsc);

	if (pe_rsc_is_clone(parent)
	&& !pcmk_is_set(parent->flags, pe_rsc_unique)) {
	/* For clone resources, if a child fails on an operation
	* with on-fail = stop, all the resources fail. Do this by preventing
	* the parent from coming up again. */
	fail_rsc = parent;
	}
	}
	crm_notice("%s will not be started under current conditions",
	fail_rsc->id);
	/* make sure it doesn't come up again */
	if (fail_rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(fail_rsc->allowed_nodes);
	}
	fail_rsc->allowed_nodes = pe__node_list2table(data_set->nodes);
	g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
	}

	pe_free_action(action);
	}

	/*!
	* \internal
	* \brief Remap informational monitor results and operation status
	*
	* For the monitor results, certain OCF codes are for providing extended information
	* to the user about services that aren't yet failed but not entirely healthy either.
	* These must be treated as the "normal" result by Pacemaker.
	*
	* For operation status, the action result can be used to determine an appropriate
	* status for the purposes of responding to the action. The status provided by the
	* executor is not directly usable since the executor does not know what was expected.
	*
	* \param[in] xml_op Operation history entry XML from CIB status
	* \param[in,out] rsc Resource that operation history entry is for
	* \param[in] node Node where operation was executed
	* \param[in] data_set Current cluster working set
	* \param[in,out] on_fail What should be done about the result
	* \param[in] target_rc Expected return code of operation
	* \param[in,out] rc Actual return code of operation
	* \param[in,out] status Operation execution status
	*
	* \note If the result is remapped and the node is not shutting down or failed,
	* the operation will be recorded in the data set's list of failed operations
	* to highlight it for the user.
	*
	* \note This may update the resource's current and next role.
	*/
	static void
	remap_operation(xmlNode xml_op, pe_resource_t rsc, pe_node_t *node,
	pe_working_set_t data_set, enum action_fail_response on_fail,
	int target_rc, int rc, int status) {
	bool is_probe = false;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *key = get_op_key(xml_op);
	const char *exit_reason = crm_element_value(xml_op,
	XML_LRM_ATTR_EXIT_REASON);

	if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_none)) {
	int remapped_rc = pcmk__effective_rc(*rc);

	if (*rc != remapped_rc) {
	crm_trace("Remapping monitor result %d to %d", *rc, remapped_rc);
	if (!node->details->shutdown \|\| node->details->online) {
	record_failed_op(xml_op, node, rsc, data_set);
	}

	*rc = remapped_rc;
	}
	}

	if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
	*status = PCMK_EXEC_DONE;
	*rc = PCMK_OCF_NOT_RUNNING;
	}

	/* If the executor reported an operation status of anything but done or
	* error, consider that final. But for done or error, we know better whether
	* it should be treated as a failure or not, because we know the expected
	* result.
	*/
	if (status != PCMK_EXEC_DONE && status != PCMK_EXEC_ERROR) {
	return;
	}

	CRM_ASSERT(rsc);
	CRM_CHECK(task != NULL,
	*status = PCMK_EXEC_ERROR; return);

	*status = PCMK_EXEC_DONE;

	if (exit_reason == NULL) {
	exit_reason = "";
	}

	is_probe = pcmk_xe_is_probe(xml_op);

	if (is_probe) {
	task = "probe";
	}

	if (target_rc < 0) {
	/* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
	* Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
	* target_rc in the transition key, which (along with the similar case
	* of a corrupted transition key in the CIB) will be reported to this
	* function as -1. Pacemaker 2.0+ does not support rolling upgrades from
	* those versions or processing of saved CIB files from those versions,
	* so we do not need to care much about this case.
	*/
	*status = PCMK_EXEC_ERROR;
	crm_warn("Expected result not found for %s on %s (corrupt or obsolete CIB?)",
	key, node->details->uname);

	} else if (target_rc != *rc) {
	*status = PCMK_EXEC_ERROR;
	pe_rsc_debug(rsc, "%s on %s: expected %d (%s), got %d (%s%s%s)",
	key, node->details->uname,
	target_rc, services_ocf_exitcode_str(target_rc),
	rc, services_ocf_exitcode_str(rc),
	(*exit_reason? ": " : ""), exit_reason);
	}

	switch (*rc) {
	case PCMK_OCF_OK:
	if (is_probe && (target_rc == PCMK_OCF_NOT_RUNNING)) {
	*status = PCMK_EXEC_DONE;
	pe_rsc_info(rsc, "Probe found %s active on %s at %s",
	rsc->id, node->details->uname,
	last_change_str(xml_op));
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe \|\| (target_rc == *rc)
	\|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)) {

	*status = PCMK_EXEC_DONE;
	rsc->role = RSC_ROLE_STOPPED;

	/* clear any previous failure actions */
	*on_fail = action_fail_ignore;
	pe__set_next_role(rsc, RSC_ROLE_UNKNOWN, "not running");
	}
	break;

	case PCMK_OCF_RUNNING_PROMOTED:
	if (is_probe && (*rc != target_rc)) {
	*status = PCMK_EXEC_DONE;
	pe_rsc_info(rsc,
	"Probe found %s active and promoted on %s at %s",
	rsc->id, node->details->uname,
	last_change_str(xml_op));
	}
	rsc->role = RSC_ROLE_PROMOTED;
	break;

	case PCMK_OCF_DEGRADED_PROMOTED:
	case PCMK_OCF_FAILED_PROMOTED:
	rsc->role = RSC_ROLE_PROMOTED;
	*status = PCMK_EXEC_ERROR;
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	*status = PCMK_EXEC_ERROR_FATAL;
	break;

	case PCMK_OCF_UNIMPLEMENT_FEATURE: {
	guint interval_ms = 0;
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	if (interval_ms > 0) {
	*status = PCMK_EXEC_NOT_SUPPORTED;
	break;
	}
	// fall through
	}

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	if (!pe_can_fence(data_set, node)
	&& !strcmp(task, CRMD_ACTION_STOP)) {
	/* If a stop fails and we can't fence, there's nothing else we can do */
	pe_proc_err("No further recovery can be attempted for %s "
	"because %s on %s failed (%s%s%s) at %s "
	CRM_XS " rc=%d id=%s", rsc->id, task,
	node->details->uname, services_ocf_exitcode_str(*rc),
	(*exit_reason? ": " : ""), exit_reason,
	last_change_str(xml_op), *rc, ID(xml_op));
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	pe__set_resource_flags(rsc, pe_rsc_block);
	}
	*status = PCMK_EXEC_ERROR_HARD;
	break;

	default:
	if (*status == PCMK_EXEC_DONE) {
	crm_info("Treating unknown exit status %d from %s of %s "
	"on %s at %s as failure",
	*rc, task, rsc->id, node->details->uname,
	last_change_str(xml_op));
	*status = PCMK_EXEC_ERROR;
	}
	break;
	}

	pe_rsc_trace(rsc, "Remapped %s status to %d", key, *status);
	}

	// return TRUE if start or monitor last failure but parameters changed
	static bool
	should_clear_for_param_change(xmlNode xml_op, const char task,
	pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set)
	{
	if (!strcmp(task, "start") \|\| !strcmp(task, "monitor")) {

	if (pe__bundle_needs_remote_name(rsc, data_set)) {
	/* We haven't allocated resources yet, so we can't reliably
	* substitute addr parameters for the REMOTE_CONTAINER_HACK.
	* When that's needed, defer the check until later.
	*/
	pe__add_param_check(xml_op, rsc, node, pe_check_last_failure,
	data_set);

	} else {
	op_digest_cache_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node, data_set);
	switch (digest_data->rc) {
	case RSC_DIGEST_UNKNOWN:
	crm_trace("Resource %s history entry %s on %s"
	" has no digest to compare",
	rsc->id, get_op_key(xml_op), node->details->id);
	break;
	case RSC_DIGEST_MATCH:
	break;
	default:
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	// Order action after fencing of remote node, given connection rsc
	static void
	order_after_remote_fencing(pe_action_t action, pe_resource_t remote_conn,
	pe_working_set_t *data_set)
	{
	pe_node_t *remote_node = pe_find_node(data_set->nodes, remote_conn->id);

	if (remote_node) {
	pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
	FALSE, data_set);

	order_actions(fence, action, pe_order_implies_then);
	}
	}

	static bool
	should_ignore_failure_timeout(pe_resource_t rsc, xmlNode xml_op,
	const char *task, guint interval_ms,
	bool is_last_failure, pe_working_set_t *data_set)
	{
	/* Clearing failures of recurring monitors has special concerns. The
	* executor reports only changes in the monitor result, so if the
	* monitor is still active and still getting the same failure result,
	* that will go undetected after the failure is cleared.
	*
	* Also, the operation history will have the time when the recurring
	* monitor result changed to the given code, not the time when the
	* result last happened.
	*
	* @TODO We probably should clear such failures only when the failure
	* timeout has passed since the last occurrence of the failed result.
	* However we don't record that information. We could maybe approximate
	* that by clearing only if there is a more recent successful monitor or
	* stop result, but we don't even have that information at this point
	* since we are still unpacking the resource's operation history.
	*
	* This is especially important for remote connection resources with a
	* reconnect interval, so in that case, we skip clearing failures
	* if the remote node hasn't been fenced.
	*/
	if (rsc->remote_reconnect_ms
	&& pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)
	&& (interval_ms != 0) && pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {

	pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);

	if (remote_node && !remote_node->details->remote_was_fenced) {
	if (is_last_failure) {
	crm_info("Waiting to clear monitor failure for remote node %s"
	" until fencing has occurred", rsc->id);
	}
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Check operation age and schedule failure clearing when appropriate
	*
	* This function has two distinct purposes. The first is to check whether an
	* operation history entry is expired (i.e. the resource has a failure timeout,
	* the entry is older than the timeout, and the resource either has no fail
	* count or its fail count is entirely older than the timeout). The second is to
	* schedule fail count clearing when appropriate (i.e. the operation is expired
	* and either the resource has an expired fail count or the operation is a
	* last_failure for a remote connection resource with a reconnect interval,
	* or the operation is a last_failure for a start or monitor operation and the
	* resource's parameters have changed since the operation).
	*
	* \param[in] rsc Resource that operation happened to
	* \param[in] node Node that operation happened on
	* \param[in] rc Actual result of operation
	* \param[in] xml_op Operation history entry XML
	* \param[in] data_set Current working set
	*
	* \return TRUE if operation history entry is expired, FALSE otherwise
	*/
	static bool
	check_operation_expiry(pe_resource_t rsc, pe_node_t node, int rc,
	xmlNode xml_op, pe_working_set_t data_set)
	{
	bool expired = FALSE;
	bool is_last_failure = pcmk__ends_with(ID(xml_op), "_last_failure_0");
	time_t last_run = 0;
	guint interval_ms = 0;
	int unexpired_fail_count = 0;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *clear_reason = NULL;

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	if ((rsc->failure_timeout > 0)
	&& (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_run) == 0)) {

	// Resource has a failure-timeout, and history entry has a timestamp

	time_t now = get_effective_time(data_set);
	time_t last_failure = 0;

	// Is this particular operation history older than the failure timeout?
	if ((now >= (last_run + rsc->failure_timeout))
	&& !should_ignore_failure_timeout(rsc, xml_op, task, interval_ms,
	is_last_failure, data_set)) {
	expired = TRUE;
	}

	// Does the resource as a whole have an unexpired fail count?
	unexpired_fail_count = pe_get_failcount(node, rsc, &last_failure,
	pe_fc_effective, xml_op,
	data_set);

	// Update scheduler recheck time according to last failure
	crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
	" last-failure@%lld",
	ID(xml_op), (long long) last_run, (expired? "" : "not "),
	(long long) now, unexpired_fail_count, rsc->failure_timeout,
	(long long) last_failure);
	last_failure += rsc->failure_timeout + 1;
	if (unexpired_fail_count && (now < last_failure)) {
	pe__update_recheck_time(last_failure, data_set);
	}
	}

	if (expired) {
	if (pe_get_failcount(node, rsc, NULL, pe_fc_default, xml_op, data_set)) {

	// There is a fail count ignoring timeout

	if (unexpired_fail_count == 0) {
	// There is no fail count considering timeout
	clear_reason = "it expired";

	} else {
	/* This operation is old, but there is an unexpired fail count.
	* In a properly functioning cluster, this should only be
	* possible if this operation is not a failure (otherwise the
	* fail count should be expired too), so this is really just a
	* failsafe.
	*/
	expired = FALSE;
	}

	} else if (is_last_failure && rsc->remote_reconnect_ms) {
	/* Clear any expired last failure when reconnect interval is set,
	* even if there is no fail count.
	*/
	clear_reason = "reconnect interval is set";
	}
	}

	if (!expired && is_last_failure
	&& should_clear_for_param_change(xml_op, task, rsc, node, data_set)) {
	clear_reason = "resource parameters have changed";
	}

	if (clear_reason != NULL) {
	// Schedule clearing of the fail count
	pe_action_t *clear_op = pe__clear_failcount(rsc, node, clear_reason,
	data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)
	&& rsc->remote_reconnect_ms) {
	/* If we're clearing a remote connection due to a reconnect
	* interval, we want to wait until any scheduled fencing
	* completes.
	*
	* We could limit this to remote_node->details->unclean, but at
	* this point, that's always true (it won't be reliable until
	* after unpack_node_history() is done).
	*/
	crm_info("Clearing %s failure will wait until any scheduled "
	"fencing of %s completes", task, rsc->id);
	order_after_remote_fencing(clear_op, rsc, data_set);
	}
	}

	if (expired && (interval_ms == 0) && pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
	switch(rc) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_PROMOTED:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_PROMOTED:
	// Don't expire probes that return these values
	expired = FALSE;
	break;
	}
	}

	return expired;
	}

	int pe__target_rc_from_xml(xmlNode *xml_op)
	{
	int target_rc = 0;
	const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);

	if (key == NULL) {
	return -1;
	}
	decode_transition_key(key, NULL, NULL, NULL, &target_rc);
	return target_rc;
	}

	static enum action_fail_response
	get_action_on_fail(pe_resource_t rsc, const char key, const char task, pe_working_set_t data_set)
	{
	enum action_fail_response result = action_fail_recover;
	pe_action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);

	result = action->on_fail;
	pe_free_action(action);

	return result;
	}

	static void
	update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, const char * task, int rc,
	xmlNode * last_failure, enum action_fail_response * on_fail, pe_working_set_t * data_set)
	{
	gboolean clear_past_failure = FALSE;

	CRM_ASSERT(rsc);
	CRM_ASSERT(xml_op);

	if (rc == PCMK_OCF_NOT_INSTALLED \|\| (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op))) {
	rsc->role = RSC_ROLE_STOPPED;

	} else if (rc == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = TRUE;

	} else if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
	if (last_failure) {
	const char *op_key = get_op_key(xml_op);
	const char *last_failure_key = get_op_key(last_failure);

	if (pcmk__str_eq(op_key, last_failure_key, pcmk__str_casei)) {
	clear_past_failure = TRUE;
	}
	}

	if (rsc->role < RSC_ROLE_STARTED) {
	set_active(rsc);
	}

	} else if (pcmk__str_eq(task, CRMD_ACTION_START, pcmk__str_casei)) {
	rsc->role = RSC_ROLE_STARTED;
	clear_past_failure = TRUE;

	} else if (pcmk__str_eq(task, CRMD_ACTION_STOP, pcmk__str_casei)) {
	rsc->role = RSC_ROLE_STOPPED;
	clear_past_failure = TRUE;

	} else if (pcmk__str_eq(task, CRMD_ACTION_PROMOTE, pcmk__str_casei)) {
	rsc->role = RSC_ROLE_PROMOTED;
	clear_past_failure = TRUE;

	} else if (pcmk__str_eq(task, CRMD_ACTION_DEMOTE, pcmk__str_casei)) {

	if (*on_fail == action_fail_demote) {
	// Demote clears an error only if on-fail=demote
	clear_past_failure = TRUE;
	}
	rsc->role = RSC_ROLE_UNPROMOTED;

	} else if (pcmk__str_eq(task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
	rsc->role = RSC_ROLE_STARTED;
	clear_past_failure = TRUE;

	} else if (pcmk__str_eq(task, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
	unpack_migrate_to_success(rsc, node, xml_op, data_set);

	} else if (rsc->role < RSC_ROLE_STARTED) {
	pe_rsc_trace(rsc, "%s active on %s", rsc->id, node->details->uname);
	set_active(rsc);
	}

	/* clear any previous failure actions */
	if (clear_past_failure) {
	switch (*on_fail) {
	case action_fail_stop:
	case action_fail_fence:
	case action_fail_migrate:
	case action_fail_standby:
	pe_rsc_trace(rsc, "%s.%s is not cleared by a completed stop",
	rsc->id, fail2text(*on_fail));
	break;

	case action_fail_block:
	case action_fail_ignore:
	case action_fail_demote:
	case action_fail_recover:
	case action_fail_restart_container:
	*on_fail = action_fail_ignore;
	pe__set_next_role(rsc, RSC_ROLE_UNKNOWN, "clear past failures");
	break;
	case action_fail_reset_remote:
	if (rsc->remote_reconnect_ms == 0) {
	/* With no reconnect interval, the connection is allowed to
	* start again after the remote node is fenced and
	* completely stopped. (With a reconnect interval, we wait
	* for the failure to be cleared entirely before attempting
	* to reconnect.)
	*/
	*on_fail = action_fail_ignore;
	pe__set_next_role(rsc, RSC_ROLE_UNKNOWN,
	"clear past failures and reset remote");
	}
	break;
	}
	}
	}

	static void
	unpack_rsc_op(pe_resource_t rsc, pe_node_t node, xmlNode *xml_op,
	xmlNode *last_failure, enum action_fail_response on_fail,
	pe_working_set_t *data_set)
	{
	int rc = 0;
	int old_rc = 0;
	int task_id = 0;
	int target_rc = 0;
	int old_target_rc = 0;
	int status = PCMK_EXEC_UNKNOWN;
	guint interval_ms = 0;
	const char *task = NULL;
	const char *task_key = NULL;
	const char *exit_reason = NULL;
	bool expired = false;
	pe_resource_t *parent = rsc;
	enum action_fail_response failure_strategy = action_fail_recover;
	bool maskable_probe_failure = false;

	CRM_CHECK(rsc && node && xml_op, return);

	target_rc = pe__target_rc_from_xml(xml_op);
	task_key = get_op_key(xml_op);
	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
	if (exit_reason == NULL) {
	exit_reason = "";
	}

	crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
	crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	CRM_CHECK(task != NULL, return);
	CRM_CHECK((status >= PCMK_EXEC_PENDING) && (status <= PCMK_EXEC_MAX),
	return);

	if (!strcmp(task, CRMD_ACTION_NOTIFY) \|\|
	!strcmp(task, CRMD_ACTION_METADATA)) {
	/* safe to ignore these */
	return;
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	parent = uber_parent(rsc);
	}

	pe_rsc_trace(rsc, "Unpacking task %s/%s (call_id=%d, status=%d, rc=%d) on %s (role=%s)",
	task_key, task, task_id, status, rc, node->details->uname, role2text(rsc->role));

	if (node->details->unclean) {
	pe_rsc_trace(rsc, "Node %s (where %s is running) is unclean."
	" Further action depends on the value of the stop's on-fail attribute",
	node->details->uname, rsc->id);
	}

	/* It should be possible to call remap_operation() first then call
	* check_operation_expiry() only if rc != target_rc, because there should
	* never be a fail count without at least one unexpected result in the
	* resource history. That would be more efficient by avoiding having to call
	* check_operation_expiry() for expected results.
	*
	* However, we do have such configurations in the scheduler regression
	* tests, even if it shouldn't be possible with the current code. It's
	* probably a good idea anyway, but that would require updating the test
	* inputs to something currently possible.
	*/

	if ((status != PCMK_EXEC_NOT_INSTALLED)
	&& check_operation_expiry(rsc, node, rc, xml_op, data_set)) {
	expired = true;
	}

	old_rc = rc;
	old_target_rc = target_rc;

	remap_operation(xml_op, rsc, node, data_set, on_fail, target_rc,
	&rc, &status);

	maskable_probe_failure = !pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op);

	if (expired && maskable_probe_failure && old_rc != old_target_rc) {
	if (rsc->role <= RSC_ROLE_STOPPED) {
	rsc->role = RSC_ROLE_UNKNOWN;
	}

	goto done;

	} else if (expired && (rc != target_rc)) {
	const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);

	if (interval_ms == 0) {
	crm_notice("Ignoring expired %s failure on %s "
	CRM_XS " actual=%d expected=%d magic=%s",
	task_key, node->details->uname, rc, target_rc, magic);
	goto done;

	} else if(node->details->online && node->details->unclean == FALSE) {
	/* Reschedule the recurring monitor. schedule_cancel() won't work at
	* this stage, so as a hacky workaround, forcibly change the restart
	* digest so pcmk__check_action_config() does what we want later.
	*
	* @TODO We should skip this if there is a newer successful monitor.
	* Also, this causes rescheduling only if the history entry
	* has an op-digest (which the expire-non-blocked-failure
	* scheduler regression test doesn't, but that may not be a
	* realistic scenario in production).
	*/
	crm_notice("Rescheduling %s after failure expired on %s "
	CRM_XS " actual=%d expected=%d magic=%s",
	task_key, node->details->uname, rc, target_rc, magic);
	crm_xml_add(xml_op, XML_LRM_ATTR_RESTART_DIGEST, "calculated-failure-timeout");
	goto done;
	}
	}

	if (maskable_probe_failure) {
	crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
	services_ocf_exitcode_str(old_rc), rsc->id, node->details->uname);
	update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure,
	on_fail, data_set);
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);

	record_failed_op(xml_op, node, rsc, data_set);
	resource_location(parent, node, -INFINITY, "masked-probe-failure", data_set);
	goto done;
	}

	switch (status) {
	case PCMK_EXEC_CANCELLED:
	// Should never happen
	pe_err("Resource history contains cancellation '%s' "
	"(%s of %s on %s at %s)",
	ID(xml_op), task, rsc->id, node->details->uname,
	last_change_str(xml_op));
	goto done;

	case PCMK_EXEC_PENDING:
	if (!strcmp(task, CRMD_ACTION_START)) {
	pe__set_resource_flags(rsc, pe_rsc_start_pending);
	set_active(rsc);

	} else if (!strcmp(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_PROMOTED;

	} else if (!strcmp(task, CRMD_ACTION_MIGRATE) && node->details->unclean) {
	/* If a pending migrate_to action is out on a unclean node,
	* we have to force the stop action on the target. */
	const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
	pe_node_t *target = pe_find_node(data_set->nodes, migrate_target);
	if (target) {
	stop_action(rsc, target, FALSE);
	}
	}

	if (rsc->pending_task == NULL) {
	if ((interval_ms != 0) \|\| strcmp(task, CRMD_ACTION_STATUS)) {
	rsc->pending_task = strdup(task);
	rsc->pending_node = node;
	} else {
	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, enable the below and the corresponding part of
	* native.c:native_pending_task().
	*/
	#if 0
	rsc->pending_task = strdup("probe");
	rsc->pending_node = node;
	#endif
	}
	}
	goto done;

	case PCMK_EXEC_DONE:
	pe_rsc_trace(rsc, "%s of %s on %s completed at %s " CRM_XS " id=%s",
	task, rsc->id, node->details->uname,
	last_change_str(xml_op), ID(xml_op));
	update_resource_state(rsc, node, xml_op, task, rc, *last_failure, on_fail, data_set);
	goto done;

	case PCMK_EXEC_NOT_INSTALLED:
	failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
	if (failure_strategy == action_fail_ignore) {
	crm_warn("Cannot ignore failed %s of %s on %s: "
	"Resource agent doesn't exist "
	CRM_XS " status=%d rc=%d id=%s",
	task, rsc->id, node->details->uname, status, rc,
	ID(xml_op));
	/* Also for printing it as "FAILED" by marking it as pe_rsc_failed later */
	*on_fail = action_fail_migrate;
	}
	resource_location(parent, node, -INFINITY, "hard-error", data_set);
	unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
	goto done;

	case PCMK_EXEC_NOT_CONNECTED:
	if (pe__is_guest_or_remote_node(node)
	&& pcmk_is_set(node->details->remote_rsc->flags, pe_rsc_managed)) {
	/* We should never get into a situation where a managed remote
	* connection resource is considered OK but a resource action
	* behind the connection gets a "not connected" status. But as a
	* fail-safe in case a bug or unusual circumstances do lead to
	* that, ensure the remote connection is considered failed.
	*/
	pe__set_resource_flags(node->details->remote_rsc,
	pe_rsc_failed\|pe_rsc_stop);
	}
	break; // Not done, do error handling

	case PCMK_EXEC_ERROR:
	case PCMK_EXEC_ERROR_HARD:
	case PCMK_EXEC_ERROR_FATAL:
	case PCMK_EXEC_TIMEOUT:
	case PCMK_EXEC_NOT_SUPPORTED:
	case PCMK_EXEC_INVALID:
	break; // Not done, do error handling

	case PCMK_EXEC_NO_FENCE_DEVICE:
	case PCMK_EXEC_NO_SECRETS:
	status = PCMK_EXEC_ERROR_HARD;
	break; // Not done, do error handling
	}

	failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
	if ((failure_strategy == action_fail_ignore)
	\|\| (failure_strategy == action_fail_restart_container
	&& !strcmp(task, CRMD_ACTION_STOP))) {

	crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s "
	"succeeded " CRM_XS " rc=%d id=%s",
	task, services_ocf_exitcode_str(rc),
	(*exit_reason? ": " : ""), exit_reason, rsc->id,
	node->details->uname, last_change_str(xml_op), rc,
	ID(xml_op));

	update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure,
	on_fail, data_set);
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	pe__set_resource_flags(rsc, pe_rsc_failure_ignored);

	record_failed_op(xml_op, node, rsc, data_set);

	if ((failure_strategy == action_fail_restart_container)
	&& cmp_on_fail(*on_fail, action_fail_recover) <= 0) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail,
	data_set);

	if (status == PCMK_EXEC_ERROR_HARD) {
	do_crm_log(rc != PCMK_OCF_NOT_INSTALLED?LOG_ERR:LOG_NOTICE,
	"Preventing %s from restarting on %s because "
	"of hard failure (%s%s%s)" CRM_XS " rc=%d id=%s",
	parent->id, node->details->uname,
	services_ocf_exitcode_str(rc),
	(*exit_reason? ": " : ""), exit_reason,
	rc, ID(xml_op));
	resource_location(parent, node, -INFINITY, "hard-error", data_set);

	} else if (status == PCMK_EXEC_ERROR_FATAL) {
	crm_err("Preventing %s from restarting anywhere because "
	"of fatal failure (%s%s%s) " CRM_XS " rc=%d id=%s",
	parent->id, services_ocf_exitcode_str(rc),
	(*exit_reason? ": " : ""), exit_reason,
	rc, ID(xml_op));
	resource_location(parent, NULL, -INFINITY, "fatal-error", data_set);
	}
	}

	done:
	pe_rsc_trace(rsc, "Resource %s after %s: role=%s, next=%s",
	rsc->id, task, role2text(rsc->role),
	role2text(rsc->next_role));
	}

	static void
	add_node_attrs(xmlNode xml_obj, pe_node_t node, bool overwrite,
	pe_working_set_t *data_set)
	{
	const char *cluster_name = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_UNAME), strdup(node->details->uname));

	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
	strdup(node->details->id));
	if (pcmk__str_eq(node->details->id, data_set->dc_uuid, pcmk__str_casei)) {
	data_set->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
	}

	cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
	if (cluster_name) {
	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
	strdup(cluster_name));
	}

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
	node->details->attrs, NULL, overwrite, data_set);

	if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
	const char *site_name = pe_node_attribute_raw(node, "site-name");

	if (site_name) {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(site_name));

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(cluster_name));
	}
	}
	}

	static GList *
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GList *gIter = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP,
	pcmk__str_none)) {
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GList *
	find_operations(const char rsc, const char node, gboolean active_filter,
	pe_working_set_t * data_set)
	{
	GList *output = NULL;
	GList *intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);

	pe_node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	for (node_state = pcmk__xe_first_child(status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {

	if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);

	if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
	continue;
	}

	this_node = pe_find_node(data_set->nodes, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(data_set, this_node);

	} else {
	determine_online_status(node_state, this_node, data_set);
	}

	if (this_node->details->online
	\|\| pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	for (lrm_rsc = pcmk__xe_first_child(tmp); lrm_rsc != NULL;
	lrm_rsc = pcmk__xe_next(lrm_rsc)) {

	if (pcmk__str_eq((const char *)lrm_rsc->name,
	XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);

	if (rsc != NULL && !pcmk__str_eq(rsc_id, rsc, pcmk__str_casei)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}
	diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
	index 5f430abfd0..d5ded2e95c 100644
	--- a/lib/pengine/utils.c
	+++ b/lib/pengine/utils.c
	@@ -1,2615 +1,2615 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/util.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include "pe_status_private.h"

	extern bool pcmk__is_daemon;

	void print_str_str(gpointer key, gpointer value, gpointer user_data);
	gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);
	static void unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
	pe_working_set_t * data_set, guint interval_ms);
	static xmlNode find_rsc_op_entry_helper(pe_resource_t rsc, const char *key,
	gboolean include_disabled);

	#if ENABLE_VERSIONED_ATTRS
	pe_rsc_action_details_t *
	pe_rsc_action_details(pe_action_t *action)
	{
	pe_rsc_action_details_t *details;

	CRM_CHECK(action != NULL, return NULL);

	if (action->action_details == NULL) {
	action->action_details = calloc(1, sizeof(pe_rsc_action_details_t));
	CRM_CHECK(action->action_details != NULL, return NULL);
	}

	details = (pe_rsc_action_details_t *) action->action_details;
	if (details->versioned_parameters == NULL) {
	details->versioned_parameters = create_xml_node(NULL,
	XML_TAG_OP_VER_ATTRS);
	}
	if (details->versioned_meta == NULL) {
	details->versioned_meta = create_xml_node(NULL, XML_TAG_OP_VER_META);
	}
	return details;
	}

	static void
	pe_free_rsc_action_details(pe_action_t *action)
	{
	pe_rsc_action_details_t *details;

	if ((action == NULL) \|\| (action->action_details == NULL)) {
	return;
	}

	details = (pe_rsc_action_details_t *) action->action_details;

	if (details->versioned_parameters) {
	free_xml(details->versioned_parameters);
	}
	if (details->versioned_meta) {
	free_xml(details->versioned_meta);
	}

	action->action_details = NULL;
	}
	#endif

	/*!
	* \internal
	* \brief Check whether we can fence a particular node
	*
	* \param[in] data_set Working set for cluster
	* \param[in] node Name of node to check
	*
	* \return true if node can be fenced, false otherwise
	*/
	bool
	pe_can_fence(pe_working_set_t data_set, pe_node_t node)
	{
	if (pe__is_guest_node(node)) {
	/* Guest nodes are fenced by stopping their container resource. We can
	* do that if the container's host is either online or fenceable.
	*/
	pe_resource_t *rsc = node->details->remote_rsc->container;

	for (GList *n = rsc->running_on; n != NULL; n = n->next) {
	pe_node_t *container_node = n->data;

	if (!container_node->details->online
	&& !pe_can_fence(data_set, container_node)) {
	return false;
	}
	}
	return true;

	} else if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	return false; /* Turned off */

	} else if (!pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {
	return false; /* No devices */

	} else if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	return true;

	} else if (data_set->no_quorum_policy == no_quorum_ignore) {
	return true;

	} else if(node == NULL) {
	return false;

	} else if(node->details->online) {
	crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname);
	return true;
	}

	crm_trace("Cannot fence %s", node->details->uname);
	return false;
	}

	/*!
	* \internal
	* \brief Copy a node object
	*
	* \param[in] this_node Node object to copy
	*
	* \return Newly allocated shallow copy of this_node
	* \note This function asserts on errors and is guaranteed to return non-NULL.
	*/
	pe_node_t *
	pe__copy_node(const pe_node_t *this_node)
	{
	pe_node_t *new_node = NULL;

	CRM_ASSERT(this_node != NULL);

	new_node = calloc(1, sizeof(pe_node_t));
	CRM_ASSERT(new_node != NULL);

	new_node->rsc_discover_mode = this_node->rsc_discover_mode;
	new_node->weight = this_node->weight;
	new_node->fixed = this_node->fixed;
	new_node->details = this_node->details;

	return new_node;
	}

	/* any node in list1 or list2 and not in the other gets a score of -INFINITY */
	void
	node_list_exclude(GHashTable * hash, GList *list, gboolean merge_scores)
	{
	GHashTable *result = hash;
	pe_node_t *other_node = NULL;
	GList *gIter = list;

	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, hash);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {

	other_node = pe_find_node_id(list, node->details->id);
	if (other_node == NULL) {
	node->weight = -INFINITY;
	} else if (merge_scores) {
	- node->weight = pe__add_scores(node->weight, other_node->weight);
	+ node->weight = pcmk__add_scores(node->weight, other_node->weight);
	}
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	other_node = pe_hash_table_lookup(result, node->details->id);

	if (other_node == NULL) {
	pe_node_t *new_node = pe__copy_node(node);

	new_node->weight = -INFINITY;
	g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
	}
	}
	}

	/*!
	* \internal
	* \brief Create a node hash table from a node list
	*
	* \param[in] list Node list
	*
	* \return Hash table equivalent of node list
	*/
	GHashTable *
	pe__node_list2table(GList *list)
	{
	GHashTable *result = NULL;

	result = pcmk__strkey_table(NULL, free);
	for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pe_node_t new_node = pe__copy_node((pe_node_t ) gIter->data);

	g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
	}
	return result;
	}

	gint
	sort_node_uname(gconstpointer a, gconstpointer b)
	{
	return pcmk__numeric_strcasecmp(((const pe_node_t *) a)->details->uname,
	((const pe_node_t *) b)->details->uname);
	}

	/*!
	* \internal
	* \brief Output node weights to stdout
	*
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	*/
	static void
	pe__output_node_weights(pe_resource_t rsc, const char comment,
	GHashTable nodes, pe_working_set_t data_set)
	{
	pcmk__output_t *out = data_set->priv;
	char score[128]; // Stack-allocated since this is called frequently

	// Sort the nodes so the output is consistent for regression tests
	GList *list = g_list_sort(g_hash_table_get_values(nodes), sort_node_uname);

	for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	score2char_stack(node->weight, score, sizeof(score));
	out->message(out, "node-weight", rsc, comment, node->details->uname, score);
	}
	g_list_free(list);
	}

	/*!
	* \internal
	* \brief Log node weights at trace level
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	*/
	static void
	pe__log_node_weights(const char file, const char function, int line,
	pe_resource_t rsc, const char comment, GHashTable *nodes)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	char score[128]; // Stack-allocated since this is called frequently

	// Don't waste time if we're not tracing at this point
	pcmk__log_else(LOG_TRACE, return);

	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	score2char_stack(node->weight, score, sizeof(score));
	if (rsc) {
	qb_log_from_external_source(function, file,
	"%s: %s allocation score on %s: %s",
	LOG_TRACE, line, 0,
	comment, rsc->id,
	node->details->uname, score);
	} else {
	qb_log_from_external_source(function, file, "%s: %s = %s",
	LOG_TRACE, line, 0,
	comment, node->details->uname,
	score);
	}
	}
	}

	/*!
	* \internal
	* \brief Log or output node weights
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] to_log Log if true, otherwise output
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes Use these nodes
	*/
	void
	pe__show_node_weights_as(const char file, const char function, int line,
	bool to_log, pe_resource_t rsc, const char comment,
	GHashTable nodes, pe_working_set_t data_set)
	{
	if (rsc != NULL && pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	// Don't show allocation scores for orphans
	return;
	}
	if (nodes == NULL) {
	// Nothing to show
	return;
	}

	if (to_log) {
	pe__log_node_weights(file, function, line, rsc, comment, nodes);
	} else {
	pe__output_node_weights(rsc, comment, nodes, data_set);
	}

	// If this resource has children, repeat recursively for each
	if (rsc && rsc->children) {
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	pe__show_node_weights_as(file, function, line, to_log, child,
	comment, child->allowed_nodes, data_set);
	}
	}
	}

	gint
	sort_rsc_index(gconstpointer a, gconstpointer b)
	{
	const pe_resource_t resource1 = (const pe_resource_t )a;
	const pe_resource_t resource2 = (const pe_resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->sort_index > resource2->sort_index) {
	return -1;
	}

	if (resource1->sort_index < resource2->sort_index) {
	return 1;
	}

	return 0;
	}

	gint
	sort_rsc_priority(gconstpointer a, gconstpointer b)
	{
	const pe_resource_t resource1 = (const pe_resource_t )a;
	const pe_resource_t resource2 = (const pe_resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->priority > resource2->priority) {
	return -1;
	}

	if (resource1->priority < resource2->priority) {
	return 1;
	}

	return 0;
	}

	static enum pe_quorum_policy
	effective_quorum_policy(pe_resource_t rsc, pe_working_set_t data_set)
	{
	enum pe_quorum_policy policy = data_set->no_quorum_policy;

	if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	policy = no_quorum_ignore;

	} else if (data_set->no_quorum_policy == no_quorum_demote) {
	switch (rsc->role) {
	case RSC_ROLE_PROMOTED:
	case RSC_ROLE_UNPROMOTED:
	if (rsc->next_role > RSC_ROLE_UNPROMOTED) {
	pe__set_next_role(rsc, RSC_ROLE_UNPROMOTED,
	"no-quorum-policy=demote");
	}
	policy = no_quorum_ignore;
	break;
	default:
	policy = no_quorum_stop;
	break;
	}
	}
	return policy;
	}

	static void
	add_singleton(pe_working_set_t data_set, pe_action_t action)
	{
	if (data_set->singletons == NULL) {
	data_set->singletons = pcmk__strkey_table(NULL, NULL);
	}
	g_hash_table_insert(data_set->singletons, action->uuid, action);
	}

	static pe_action_t *
	lookup_singleton(pe_working_set_t data_set, const char action_uuid)
	{
	if (data_set->singletons == NULL) {
	return NULL;
	}
	return g_hash_table_lookup(data_set->singletons, action_uuid);
	}

	/*!
	* \internal
	* \brief Find an existing action that matches arguments
	*
	* \param[in] key Action key to match
	* \param[in] rsc Resource to match (if any)
	* \param[in] node Node to match (if any)
	* \param[in] data_set Cluster working set
	*
	* \return Existing action that matches arguments (or NULL if none)
	*/
	static pe_action_t *
	find_existing_action(const char key, pe_resource_t rsc, pe_node_t *node,
	pe_working_set_t *data_set)
	{
	GList *matches = NULL;
	pe_action_t *action = NULL;

	/* When rsc is NULL, it would be quicker to check data_set->singletons,
	* but checking all data_set->actions takes the node into account.
	*/
	matches = find_actions(((rsc == NULL)? data_set->actions : rsc->actions),
	key, node);
	if (matches == NULL) {
	return NULL;
	}
	CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));

	action = matches->data;
	g_list_free(matches);
	return action;
	}

	/*!
	* \internal
	* \brief Create a new action object
	*
	* \param[in] key Action key
	* \param[in] task Action name
	* \param[in] rsc Resource that action is for (if any)
	* \param[in] node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in] for_graph Whether action should be recorded in transition graph
	* \param[in] data_set Cluster working set
	*
	* \return Newly allocated action
	* \note This function takes ownership of \p key. It is the caller's
	* responsibility to free the return value with pe_free_action().
	*/
	static pe_action_t *
	new_action(char key, const char task, pe_resource_t rsc, pe_node_t node,
	bool optional, bool for_graph, pe_working_set_t *data_set)
	{
	pe_action_t *action = calloc(1, sizeof(pe_action_t));

	CRM_ASSERT(action != NULL);

	action->rsc = rsc;
	action->task = strdup(task); CRM_ASSERT(action->task != NULL);
	action->uuid = key;
	action->extra = pcmk__strkey_table(free, free);
	action->meta = pcmk__strkey_table(free, free);

	if (node) {
	action->node = pe__copy_node(node);
	}

	if (pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	// Resource history deletion for a node can be done on the DC
	pe__set_action_flags(action, pe_action_dc);
	}

	pe__set_action_flags(action, pe_action_runnable);
	if (optional) {
	pe__set_action_flags(action, pe_action_optional);
	} else {
	pe__clear_action_flags(action, pe_action_optional);
	}

	if (rsc != NULL) {
	guint interval_ms = 0;

	action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE);
	parse_op_key(key, NULL, NULL, &interval_ms);
	unpack_operation(action, action->op_entry, rsc->container, data_set,
	interval_ms);
	}

	if (for_graph) {
	pe_rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
	(optional? "optional" : "required"),
	data_set->action_id, key, task,
	((rsc == NULL)? "no resource" : rsc->id),
	((node == NULL)? "no node" : node->details->uname));
	action->id = data_set->action_id++;

	data_set->actions = g_list_prepend(data_set->actions, action);
	if (rsc == NULL) {
	add_singleton(data_set, action);
	} else {
	rsc->actions = g_list_prepend(rsc->actions, action);
	}
	}
	return action;
	}

	/*!
	* \internal
	* \brief Evaluate node attribute values for an action
	*
	* \param[in] action Action to unpack attributes for
	* \param[in] data_set Cluster working set
	*/
	static void
	unpack_action_node_attributes(pe_action_t action, pe_working_set_t data_set)
	{
	if (!pcmk_is_set(action->flags, pe_action_have_node_attrs)
	&& (action->op_entry != NULL)) {

	pe_rule_eval_data_t rule_data = {
	.node_hash = action->node->details->attrs,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe__set_action_flags(action, pe_action_have_node_attrs);
	pe__unpack_dataset_nvpairs(action->op_entry, XML_TAG_ATTR_SETS,
	&rule_data, action->extra, NULL,
	FALSE, data_set);
	}
	}

	/*!
	* \internal
	* \brief Update an action's optional flag
	*
	* \param[in] action Action to update
	* \param[in] optional Requested optional status
	*/
	static void
	update_action_optional(pe_action_t *action, gboolean optional)
	{
	// Force a non-recurring action to be optional if its resource is unmanaged
	if ((action->rsc != NULL) && (action->node != NULL)
	&& !pcmk_is_set(action->flags, pe_action_pseudo)
	&& !pcmk_is_set(action->rsc->flags, pe_rsc_managed)
	&& (g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS) == NULL)) {
	pe_rsc_debug(action->rsc, "%s on %s is optional (%s is unmanaged)",
	action->uuid, action->node->details->uname,
	action->rsc->id);
	pe__set_action_flags(action, pe_action_optional);
	// We shouldn't clear runnable here because ... something

	// Otherwise require the action if requested
	} else if (!optional) {
	pe__clear_action_flags(action, pe_action_optional);
	}
	}

	/*!
	* \internal
	* \brief Update a resource action's runnable flag
	*
	* \param[in] action Action to update
	* \param[in] for_graph Whether action should be recorded in transition graph
	* \param[in] data_set Cluster working set
	*
	* \note This may also schedule fencing if a stop is unrunnable.
	*/
	static void
	update_resource_action_runnable(pe_action_t *action, bool for_graph,
	pe_working_set_t *data_set)
	{
	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	return;
	}

	if (action->node == NULL) {
	pe_rsc_trace(action->rsc, "%s is unrunnable (unallocated)",
	action->uuid);
	pe__clear_action_flags(action, pe_action_runnable);

	} else if (!pcmk_is_set(action->flags, pe_action_dc)
	&& !(action->node->details->online)
	&& (!pe__is_guest_node(action->node)
	\|\| action->node->details->remote_requires_reset)) {
	pe__clear_action_flags(action, pe_action_runnable);
	do_crm_log((for_graph? LOG_WARNING: LOG_TRACE),
	"%s on %s is unrunnable (node is offline)",
	action->uuid, action->node->details->uname);
	if (pcmk_is_set(action->rsc->flags, pe_rsc_managed)
	&& for_graph
	&& pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)
	&& !(action->node->details->unclean)) {
	pe_fence_node(data_set, action->node, "stop is unrunnable", false);
	}

	} else if (!pcmk_is_set(action->flags, pe_action_dc)
	&& action->node->details->pending) {
	pe__clear_action_flags(action, pe_action_runnable);
	do_crm_log((for_graph? LOG_WARNING: LOG_TRACE),
	"Action %s on %s is unrunnable (node is pending)",
	action->uuid, action->node->details->uname);

	} else if (action->needs == rsc_req_nothing) {
	pe_action_set_reason(action, NULL, TRUE);
	if (pe__is_guest_node(action->node)
	&& !pe_can_fence(data_set, action->node)) {
	/* An action that requires nothing usually does not require any
	* fencing in order to be runnable. However, there is an exception:
	* such an action cannot be completed if it is on a guest node whose
	* host is unclean and cannot be fenced.
	*/
	pe_rsc_debug(action->rsc, "%s on %s is unrunnable "
	"(node's host cannot be fenced)",
	action->uuid, action->node->details->uname);
	pe__clear_action_flags(action, pe_action_runnable);
	} else {
	pe_rsc_trace(action->rsc,
	"%s on %s does not require fencing or quorum",
	action->uuid, action->node->details->uname);
	pe__set_action_flags(action, pe_action_runnable);
	}

	} else {
	switch (effective_quorum_policy(action->rsc, data_set)) {
	case no_quorum_stop:
	pe_rsc_debug(action->rsc, "%s on %s is unrunnable (no quorum)",
	action->uuid, action->node->details->uname);
	pe__clear_action_flags(action, pe_action_runnable);
	pe_action_set_reason(action, "no quorum", true);
	break;

	case no_quorum_freeze:
	if (!action->rsc->fns->active(action->rsc, TRUE)
	\|\| (action->rsc->next_role > action->rsc->role)) {
	pe_rsc_debug(action->rsc,
	"%s on %s is unrunnable (no quorum)",
	action->uuid, action->node->details->uname);
	pe__clear_action_flags(action, pe_action_runnable);
	pe_action_set_reason(action, "quorum freeze", true);
	}
	break;

	default:
	//pe_action_set_reason(action, NULL, TRUE);
	pe__set_action_flags(action, pe_action_runnable);
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource object's flags for a new action on it
	*
	* \param[in] rsc Resource that action is for (if any)
	* \param[in] action New action
	*/
	static void
	update_resource_flags_for_action(pe_resource_t rsc, pe_action_t action)
	{
	/* @COMPAT pe_rsc_starting and pe_rsc_stopping are not actually used
	* within Pacemaker, and should be deprecated and eventually removed
	*/
	if (pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pe_rsc_stopping);

	} else if (pcmk__str_eq(action->task, CRMD_ACTION_START, pcmk__str_casei)) {
	if (pcmk_is_set(action->flags, pe_action_runnable)) {
	pe__set_resource_flags(rsc, pe_rsc_starting);
	} else {
	pe__clear_resource_flags(rsc, pe_rsc_starting);
	}
	}
	}

	/*!
	* \brief Create or update an action object
	*
	* \param[in] rsc Resource that action is for (if any)
	* \param[in] key Action key (must be non-NULL)
	* \param[in] task Action name (must be non-NULL)
	* \param[in] on_node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in] save_action Whether action should be recorded in transition graph
	* \param[in] data_set Cluster working set
	*
	* \return Action object corresponding to arguments
	* \note This function takes ownership of (and might free) \p key. If
	* \p save_action is true, \p data_set will own the returned action,
	* otherwise it is the caller's responsibility to free the return value
	* with pe_free_action().
	*/
	pe_action_t *
	custom_action(pe_resource_t rsc, char key, const char *task,
	pe_node_t *on_node, gboolean optional, gboolean save_action,
	pe_working_set_t *data_set)
	{
	pe_action_t *action = NULL;

	CRM_ASSERT((key != NULL) && (task != NULL) && (data_set != NULL));

	if (save_action) {
	action = find_existing_action(key, rsc, on_node, data_set);
	}

	if (action == NULL) {
	action = new_action(key, task, rsc, on_node, optional, save_action,
	data_set);
	} else {
	free(key);
	}

	update_action_optional(action, optional);

	if (rsc != NULL) {
	if (action->node != NULL) {
	unpack_action_node_attributes(action, data_set);
	}

	update_resource_action_runnable(action, save_action, data_set);

	if (save_action) {
	update_resource_flags_for_action(rsc, action);
	}
	}

	return action;
	}

	static bool
	valid_stop_on_fail(const char *value)
	{
	return !pcmk__strcase_any_of(value, "standby", "demote", "stop", NULL);
	}

	static const char *
	unpack_operation_on_fail(pe_action_t * action)
	{

	const char *name = NULL;
	const char *role = NULL;
	const char *on_fail = NULL;
	const char *interval_spec = NULL;
	const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);

	if (pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)
	&& !valid_stop_on_fail(value)) {

	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
	"action to default value because '%s' is not "
	"allowed for stop", action->rsc->id, value);
	return NULL;

	} else if (pcmk__str_eq(action->task, CRMD_ACTION_DEMOTE, pcmk__str_casei) && !value) {
	// demote on_fail defaults to monitor value for promoted role if present
	xmlNode *operation = NULL;

	CRM_CHECK(action->rsc != NULL, return NULL);

	for (operation = pcmk__xe_first_child(action->rsc->ops_xml);
	(operation != NULL) && (value == NULL);
	operation = pcmk__xe_next(operation)) {
	bool enabled = false;

	if (!pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	continue;
	}
	name = crm_element_value(operation, "name");
	role = crm_element_value(operation, "role");
	on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (!on_fail) {
	continue;
	} else if (pcmk__xe_get_bool_attr(operation, "enabled", &enabled) == pcmk_rc_ok && !enabled) {
	continue;
	} else if (!pcmk__str_eq(name, "monitor", pcmk__str_casei)
	\|\| !pcmk__strcase_any_of(role, RSC_ROLE_PROMOTED_S,
	RSC_ROLE_PROMOTED_LEGACY_S,
	NULL)) {
	continue;
	} else if (crm_parse_interval_spec(interval_spec) == 0) {
	continue;
	} else if (pcmk__str_eq(on_fail, "demote", pcmk__str_casei)) {
	continue;
	}

	value = on_fail;
	}
	} else if (pcmk__str_eq(action->task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	value = "ignore";

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	name = crm_element_value(action->op_entry, "name");
	role = crm_element_value(action->op_entry, "role");
	interval_spec = crm_element_value(action->op_entry,
	XML_LRM_ATTR_INTERVAL);

	if (!pcmk__str_eq(name, CRMD_ACTION_PROMOTE, pcmk__str_casei)
	&& (!pcmk__str_eq(name, CRMD_ACTION_STATUS, pcmk__str_casei)
	\|\| !pcmk__strcase_any_of(role, RSC_ROLE_PROMOTED_S,
	RSC_ROLE_PROMOTED_LEGACY_S, NULL)
	\|\| (crm_parse_interval_spec(interval_spec) == 0))) {
	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s "
	"action to default value because 'demote' is not "
	"allowed for it", action->rsc->id, name);
	return NULL;
	}
	}

	return value;
	}

	static xmlNode *
	find_min_interval_mon(pe_resource_t * rsc, gboolean include_disabled)
	{
	guint interval_ms = 0;
	guint min_interval_ms = G_MAXUINT;
	const char *name = NULL;
	const char *interval_spec = NULL;
	xmlNode *op = NULL;
	xmlNode *operation = NULL;

	for (operation = pcmk__xe_first_child(rsc->ops_xml);
	operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	bool enabled = false;

	name = crm_element_value(operation, "name");
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (!include_disabled && pcmk__xe_get_bool_attr(operation, "enabled", &enabled) == pcmk_rc_ok &&
	!enabled) {
	continue;
	}

	if (!pcmk__str_eq(name, RSC_STATUS, pcmk__str_casei)) {
	continue;
	}

	interval_ms = crm_parse_interval_spec(interval_spec);

	if (interval_ms && (interval_ms < min_interval_ms)) {
	min_interval_ms = interval_ms;
	op = operation;
	}
	}
	}

	return op;
	}

	static int
	unpack_start_delay(const char value, GHashTable meta)
	{
	int start_delay = 0;

	if (value != NULL) {
	start_delay = crm_get_msec(value);

	if (start_delay < 0) {
	start_delay = 0;
	}

	if (meta) {
	g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY),
	pcmk__itoa(start_delay));
	}
	}

	return start_delay;
	}

	// true if value contains valid, non-NULL interval origin for recurring op
	static bool
	unpack_interval_origin(const char value, xmlNode xml_obj, guint interval_ms,
	crm_time_t now, long long start_delay)
	{
	long long result = 0;
	guint interval_sec = interval_ms / 1000;
	crm_time_t *origin = NULL;

	// Ignore unspecified values and non-recurring operations
	if ((value == NULL) \|\| (interval_ms == 0) \|\| (now == NULL)) {
	return false;
	}

	// Parse interval origin from text
	origin = crm_time_new(value);
	if (origin == NULL) {
	pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
	"'%s' because '%s' is not valid",
	(ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
	return false;
	}

	// Get seconds since origin (negative if origin is in the future)
	result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
	crm_time_free(origin);

	// Calculate seconds from closest interval to now
	result = result % interval_sec;

	// Calculate seconds remaining until next interval
	result = ((result <= 0)? 0 : interval_sec) - result;
	crm_info("Calculated a start delay of %llds for operation '%s'",
	result,
	(ID(xml_obj)? ID(xml_obj) : "(unspecified)"));

	if (start_delay != NULL) {
	start_delay = result 1000; // milliseconds
	}
	return true;
	}

	static int
	unpack_timeout(const char *value)
	{
	int timeout_ms = crm_get_msec(value);

	if (timeout_ms < 0) {
	timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
	}
	return timeout_ms;
	}

	int
	pe_get_configured_timeout(pe_resource_t rsc, const char action, pe_working_set_t *data_set)
	{
	xmlNode *child = NULL;
	GHashTable *action_meta = NULL;
	const char *timeout_spec = NULL;
	int timeout_ms = 0;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	child != NULL; child = crm_next_same_xml(child)) {
	if (pcmk__str_eq(action, crm_element_value(child, XML_NVPAIR_ATTR_NAME),
	pcmk__str_casei)) {
	timeout_spec = crm_element_value(child, XML_ATTR_TIMEOUT);
	break;
	}
	}

	if (timeout_spec == NULL && data_set->op_defaults) {
	action_meta = pcmk__strkey_table(free, free);
	pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS,
	&rule_data, action_meta, NULL, FALSE, data_set);
	timeout_spec = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
	}

	// @TODO check meta-attributes (including versioned meta-attributes)
	// @TODO maybe use min-interval monitor timeout as default for monitors

	timeout_ms = crm_get_msec(timeout_spec);
	if (timeout_ms < 0) {
	timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
	}

	if (action_meta != NULL) {
	g_hash_table_destroy(action_meta);
	}
	return timeout_ms;
	}

	#if ENABLE_VERSIONED_ATTRS
	static void
	unpack_versioned_meta(xmlNode versioned_meta, xmlNode xml_obj,
	guint interval_ms, crm_time_t *now)
	{
	xmlNode *attrs = NULL;
	xmlNode *attr = NULL;

	for (attrs = pcmk__xe_first_child(versioned_meta); attrs != NULL;
	attrs = pcmk__xe_next(attrs)) {

	for (attr = pcmk__xe_first_child(attrs); attr != NULL;
	attr = pcmk__xe_next(attr)) {

	const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
	const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);

	if (pcmk__str_eq(name, XML_OP_ATTR_START_DELAY, pcmk__str_casei)) {
	int start_delay = unpack_start_delay(value, NULL);

	crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
	} else if (pcmk__str_eq(name, XML_OP_ATTR_ORIGIN, pcmk__str_casei)) {
	long long start_delay = 0;

	if (unpack_interval_origin(value, xml_obj, interval_ms, now,
	&start_delay)) {
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME,
	XML_OP_ATTR_START_DELAY);
	crm_xml_add_ll(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
	}
	} else if (pcmk__str_eq(name, XML_ATTR_TIMEOUT, pcmk__str_casei)) {
	int timeout_ms = unpack_timeout(value);

	crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, timeout_ms);
	}
	}
	}
	}
	#endif

	/*!
	* \brief Unpack operation XML into an action structure
	*
	* Unpack an operation's meta-attributes (normalizing the interval, timeout,
	* and start delay values as integer milliseconds), requirements, and
	* failure policy.
	*
	* \param[in,out] action Action to unpack into
	* \param[in] xml_obj Operation XML (or NULL if all defaults)
	* \param[in] container Resource that contains affected resource, if any
	* \param[in] data_set Cluster state
	* \param[in] interval_ms How frequently to perform the operation
	*/
	static void
	unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
	pe_working_set_t * data_set, guint interval_ms)
	{
	int timeout_ms = 0;
	const char *value = NULL;
	bool is_probe = false;
	#if ENABLE_VERSIONED_ATTRS
	pe_rsc_action_details_t *rsc_details = NULL;
	#endif

	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(action->rsc->xml, XML_EXPR_ATTR_TYPE)
	};

	pe_op_eval_data_t op_rule_data = {
	.op_name = action->task,
	.interval = interval_ms
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = &op_rule_data
	};

	CRM_CHECK(action && action->rsc, return);

	is_probe = pcmk_is_probe(action->task, interval_ms);

	// Cluster-wide <op_defaults> <meta_attributes>
	pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, &rule_data,
	action->meta, NULL, FALSE, data_set);

	// Determine probe default timeout differently
	if (is_probe) {
	xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE);

	if (min_interval_mon) {
	value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT);
	if (value) {
	crm_trace("\t%s: Setting default timeout to minimum-interval "
	"monitor's timeout '%s'", action->uuid, value);
	g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
	strdup(value));
	}
	}
	}

	if (xml_obj) {
	xmlAttrPtr xIter = NULL;

	// <op> <meta_attributes> take precedence over defaults
	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_META_SETS, &rule_data,
	action->meta, NULL, TRUE, data_set);

	#if ENABLE_VERSIONED_ATTRS
	rsc_details = pe_rsc_action_details(action);

	pe_eval_versioned_attributes(data_set->input, xml_obj,
	XML_TAG_ATTR_SETS, &rule_data,
	rsc_details->versioned_parameters,
	NULL);
	pe_eval_versioned_attributes(data_set->input, xml_obj,
	XML_TAG_META_SETS, &rule_data,
	rsc_details->versioned_meta,
	NULL);
	#endif

	/* Anything set as an <op> XML property has highest precedence.
	* This ensures we use the name and interval from the <op> tag.
	*/
	for (xIter = xml_obj->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = crm_element_value(xml_obj, prop_name);

	g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value));
	}
	}

	g_hash_table_remove(action->meta, "id");

	// Normalize interval to milliseconds
	if (interval_ms > 0) {
	g_hash_table_replace(action->meta, strdup(XML_LRM_ATTR_INTERVAL),
	crm_strdup_printf("%u", interval_ms));
	} else {
	g_hash_table_remove(action->meta, XML_LRM_ATTR_INTERVAL);
	}

	/*
	* Timeout order of precedence:
	* 1. pcmk_monitor_timeout (if rsc has pcmk_ra_cap_fence_params
	* and task is start or a probe; pcmk_monitor_timeout works
	* by default for a recurring monitor)
	* 2. explicit op timeout on the primitive
	* 3. default op timeout
	* a. if probe, then min-interval monitor's timeout
	* b. else, in XML_CIB_TAG_OPCONFIG
	* 4. CRM_DEFAULT_OP_TIMEOUT_S
	*
	* #1 overrides general rule of <op> XML property having highest
	* precedence.
	*/
	if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
	pcmk_ra_cap_fence_params)
	&& (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
	\|\| is_probe)) {

	GHashTable *params = pe_rsc_params(action->rsc, action->node, data_set);

	value = g_hash_table_lookup(params, "pcmk_monitor_timeout");

	if (value) {
	crm_trace("\t%s: Setting timeout to pcmk_monitor_timeout '%s', "
	"overriding default", action->uuid, value);
	g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
	strdup(value));
	}
	}

	// Normalize timeout to positive milliseconds
	value = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
	timeout_ms = unpack_timeout(value);
	g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
	pcmk__itoa(timeout_ms));

	if (!pcmk__strcase_any_of(action->task, RSC_START, RSC_PROMOTE, NULL)) {
	action->needs = rsc_req_nothing;
	value = "nothing (not start or promote)";

	} else if (pcmk_is_set(action->rsc->flags, pe_rsc_needs_fencing)) {
	action->needs = rsc_req_stonith;
	value = "fencing";

	} else if (pcmk_is_set(action->rsc->flags, pe_rsc_needs_quorum)) {
	action->needs = rsc_req_quorum;
	value = "quorum";

	} else {
	action->needs = rsc_req_nothing;
	value = "nothing";
	}
	pe_rsc_trace(action->rsc, "%s requires %s", action->uuid, value);

	value = unpack_operation_on_fail(action);

	if (value == NULL) {

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	action->on_fail = action_fail_block;
	g_hash_table_insert(action->meta, strdup(XML_OP_ATTR_ON_FAIL), strdup("block"));
	value = "block"; // The above could destroy the original string

	} else if (pcmk__str_eq(value, "fence", pcmk__str_casei)) {
	action->on_fail = action_fail_fence;
	value = "node fencing";

	if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
	"operation '%s' to 'stop' because 'fence' is not "
	"valid when fencing is disabled", action->uuid);
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop resource";
	}

	} else if (pcmk__str_eq(value, "standby", pcmk__str_casei)) {
	action->on_fail = action_fail_standby;
	value = "node standby";

	} else if (pcmk__strcase_any_of(value, "ignore", "nothing", NULL)) {
	action->on_fail = action_fail_ignore;
	value = "ignore";

	} else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
	action->on_fail = action_fail_migrate;
	value = "force migration";

	} else if (pcmk__str_eq(value, "stop", pcmk__str_casei)) {
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop resource";

	} else if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	action->on_fail = action_fail_recover;
	value = "restart (and possibly migrate)";

	} else if (pcmk__str_eq(value, "restart-container", pcmk__str_casei)) {
	if (container) {
	action->on_fail = action_fail_restart_container;
	value = "restart container (and possibly migrate)";

	} else {
	value = NULL;
	}

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	action->on_fail = action_fail_demote;
	value = "demote instance";

	} else {
	pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
	value = NULL;
	}

	/* defaults */
	if (value == NULL && container) {
	action->on_fail = action_fail_restart_container;
	value = "restart container (and possibly migrate) (default)";

	/* For remote nodes, ensure that any failure that results in dropping an
	* active connection to the node results in fencing of the node.
	*
	* There are only two action failures that don't result in fencing.
	* 1. probes - probe failures are expected.
	* 2. start - a start failure indicates that an active connection does not already
	* exist. The user can set op on-fail=fence if they really want to fence start
	* failures. */
	} else if (((value == NULL) \|\| !pcmk_is_set(action->rsc->flags, pe_rsc_managed))
	&& pe__resource_is_remote_conn(action->rsc, data_set)
	&& !(pcmk__str_eq(action->task, CRMD_ACTION_STATUS, pcmk__str_casei)
	&& (interval_ms == 0))
	&& !pcmk__str_eq(action->task, CRMD_ACTION_START, pcmk__str_casei)) {

	if (!pcmk_is_set(action->rsc->flags, pe_rsc_managed)) {
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop unmanaged remote node (enforcing default)";

	} else {
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	value = "fence remote node (default)";
	} else {
	value = "recover remote node connection (default)";
	}

	if (action->rsc->remote_reconnect_ms) {
	action->fail_role = RSC_ROLE_STOPPED;
	}
	action->on_fail = action_fail_reset_remote;
	}

	} else if (value == NULL && pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)) {
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	action->on_fail = action_fail_fence;
	value = "resource fence (default)";

	} else {
	action->on_fail = action_fail_block;
	value = "resource block (default)";
	}

	} else if (value == NULL) {
	action->on_fail = action_fail_recover;
	value = "restart (and possibly migrate) (default)";
	}

	pe_rsc_trace(action->rsc, "%s failure handling: %s",
	action->uuid, value);

	value = NULL;
	if (xml_obj != NULL) {
	value = g_hash_table_lookup(action->meta, "role_after_failure");
	if (value) {
	pe_warn_once(pe_wo_role_after,
	"Support for role_after_failure is deprecated and will be removed in a future release");
	}
	}
	if (value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) {
	action->fail_role = text2role(value);
	}
	/* defaults */
	if (action->fail_role == RSC_ROLE_UNKNOWN) {
	if (pcmk__str_eq(action->task, CRMD_ACTION_PROMOTE, pcmk__str_casei)) {
	action->fail_role = RSC_ROLE_UNPROMOTED;
	} else {
	action->fail_role = RSC_ROLE_STARTED;
	}
	}
	pe_rsc_trace(action->rsc, "%s failure results in: %s",
	action->uuid, role2text(action->fail_role));

	value = g_hash_table_lookup(action->meta, XML_OP_ATTR_START_DELAY);
	if (value) {
	unpack_start_delay(value, action->meta);
	} else {
	long long start_delay = 0;

	value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN);
	if (unpack_interval_origin(value, xml_obj, interval_ms, data_set->now,
	&start_delay)) {
	g_hash_table_replace(action->meta, strdup(XML_OP_ATTR_START_DELAY),
	crm_strdup_printf("%lld", start_delay));
	}
	}

	#if ENABLE_VERSIONED_ATTRS
	unpack_versioned_meta(rsc_details->versioned_meta, xml_obj, interval_ms,
	data_set->now);
	#endif
	}

	static xmlNode *
	find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key, gboolean include_disabled)
	{
	guint interval_ms = 0;
	gboolean do_retry = TRUE;
	char *local_key = NULL;
	const char *name = NULL;
	const char *interval_spec = NULL;
	char *match_key = NULL;
	xmlNode *op = NULL;
	xmlNode *operation = NULL;

	retry:
	for (operation = pcmk__xe_first_child(rsc->ops_xml); operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	bool enabled = false;

	name = crm_element_value(operation, "name");
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (!include_disabled && pcmk__xe_get_bool_attr(operation, "enabled", &enabled) == pcmk_rc_ok &&
	!enabled) {
	continue;
	}

	interval_ms = crm_parse_interval_spec(interval_spec);
	match_key = pcmk__op_key(rsc->id, name, interval_ms);
	if (pcmk__str_eq(key, match_key, pcmk__str_casei)) {
	op = operation;
	}
	free(match_key);

	if (rsc->clone_name) {
	match_key = pcmk__op_key(rsc->clone_name, name, interval_ms);
	if (pcmk__str_eq(key, match_key, pcmk__str_casei)) {
	op = operation;
	}
	free(match_key);
	}

	if (op != NULL) {
	free(local_key);
	return op;
	}
	}
	}

	free(local_key);
	if (do_retry == FALSE) {
	return NULL;
	}

	do_retry = FALSE;
	if (strstr(key, CRMD_ACTION_MIGRATE) \|\| strstr(key, CRMD_ACTION_MIGRATED)) {
	local_key = pcmk__op_key(rsc->id, "migrate", 0);
	key = local_key;
	goto retry;

	} else if (strstr(key, "_notify_")) {
	local_key = pcmk__op_key(rsc->id, "notify", 0);
	key = local_key;
	goto retry;
	}

	return NULL;
	}

	xmlNode *
	find_rsc_op_entry(pe_resource_t * rsc, const char *key)
	{
	return find_rsc_op_entry_helper(rsc, key, FALSE);
	}

	/*
	* Used by the HashTable for-loop
	*/
	void
	print_str_str(gpointer key, gpointer value, gpointer user_data)
	{
	crm_trace("%s%s %s ==> %s",
	user_data == NULL ? "" : (char *)user_data,
	user_data == NULL ? "" : ": ", (char )key, (char )value);
	}

	void
	pe_free_action(pe_action_t * action)
	{
	if (action == NULL) {
	return;
	}
	g_list_free_full(action->actions_before, free); /* pe_action_wrapper_t* */
	g_list_free_full(action->actions_after, free); /* pe_action_wrapper_t* */
	if (action->extra) {
	g_hash_table_destroy(action->extra);
	}
	if (action->meta) {
	g_hash_table_destroy(action->meta);
	}
	#if ENABLE_VERSIONED_ATTRS
	if (action->rsc) {
	pe_free_rsc_action_details(action);
	}
	#endif
	free(action->cancel_task);
	free(action->reason);
	free(action->task);
	free(action->uuid);
	free(action->node);
	free(action);
	}

	GList *
	find_recurring_actions(GList input, pe_node_t not_on_node)
	{
	const char *value = NULL;
	GList *result = NULL;
	GList *gIter = input;

	CRM_CHECK(input != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL_MS);
	if (value == NULL) {
	/* skip */
	} else if (pcmk__str_eq(value, "0", pcmk__str_casei)) {
	/* skip */
	} else if (pcmk__str_eq(CRMD_ACTION_CANCEL, action->task, pcmk__str_casei)) {
	/* skip */
	} else if (not_on_node == NULL) {
	crm_trace("(null) Found: %s", action->uuid);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	/* skip */
	} else if (action->node->details != not_on_node->details) {
	crm_trace("Found: %s", action->uuid);
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	enum action_tasks
	get_complex_task(pe_resource_t * rsc, const char *name, gboolean allow_non_atomic)
	{
	enum action_tasks task = text2task(name);

	if (rsc == NULL) {
	return task;

	} else if (allow_non_atomic == FALSE \|\| rsc->variant == pe_native) {
	switch (task) {
	case stopped_rsc:
	case started_rsc:
	case action_demoted:
	case action_promoted:
	crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id);
	return task - 1;
	default:
	break;
	}
	}
	return task;
	}

	pe_action_t *
	find_first_action(GList input, const char uuid, const char task, pe_node_t on_node)
	{
	GList *gIter = NULL;

	CRM_CHECK(uuid \|\| task, return NULL);

	for (gIter = input; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	return action;

	} else if (action->node == NULL) {
	continue;

	} else if (on_node->details == action->node->details) {
	return action;
	}
	}

	return NULL;
	}

	GList *
	find_actions(GList input, const char key, const pe_node_t *on_node)
	{
	GList *gIter = input;
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	crm_trace("Action %s matches (ignoring node)", key);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	crm_trace("Action %s matches (unallocated, assigning to %s)",
	key, on_node->details->uname);

	action->node = pe__copy_node(on_node);
	result = g_list_prepend(result, action);

	} else if (on_node->details == action->node->details) {
	crm_trace("Action %s on %s matches", key, on_node->details->uname);
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	GList *
	find_actions_exact(GList input, const char key, const pe_node_t *on_node)
	{
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	if (on_node == NULL) {
	return NULL;
	}

	for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if ((action->node != NULL)
	&& pcmk__str_eq(key, action->uuid, pcmk__str_casei)
	&& pcmk__str_eq(on_node->details->id, action->node->details->id,
	pcmk__str_casei)) {

	crm_trace("Action %s on %s matches", key, on_node->details->uname);
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	/*!
	* \brief Find all actions of given type for a resource
	*
	* \param[in] rsc Resource to search
	* \param[in] node Find only actions scheduled on this node
	* \param[in] task Action name to search for
	* \param[in] require_node If TRUE, NULL node or action node will not match
	*
	* \return List of actions found (or NULL if none)
	* \note If node is not NULL and require_node is FALSE, matching actions
	* without a node will be assigned to node.
	*/
	GList *
	pe__resource_actions(const pe_resource_t rsc, const pe_node_t node,
	const char *task, bool require_node)
	{
	GList *result = NULL;
	char *key = pcmk__op_key(rsc->id, task, 0);

	if (require_node) {
	result = find_actions_exact(rsc->actions, key, node);
	} else {
	result = find_actions(rsc->actions, key, node);
	}
	free(key);
	return result;
	}

	static void
	resource_node_score(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag)
	{
	pe_node_t *match = NULL;

	if ((rsc->exclusive_discover \|\| (node->rsc_discover_mode == pe_discover_never))
	&& pcmk__str_eq(tag, "symmetric_default", pcmk__str_casei)) {
	/* This string comparision may be fragile, but exclusive resources and
	* exclusive nodes should not have the symmetric_default constraint
	* applied to them.
	*/
	return;

	} else if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	resource_node_score(child_rsc, node, score, tag);
	}
	}

	pe_rsc_trace(rsc, "Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score);
	match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match == NULL) {
	match = pe__copy_node(node);
	g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
	}
	- match->weight = pe__add_scores(match->weight, score);
	+ match->weight = pcmk__add_scores(match->weight, score);
	}

	void
	resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
	pe_working_set_t * data_set)
	{
	if (node != NULL) {
	resource_node_score(rsc, node, score, tag);

	} else if (data_set != NULL) {
	GList *gIter = data_set->nodes;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node_iter = (pe_node_t ) gIter->data;

	resource_node_score(rsc, node_iter, score, tag);
	}

	} else {
	GHashTableIter iter;
	pe_node_t *node_iter = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
	resource_node_score(rsc, node_iter, score, tag);
	}
	}

	if (node == NULL && score == -INFINITY) {
	if (rsc->allocated_to) {
	crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname);
	free(rsc->allocated_to);
	rsc->allocated_to = NULL;
	}
	}
	}

	#define sort_return(an_int, why) do { \
	free(a_uuid); \
	free(b_uuid); \
	crm_trace("%s (%d) %c %s (%d) : %s", \
	a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
	b_xml_id, b_call_id, why); \
	return an_int; \
	} while(0)

	gint
	sort_op_by_callid(gconstpointer a, gconstpointer b)
	{
	int a_call_id = -1;
	int b_call_id = -1;

	char *a_uuid = NULL;
	char *b_uuid = NULL;

	const xmlNode *xml_a = a;
	const xmlNode *xml_b = b;

	const char *a_xml_id = crm_element_value(xml_a, XML_ATTR_ID);
	const char *b_xml_id = crm_element_value(xml_b, XML_ATTR_ID);

	if (pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_casei)) {
	/* We have duplicate lrm_rsc_op entries in the status
	* section which is unlikely to be a good thing
	* - we can handle it easily enough, but we need to get
	* to the bottom of why it's happening.
	*/
	pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
	sort_return(0, "duplicate");
	}

	crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
	crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);

	if (a_call_id == -1 && b_call_id == -1) {
	/* both are pending ops so it doesn't matter since
	* stops are never pending
	*/
	sort_return(0, "pending");

	} else if (a_call_id >= 0 && a_call_id < b_call_id) {
	sort_return(-1, "call id");

	} else if (b_call_id >= 0 && a_call_id > b_call_id) {
	sort_return(1, "call id");

	} else if (b_call_id >= 0 && a_call_id == b_call_id) {
	/*
	* The op and last_failed_op are the same
	* Order on last-rc-change
	*/
	time_t last_a = -1;
	time_t last_b = -1;

	crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
	crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);

	crm_trace("rc-change: %lld vs %lld",
	(long long) last_a, (long long) last_b);
	if (last_a >= 0 && last_a < last_b) {
	sort_return(-1, "rc-change");

	} else if (last_b >= 0 && last_a > last_b) {
	sort_return(1, "rc-change");
	}
	sort_return(0, "rc-change");

	} else {
	/* One of the inputs is a pending operation
	* Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
	*/

	int a_id = -1;
	int b_id = -1;

	const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
	const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);

	CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
	if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic a");
	}
	if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic b");
	}
	/* try to determine the relative age of the operation...
	* some pending operations (e.g. a start) may have been superseded
	* by a subsequent stop
	*
	* [a\|b]_id == -1 means it's a shutdown operation and _always_ comes last
	*/
	if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) \|\| a_id == b_id) {
	/*
	* some of the logic in here may be redundant...
	*
	* if the UUID from the TE doesn't match then one better
	* be a pending operation.
	* pending operations don't survive between elections and joins
	* because we query the LRM directly
	*/

	if (b_call_id == -1) {
	sort_return(-1, "transition + call");

	} else if (a_call_id == -1) {
	sort_return(1, "transition + call");
	}

	} else if ((a_id >= 0 && a_id < b_id) \|\| b_id == -1) {
	sort_return(-1, "transition");

	} else if ((b_id >= 0 && a_id > b_id) \|\| a_id == -1) {
	sort_return(1, "transition");
	}
	}

	/* we should never end up here */
	CRM_CHECK(FALSE, sort_return(0, "default"));

	}

	time_t
	get_effective_time(pe_working_set_t * data_set)
	{
	if(data_set) {
	if (data_set->now == NULL) {
	crm_trace("Recording a new 'now'");
	data_set->now = crm_time_new(NULL);
	}
	return crm_time_get_seconds_since_epoch(data_set->now);
	}

	crm_trace("Defaulting to 'now'");
	return time(NULL);
	}

	gboolean
	get_target_role(pe_resource_t * rsc, enum rsc_role_e * role)
	{
	enum rsc_role_e local_role = RSC_ROLE_UNKNOWN;
	const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);

	CRM_CHECK(role != NULL, return FALSE);

	if (pcmk__str_eq(value, "started", pcmk__str_null_matches \| pcmk__str_casei)
	\|\| pcmk__str_eq("default", value, pcmk__str_casei)) {
	return FALSE;
	}

	local_role = text2role(value);
	if (local_role == RSC_ROLE_UNKNOWN) {
	pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
	"because '%s' is not valid", rsc->id, value);
	return FALSE;

	} else if (local_role > RSC_ROLE_STARTED) {
	if (pcmk_is_set(uber_parent(rsc)->flags, pe_rsc_promotable)) {
	if (local_role > RSC_ROLE_UNPROMOTED) {
	/* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
	return FALSE;
	}

	} else {
	pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
	"because '%s' only makes sense for promotable "
	"clones", rsc->id, value);
	return FALSE;
	}
	}

	*role = local_role;
	return TRUE;
	}

	gboolean
	order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order)
	{
	GList *gIter = NULL;
	pe_action_wrapper_t *wrapper = NULL;
	GList *list = NULL;

	if (order == pe_order_none) {
	return FALSE;
	}

	if (lh_action == NULL \|\| rh_action == NULL) {
	return FALSE;
	}

	crm_trace("Creating action wrappers for ordering: %s then %s",
	lh_action->uuid, rh_action->uuid);

	/* Ensure we never create a dependency on ourselves... it's happened */
	CRM_ASSERT(lh_action != rh_action);

	/* Filter dups, otherwise update_action_states() has too much work to do */
	gIter = lh_action->actions_after;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_wrapper_t after = (pe_action_wrapper_t ) gIter->data;

	if (after->action == rh_action && (after->type & order)) {
	return FALSE;
	}
	}

	wrapper = calloc(1, sizeof(pe_action_wrapper_t));
	wrapper->action = rh_action;
	wrapper->type = order;
	list = lh_action->actions_after;
	list = g_list_prepend(list, wrapper);
	lh_action->actions_after = list;

	wrapper = calloc(1, sizeof(pe_action_wrapper_t));
	wrapper->action = lh_action;
	wrapper->type = order;
	list = rh_action->actions_before;
	list = g_list_prepend(list, wrapper);
	rh_action->actions_before = list;
	return TRUE;
	}

	pe_action_t *
	get_pseudo_op(const char name, pe_working_set_t data_set)
	{
	pe_action_t *op = lookup_singleton(data_set, name);

	if (op == NULL) {
	op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);
	}
	return op;
	}

	void
	destroy_ticket(gpointer data)
	{
	pe_ticket_t *ticket = data;

	if (ticket->state) {
	g_hash_table_destroy(ticket->state);
	}
	free(ticket->id);
	free(ticket);
	}

	pe_ticket_t *
	ticket_new(const char ticket_id, pe_working_set_t data_set)
	{
	pe_ticket_t *ticket = NULL;

	if (pcmk__str_empty(ticket_id)) {
	return NULL;
	}

	if (data_set->tickets == NULL) {
	data_set->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {

	ticket = calloc(1, sizeof(pe_ticket_t));
	if (ticket == NULL) {
	crm_err("Cannot allocate ticket '%s'", ticket_id);
	return NULL;
	}

	crm_trace("Creaing ticket entry for %s", ticket_id);

	ticket->id = strdup(ticket_id);
	ticket->granted = FALSE;
	ticket->last_granted = -1;
	ticket->standby = FALSE;
	ticket->state = pcmk__strkey_table(free, free);

	g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket);
	}

	return ticket;
	}

	const char rsc_printable_id(pe_resource_t rsc)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	return ID(rsc->xml);
	}
	return rsc->id;
	}

	void
	pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags)
	{
	pe__clear_resource_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__clear_resource_flags_recursive((pe_resource_t *) gIter->data, flags);
	}
	}

	void
	pe__clear_resource_flags_on_all(pe_working_set_t *data_set, uint64_t flag)
	{
	for (GList *lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t r = (pe_resource_t ) lpc->data;
	pe__clear_resource_flags_recursive(r, flag);
	}
	}

	void
	pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags)
	{
	pe__set_resource_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__set_resource_flags_recursive((pe_resource_t *) gIter->data, flags);
	}
	}

	static GList *
	find_unfencing_devices(GList candidates, GList matches)
	{
	for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *candidate = gIter->data;
	const char *provides = g_hash_table_lookup(candidate->meta,
	PCMK_STONITH_PROVIDES);
	const char *requires = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_REQUIRES);

	if(candidate->children) {
	matches = find_unfencing_devices(candidate->children, matches);
	} else if (!pcmk_is_set(candidate->flags, pe_rsc_fence_device)) {
	continue;

	} else if (pcmk__str_eq(provides, "unfencing", pcmk__str_casei) \|\| pcmk__str_eq(requires, "unfencing", pcmk__str_casei)) {
	matches = g_list_prepend(matches, candidate);
	}
	}
	return matches;
	}

	static int
	node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set)
	{
	int member_count = 0;
	int online_count = 0;
	int top_priority = 0;
	int lowest_priority = 0;
	GList *gIter = NULL;

	// `priority-fencing-delay` is disabled
	if (data_set->priority_fencing_delay <= 0) {
	return 0;
	}

	/* No need to request a delay if the fencing target is not a normal cluster
	* member, for example if it's a remote node or a guest node. */
	if (node->details->type != node_member) {
	return 0;
	}

	// No need to request a delay if the fencing target is in our partition
	if (node->details->online) {
	return 0;
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t *n = gIter->data;

	if (n->details->type != node_member) {
	continue;
	}

	member_count ++;

	if (n->details->online) {
	online_count++;
	}

	if (member_count == 1
	\|\| n->details->priority > top_priority) {
	top_priority = n->details->priority;
	}

	if (member_count == 1
	\|\| n->details->priority < lowest_priority) {
	lowest_priority = n->details->priority;
	}
	}

	// No need to delay if we have more than half of the cluster members
	if (online_count > member_count / 2) {
	return 0;
	}

	/* All the nodes have equal priority.
	* Any configured corresponding `pcmk_delay_base/max` will be applied. */
	if (lowest_priority == top_priority) {
	return 0;
	}

	if (node->details->priority < top_priority) {
	return 0;
	}

	return data_set->priority_fencing_delay;
	}

	pe_action_t *
	pe_fence_op(pe_node_t * node, const char op, bool optional, const char reason,
	bool priority_delay, pe_working_set_t * data_set)
	{
	char *op_key = NULL;
	pe_action_t *stonith_op = NULL;

	if(op == NULL) {
	op = data_set->stonith_action;
	}

	op_key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);

	stonith_op = lookup_singleton(data_set, op_key);
	if(stonith_op == NULL) {
	stonith_op = custom_action(NULL, op_key, CRM_OP_FENCE, node, TRUE, TRUE, data_set);

	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
	add_hash_param(stonith_op->meta, "stonith_action", op);

	if (pe__is_guest_or_remote_node(node)
	&& pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)) {
	/* Extra work to detect device changes on remotes
	*
	* We may do this for all nodes in the future, but for now
	* the pcmk__check_action_config() based stuff works fine.
	*/
	long max = 1024;
	long digests_all_offset = 0;
	long digests_secure_offset = 0;

	char *digests_all = calloc(max, sizeof(char));
	char *digests_secure = calloc(max, sizeof(char));
	GList *matches = find_unfencing_devices(data_set->resources, NULL);

	for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *match = gIter->data;
	const char *agent = g_hash_table_lookup(match->meta,
	XML_ATTR_TYPE);
	op_digest_cache_t *data = NULL;

	data = pe__compare_fencing_digest(match, agent, node, data_set);
	if(data->rc == RSC_DIGEST_ALL) {
	optional = FALSE;
	crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id);
	if (!pcmk__is_daemon && data_set->priv != NULL) {
	pcmk__output_t *out = data_set->priv;
	out->info(out, "notice: Unfencing %s (remote): because the definition of %s changed",
	node->details->uname, match->id);
	}
	}

	digests_all_offset += snprintf(
	digests_all+digests_all_offset, max-digests_all_offset,
	"%s:%s:%s,", match->id, agent, data->digest_all_calc);

	digests_secure_offset += snprintf(
	digests_secure+digests_secure_offset, max-digests_secure_offset,
	"%s:%s:%s,", match->id, agent, data->digest_secure_calc);
	}
	g_hash_table_insert(stonith_op->meta,
	strdup(XML_OP_ATTR_DIGESTS_ALL),
	digests_all);
	g_hash_table_insert(stonith_op->meta,
	strdup(XML_OP_ATTR_DIGESTS_SECURE),
	digests_secure);
	}

	} else {
	free(op_key);
	}

	if (data_set->priority_fencing_delay > 0

	/* It's a suitable case where `priority-fencing-delay` applies.
	* At least add `priority-fencing-delay` field as an indicator. */
	&& (priority_delay

	/* Re-calculate priority delay for the suitable case when
	* pe_fence_op() is called again by stage6() after node priority has
	* been actually calculated with native_add_running() */
	\|\| g_hash_table_lookup(stonith_op->meta,
	XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) {

	/* Add `priority-fencing-delay` to the fencing op even if it's 0 for
	* the targeting node. So that it takes precedence over any possible
	* `pcmk_delay_base/max`.
	*/
	char *delay_s = pcmk__itoa(node_priority_fencing_delay(node, data_set));

	g_hash_table_insert(stonith_op->meta,
	strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY),
	delay_s);
	}

	if(optional == FALSE && pe_can_fence(data_set, node)) {
	pe__clear_action_flags(stonith_op, pe_action_optional);
	pe_action_set_reason(stonith_op, reason, false);

	} else if(reason && stonith_op->reason == NULL) {
	stonith_op->reason = strdup(reason);
	}

	return stonith_op;
	}

	void
	trigger_unfencing(
	pe_resource_t * rsc, pe_node_t node, const char reason, pe_action_t dependency, pe_working_set_t data_set)
	{
	if (!pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)) {
	/* No resources require it */
	return;

	} else if ((rsc != NULL)
	&& !pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
	/* Wasn't a stonith device */
	return;

	} else if(node
	&& node->details->online
	&& node->details->unclean == FALSE
	&& node->details->shutdown == FALSE) {
	pe_action_t *unfence = pe_fence_op(node, "on", FALSE, reason, FALSE, data_set);

	if(dependency) {
	order_actions(unfence, dependency, pe_order_optional);
	}

	} else if(rsc) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
	trigger_unfencing(rsc, node, reason, dependency, data_set);
	}
	}
	}
	}

	gboolean
	add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref)
	{
	pe_tag_t *tag = NULL;
	GList *gIter = NULL;
	gboolean is_existing = FALSE;

	CRM_CHECK(tags && tag_name && obj_ref, return FALSE);

	tag = g_hash_table_lookup(tags, tag_name);
	if (tag == NULL) {
	tag = calloc(1, sizeof(pe_tag_t));
	if (tag == NULL) {
	return FALSE;
	}
	tag->id = strdup(tag_name);
	tag->refs = NULL;
	g_hash_table_insert(tags, strdup(tag_name), tag);
	}

	for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
	const char existing_ref = (const char ) gIter->data;

	if (pcmk__str_eq(existing_ref, obj_ref, pcmk__str_none)){
	is_existing = TRUE;
	break;
	}
	}

	if (is_existing == FALSE) {
	tag->refs = g_list_append(tag->refs, strdup(obj_ref));
	crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Create an action reason string based on the action itself
	*
	* \param[in] action Action to create reason string for
	* \param[in] flag Action flag that was cleared
	*
	* \return Newly allocated string suitable for use as action reason
	* \note It is the caller's responsibility to free() the result.
	*/
	char *
	pe__action2reason(pe_action_t *action, enum pe_action_flags flag)
	{
	const char *change = NULL;

	switch (flag) {
	case pe_action_runnable:
	case pe_action_migrate_runnable:
	change = "unrunnable";
	break;
	case pe_action_optional:
	change = "required";
	break;
	default:
	// Bug: caller passed unsupported flag
	CRM_CHECK(change != NULL, change = "");
	break;
	}
	return crm_strdup_printf("%s%s%s %s", change,
	(action->rsc == NULL)? "" : " ",
	(action->rsc == NULL)? "" : action->rsc->id,
	action->task);
	}

	void pe_action_set_reason(pe_action_t action, const char reason, bool overwrite)
	{
	if (action->reason != NULL && overwrite) {
	pe_rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
	action->uuid, action->reason, crm_str(reason));
	} else if (action->reason == NULL) {
	pe_rsc_trace(action->rsc, "Set %s reason to '%s'",
	action->uuid, crm_str(reason));
	} else {
	// crm_assert(action->reason != NULL && !overwrite);
	return;
	}

	pcmk__str_update(&action->reason, reason);
	}

	/*!
	* \internal
	* \brief Check whether shutdown has been requested for a node
	*
	* \param[in] node Node to check
	*
	* \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
	* \note This differs from simply using node->details->shutdown in that it can
	* be used before that has been determined (and in fact to determine it),
	* and it can also be used to distinguish requested shutdown from implicit
	* shutdown of remote nodes by virtue of their connection stopping.
	*/
	bool
	pe__shutdown_requested(pe_node_t *node)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);

	return !pcmk__str_eq(shutdown, "0", pcmk__str_null_matches);
	}

	/*!
	* \internal
	* \brief Update a data set's "recheck by" time
	*
	* \param[in] recheck Epoch time when recheck should happen
	* \param[in,out] data_set Current working set
	*/
	void
	pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set)
	{
	if ((recheck > get_effective_time(data_set))
	&& ((data_set->recheck_by == 0)
	\|\| (data_set->recheck_by > recheck))) {
	data_set->recheck_by = recheck;
	}
	}

	/*!
	* \internal
	* \brief Wrapper for pe_unpack_nvpairs() using a cluster working set
	*/
	void
	pe__unpack_dataset_nvpairs(xmlNode xml_obj, const char set_name,
	pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	pe_working_set_t *data_set)
	{
	crm_time_t *next_change = crm_time_new_undefined();

	pe_eval_nvpairs(data_set->input, xml_obj, set_name, rule_data, hash,
	always_first, overwrite, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, data_set);
	}
	crm_time_free(next_change);
	}

	bool
	pe__resource_is_disabled(pe_resource_t *rsc)
	{
	const char *target_role = NULL;

	CRM_CHECK(rsc != NULL, return false);
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	if (target_role) {
	enum rsc_role_e target_role_e = text2role(target_role);

	if ((target_role_e == RSC_ROLE_STOPPED)
	\|\| ((target_role_e == RSC_ROLE_UNPROMOTED)
	&& pcmk_is_set(uber_parent(rsc)->flags, pe_rsc_promotable))) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Create an action to clear a resource's history from CIB
	*
	* \param[in] rsc Resource to clear
	* \param[in] node Node to clear history on
	*
	* \return New action to clear resource history
	*/
	pe_action_t *
	pe__clear_resource_history(pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set)
	{
	char *key = NULL;

	CRM_ASSERT(rsc && node);
	key = pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0);
	return custom_action(rsc, key, CRM_OP_LRM_DELETE, node, FALSE, TRUE,
	data_set);
	}

	bool
	pe__rsc_running_on_any(pe_resource_t rsc, GList node_list)
	{
	for (GList *ele = rsc->running_on; ele; ele = ele->next) {
	pe_node_t node = (pe_node_t ) ele->data;
	if (pcmk__str_in_list(node->details->uname, node_list,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	return true;
	}
	}

	return false;
	}

	bool
	pcmk__rsc_filtered_by_node(pe_resource_t rsc, GList only_node)
	{
	return (rsc->fns->active(rsc, FALSE) && !pe__rsc_running_on_any(rsc, only_node));
	}

	GList *
	pe__filter_rsc_list(GList rscs, GList filter)
	{
	GList *retval = NULL;

	for (GList *gIter = rscs; gIter; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* I think the second condition is safe here for all callers of this
	* function. If not, it needs to move into pe__node_text.
	*/
	if (pcmk__str_in_list(rsc_printable_id(rsc), filter, pcmk__str_star_matches) \|\|
	(rsc->parent && pcmk__str_in_list(rsc_printable_id(rsc->parent), filter, pcmk__str_star_matches))) {
	retval = g_list_prepend(retval, rsc);
	}
	}

	return retval;
	}

	GList *
	pe__build_node_name_list(pe_working_set_t data_set, const char s) {
	GList *nodes = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	/* Nothing was given so return a list of all node names. Or, '*' was
	* given. This would normally fall into the pe__unames_with_tag branch
	* where it will return an empty list. Catch it here instead.
	*/
	nodes = g_list_prepend(nodes, strdup("*"));
	} else {
	pe_node_t *node = pe_find_node(data_set->nodes, s);

	if (node) {
	/* The given string was a valid uname for a node. Return a
	* singleton list containing just that uname.
	*/
	nodes = g_list_prepend(nodes, strdup(s));
	} else {
	/* The given string was not a valid uname. It's either a tag or
	* it's a typo or something. In the first case, we'll return a
	* list of all the unames of the nodes with the given tag. In the
	* second case, we'll return a NULL pointer and nothing will
	* get displayed.
	*/
	nodes = pe__unames_with_tag(data_set, s);
	}
	}

	return nodes;
	}

	GList *
	pe__build_rsc_list(pe_working_set_t data_set, const char s) {
	GList *resources = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	resources = g_list_prepend(resources, strdup("*"));
	} else {
	pe_resource_t *rsc = pe_find_resource_with_flags(data_set->resources, s,
	pe_find_renamed\|pe_find_any);

	if (rsc) {
	/* A colon in the name we were given means we're being asked to filter
	* on a specific instance of a cloned resource. Put that exact string
	* into the filter list. Otherwise, use the printable ID of whatever
	* resource was found that matches what was asked for.
	*/
	if (strstr(s, ":") != NULL) {
	resources = g_list_prepend(resources, strdup(rsc->id));
	} else {
	resources = g_list_prepend(resources, strdup(rsc_printable_id(rsc)));
	}
	} else {
	/* The given string was not a valid resource name. It's either
	* a tag or it's a typo or something. See build_uname_list for
	* more detail.
	*/
	resources = pe__rscs_with_tag(data_set, s);
	}
	}

	return resources;
	}

	xmlNode *
	pe__failed_probe_for_rsc(pe_resource_t rsc, const char name)
	{
	pe_resource_t *parent = uber_parent(rsc);
	const char *rsc_id = rsc->id;

	if (rsc->variant == pe_clone) {
	rsc_id = pe__clone_child_id(rsc);
	} else if (parent->variant == pe_clone) {
	rsc_id = pe__clone_child_id(parent);
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(rsc->cluster->failed); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	const char *value = NULL;
	char *op_id = NULL;

	/* This resource operation is not a failed probe. */
	if (!pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	/* This resource operation was not run on the given node. Note that if name is
	* NULL, this will always succeed.
	*/
	value = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);
	if (value == NULL \|\| !pcmk__str_eq(value, name, pcmk__str_casei\|pcmk__str_null_matches)) {
	continue;
	}

	/* This resource operation has no operation_key. */
	value = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
	if (!parse_op_key(value ? value : ID(xml_op), &op_id, NULL, NULL)) {
	continue;
	}

	/* This resource operation's ID does not match the rsc_id we are looking for. */
	if (!pcmk__str_eq(op_id, rsc_id, pcmk__str_none)) {
	free(op_id);
	continue;
	}

	free(op_id);
	return xml_op;
	}

	return NULL;
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Jul 21, 2:51 AM (4 h, 39 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2082942
Default Alt Text: (731 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions