No OneTemporary
Actions

Size

655 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/crm/common/logging_internal.h b/include/crm/common/logging_internal.h
	index 981ddf379d..b6db238e8d 100644
	--- a/include/crm/common/logging_internal.h
	+++ b/include/crm/common/logging_internal.h
	@@ -1,185 +1,183 @@
	/*
	* Copyright 2015-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#ifdef __cplusplus
	extern "C" {
	#endif

	#ifndef PCMK__LOGGING_INTERNAL_H
	# define PCMK__LOGGING_INTERNAL_H

	# include <glib.h>

	# include <crm/common/logging.h>
	# include <crm/common/output_internal.h>

	typedef void (pcmk__config_error_func) (void ctx, const char *msg, ...);
	typedef void (pcmk__config_warning_func) (void ctx, const char *msg, ...);

	extern pcmk__config_error_func pcmk__config_error_handler;
	extern pcmk__config_warning_func pcmk__config_warning_handler;

	extern void *pcmk__config_error_context;
	extern void *pcmk__config_warning_context;

	void pcmk__set_config_error_handler(pcmk__config_error_func error_handler, void *error_context);
	void pcmk__set_config_warning_handler(pcmk__config_warning_func warning_handler, void *warning_context);

	/*!
	* \internal
	- * \brief Log a configuration error
	+ * \brief Log an error and make crm_verify return failure status
	*
	- * \param[in] fmt printf(3)-style format string
	- * \param[in] ... Arguments for format string
	+ * \param[in] fmt... printf(3)-style format string and arguments
	*/
	# define pcmk__config_err(fmt...) do { \
	crm_config_error = TRUE; \
	if (pcmk__config_error_handler == NULL) { \
	crm_err(fmt); \
	} else { \
	pcmk__config_error_handler(pcmk__config_error_context, fmt); \
	} \
	} while (0)

	/*!
	* \internal
	- * \brief Log a configuration warning
	+ * \brief Log a warning and make crm_verify return failure status
	*
	- * \param[in] fmt printf(3)-style format string
	- * \param[in] ... Arguments for format string
	+ * \param[in] fmt... printf(3)-style format string and arguments
	*/
	# define pcmk__config_warn(fmt...) do { \
	crm_config_warning = TRUE; \
	if (pcmk__config_warning_handler == NULL) { \
	crm_warn(fmt); \
	} else { \
	pcmk__config_warning_handler(pcmk__config_warning_context, fmt); \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Execute code depending on whether trace logging is enabled
	*
	* This is similar to \p do_crm_log_unlikely() except instead of logging, it
	* selects one of two code blocks to execute.
	*
	* \param[in] if_action Code block to execute if trace logging is enabled
	* \param[in] else_action Code block to execute if trace logging is not enabled
	*
	* \note Neither \p if_action nor \p else_action can contain a \p break or
	* \p continue statement.
	*/
	# define pcmk__if_tracing(if_action, else_action) do { \
	static struct qb_log_callsite *trace_cs = NULL; \
	\
	if (trace_cs == NULL) { \
	trace_cs = qb_log_callsite_get(__func__, __FILE__, \
	"if_tracing", LOG_TRACE, \
	__LINE__, crm_trace_nonlog); \
	} \
	if (crm_is_callsite_active(trace_cs, LOG_TRACE, \
	crm_trace_nonlog)) { \
	if_action; \
	} else { \
	else_action; \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Log XML changes line-by-line in a formatted fashion
	*
	* \param[in] level Priority at which to log the messages
	* \param[in] xml XML to log
	*
	* \note This does nothing when \p level is \c LOG_STDOUT.
	*/
	#define pcmk__log_xml_changes(level, xml) do { \
	uint8_t _level = pcmk__clip_log_level(level); \
	static struct qb_log_callsite *xml_cs = NULL; \
	\
	switch (_level) { \
	case LOG_STDOUT: \
	case LOG_NEVER: \
	break; \
	default: \
	if (xml_cs == NULL) { \
	xml_cs = qb_log_callsite_get(__func__, __FILE__, \
	"xml-changes", _level, \
	__LINE__, 0); \
	} \
	if (crm_is_callsite_active(xml_cs, _level, 0)) { \
	pcmk__log_xml_changes_as(__FILE__, __func__, __LINE__, \
	0, _level, xml); \
	} \
	break; \
	} \
	} while(0)

	/*!
	* \internal
	* \brief Log an XML patchset line-by-line in a formatted fashion
	*
	* \param[in] level Priority at which to log the messages
	* \param[in] patchset XML patchset to log
	*
	* \note This does nothing when \p level is \c LOG_STDOUT.
	*/
	#define pcmk__log_xml_patchset(level, patchset) do { \
	uint8_t _level = pcmk__clip_log_level(level); \
	static struct qb_log_callsite *xml_cs = NULL; \
	\
	switch (_level) { \
	case LOG_STDOUT: \
	case LOG_NEVER: \
	break; \
	default: \
	if (xml_cs == NULL) { \
	xml_cs = qb_log_callsite_get(__func__, __FILE__, \
	"xml-patchset", _level, \
	__LINE__, 0); \
	} \
	if (crm_is_callsite_active(xml_cs, _level, 0)) { \
	pcmk__log_xml_patchset_as(__FILE__, __func__, __LINE__, \
	0, _level, patchset); \
	} \
	break; \
	} \
	} while(0)

	void pcmk__log_xml_changes_as(const char file, const char function,
	uint32_t line, uint32_t tags, uint8_t level,
	const xmlNode *xml);

	void pcmk__log_xml_patchset_as(const char file, const char function,
	uint32_t line, uint32_t tags, uint8_t level,
	const xmlNode *patchset);

	/*!
	* \internal
	* \brief Initialize logging for command line tools
	*
	* \param[in] name The name of the program
	* \param[in] verbosity How verbose to be in logging
	*
	* \note \p verbosity is not the same as the logging level (LOG_ERR, etc.).
	*/
	void pcmk__cli_init_logging(const char *name, unsigned int verbosity);

	int pcmk__add_logfile(const char *filename);
	void pcmk__add_logfiles(gchar *log_files, pcmk__output_t out);

	void pcmk__free_common_logger(void);

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c
	index 4a5487e72f..1645277e09 100644
	--- a/lib/pacemaker/pcmk_sched_colocation.c
	+++ b/lib/pacemaker/pcmk_sched_colocation.c
	@@ -1,1908 +1,1909 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include "crm/common/util.h"
	#include "crm/common/xml_internal.h"
	#include "crm/msg_xml.h"
	#include "libpacemaker_private.h"

	// Used to temporarily mark a node as unusable
	#define INFINITY_HACK (INFINITY * -100)

	/*!
	* \internal
	* \brief Compare two colocations according to priority
	*
	* Compare two colocations according to the order in which they should be
	* considered, based on either their dependent resources or their primary
	* resources -- preferring (in order):
	* * Colocation that is not \c NULL
	* * Colocation whose resource has higher priority
	* * Colocation whose resource is of a higher-level variant
	* (bundle > clone > group > primitive)
	* * Colocation whose resource is promotable, if both are clones
	* * Colocation whose resource has lower ID in lexicographic order
	*
	* \param[in] colocation1 First colocation to compare
	* \param[in] colocation2 Second colocation to compare
	* \param[in] dependent If \c true, compare colocations by dependent
	* priority; otherwise compare them by primary priority
	*
	* \return A negative number if \p colocation1 should be considered first,
	* a positive number if \p colocation2 should be considered first,
	* or 0 if order doesn't matter
	*/
	static gint
	cmp_colocation_priority(const pcmk__colocation_t *colocation1,
	const pcmk__colocation_t *colocation2, bool dependent)
	{
	const pcmk_resource_t *rsc1 = NULL;
	const pcmk_resource_t *rsc2 = NULL;

	if (colocation1 == NULL) {
	return 1;
	}
	if (colocation2 == NULL) {
	return -1;
	}

	if (dependent) {
	rsc1 = colocation1->dependent;
	rsc2 = colocation2->dependent;
	CRM_ASSERT(colocation1->primary != NULL);
	} else {
	rsc1 = colocation1->primary;
	rsc2 = colocation2->primary;
	CRM_ASSERT(colocation1->dependent != NULL);
	}
	CRM_ASSERT((rsc1 != NULL) && (rsc2 != NULL));

	if (rsc1->priority > rsc2->priority) {
	return -1;
	}
	if (rsc1->priority < rsc2->priority) {
	return 1;
	}

	// Process clones before primitives and groups
	if (rsc1->variant > rsc2->variant) {
	return -1;
	}
	if (rsc1->variant < rsc2->variant) {
	return 1;
	}

	/* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
	* clones (probably unnecessary, but avoids having to update regression
	* tests)
	*/
	if (rsc1->variant == pcmk_rsc_variant_clone) {
	if (pcmk_is_set(rsc1->flags, pcmk_rsc_promotable)
	&& !pcmk_is_set(rsc2->flags, pcmk_rsc_promotable)) {
	return -1;
	}
	if (!pcmk_is_set(rsc1->flags, pcmk_rsc_promotable)
	&& pcmk_is_set(rsc2->flags, pcmk_rsc_promotable)) {
	return 1;
	}
	}

	return strcmp(rsc1->id, rsc2->id);
	}

	/*!
	* \internal
	* \brief Compare two colocations according to priority based on dependents
	*
	* Compare two colocations according to the order in which they should be
	* considered, based on their dependent resources -- preferring (in order):
	* * Colocation that is not \c NULL
	* * Colocation whose resource has higher priority
	* * Colocation whose resource is of a higher-level variant
	* (bundle > clone > group > primitive)
	* * Colocation whose resource is promotable, if both are clones
	* * Colocation whose resource has lower ID in lexicographic order
	*
	* \param[in] a First colocation to compare
	* \param[in] b Second colocation to compare
	*
	* \return A negative number if \p a should be considered first,
	* a positive number if \p b should be considered first,
	* or 0 if order doesn't matter
	*/
	static gint
	cmp_dependent_priority(gconstpointer a, gconstpointer b)
	{
	return cmp_colocation_priority(a, b, true);
	}

	/*!
	* \internal
	* \brief Compare two colocations according to priority based on primaries
	*
	* Compare two colocations according to the order in which they should be
	* considered, based on their primary resources -- preferring (in order):
	* * Colocation that is not \c NULL
	* * Colocation whose primary has higher priority
	* * Colocation whose primary is of a higher-level variant
	* (bundle > clone > group > primitive)
	* * Colocation whose primary is promotable, if both are clones
	* * Colocation whose primary has lower ID in lexicographic order
	*
	* \param[in] a First colocation to compare
	* \param[in] b Second colocation to compare
	*
	* \return A negative number if \p a should be considered first,
	* a positive number if \p b should be considered first,
	* or 0 if order doesn't matter
	*/
	static gint
	cmp_primary_priority(gconstpointer a, gconstpointer b)
	{
	return cmp_colocation_priority(a, b, false);
	}

	/*!
	* \internal
	* \brief Add a "this with" colocation constraint to a sorted list
	*
	* \param[in,out] list List of constraints to add \p colocation to
	* \param[in] colocation Colocation constraint to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The list will be sorted using cmp_primary_priority().
	*/
	void
	pcmk__add_this_with(GList *list, const pcmk__colocation_t colocation,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (colocation != NULL) && (rsc != NULL));

	pcmk__rsc_trace(rsc,
	"Adding colocation %s (%s with %s using %s @%s) to "
	"'this with' list for %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, colocation->node_attribute,
	pcmk_readable_score(colocation->score), rsc->id);
	list = g_list_insert_sorted(list, (gpointer) colocation,
	cmp_primary_priority);
	}

	/*!
	* \internal
	* \brief Add a list of "this with" colocation constraints to a list
	*
	* \param[in,out] list List of constraints to add \p addition to
	* \param[in] addition List of colocation constraints to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The lists must be pre-sorted by cmp_primary_priority().
	*/
	void
	pcmk__add_this_with_list(GList *list, GList addition,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (rsc != NULL));

	pcmk__if_tracing(
	{}, // Always add each colocation individually if tracing
	{
	if (*list == NULL) {
	// Trivial case for efficiency if not tracing
	*list = g_list_copy(addition);
	return;
	}
	}
	);

	for (const GList *iter = addition; iter != NULL; iter = iter->next) {
	pcmk__add_this_with(list, addition->data, rsc);
	}
	}

	/*!
	* \internal
	* \brief Add a "with this" colocation constraint to a sorted list
	*
	* \param[in,out] list List of constraints to add \p colocation to
	* \param[in] colocation Colocation constraint to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The list will be sorted using cmp_dependent_priority().
	*/
	void
	pcmk__add_with_this(GList *list, const pcmk__colocation_t colocation,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (colocation != NULL) && (rsc != NULL));

	pcmk__rsc_trace(rsc,
	"Adding colocation %s (%s with %s using %s @%s) to "
	"'with this' list for %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, colocation->node_attribute,
	pcmk_readable_score(colocation->score), rsc->id);
	list = g_list_insert_sorted(list, (gpointer) colocation,
	cmp_dependent_priority);
	}

	/*!
	* \internal
	* \brief Add a list of "with this" colocation constraints to a list
	*
	* \param[in,out] list List of constraints to add \p addition to
	* \param[in] addition List of colocation constraints to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The lists must be pre-sorted by cmp_dependent_priority().
	*/
	void
	pcmk__add_with_this_list(GList *list, GList addition,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (rsc != NULL));

	pcmk__if_tracing(
	{}, // Always add each colocation individually if tracing
	{
	if (*list == NULL) {
	// Trivial case for efficiency if not tracing
	*list = g_list_copy(addition);
	return;
	}
	}
	);

	for (const GList *iter = addition; iter != NULL; iter = iter->next) {
	pcmk__add_with_this(list, addition->data, rsc);
	}
	}

	/*!
	* \internal
	* \brief Add orderings necessary for an anti-colocation constraint
	*
	* \param[in,out] first_rsc One resource in an anti-colocation
	* \param[in] first_role Anti-colocation role of \p first_rsc
	* \param[in] then_rsc Other resource in the anti-colocation
	* \param[in] then_role Anti-colocation role of \p then_rsc
	*/
	static void
	anti_colocation_order(pcmk_resource_t *first_rsc, int first_role,
	pcmk_resource_t *then_rsc, int then_role)
	{
	const char *first_tasks[] = { NULL, NULL };
	const char *then_tasks[] = { NULL, NULL };

	/* Actions to make first_rsc lose first_role */
	if (first_role == pcmk_role_promoted) {
	first_tasks[0] = PCMK_ACTION_DEMOTE;

	} else {
	first_tasks[0] = PCMK_ACTION_STOP;

	if (first_role == pcmk_role_unpromoted) {
	first_tasks[1] = PCMK_ACTION_PROMOTE;
	}
	}

	/* Actions to make then_rsc gain then_role */
	if (then_role == pcmk_role_promoted) {
	then_tasks[0] = PCMK_ACTION_PROMOTE;

	} else {
	then_tasks[0] = PCMK_ACTION_START;

	if (then_role == pcmk_role_unpromoted) {
	then_tasks[1] = PCMK_ACTION_DEMOTE;
	}
	}

	for (int first_lpc = 0;
	(first_lpc <= 1) && (first_tasks[first_lpc] != NULL); first_lpc++) {

	for (int then_lpc = 0;
	(then_lpc <= 1) && (then_tasks[then_lpc] != NULL); then_lpc++) {

	pcmk__order_resource_actions(first_rsc, first_tasks[first_lpc],
	then_rsc, then_tasks[then_lpc],
	pcmk__ar_if_required_on_same_node);
	}
	}
	}

	/*!
	* \internal
	* \brief Add a new colocation constraint to scheduler data
	*
	* \param[in] id XML ID for this constraint
	* \param[in] node_attr Colocate by this attribute (NULL for #uname)
	* \param[in] score Constraint score
	* \param[in,out] dependent Resource to be colocated
	* \param[in,out] primary Resource to colocate \p dependent with
	* \param[in] dependent_role Current role of \p dependent
	* \param[in] primary_role Current role of \p primary
	* \param[in] flags Group of enum pcmk__coloc_flags
	*/
	void
	pcmk__new_colocation(const char id, const char node_attr, int score,
	pcmk_resource_t dependent, pcmk_resource_t primary,
	const char dependent_role, const char primary_role,
	uint32_t flags)
	{
	pcmk__colocation_t *new_con = NULL;

	CRM_CHECK(id != NULL, return);

	if ((dependent == NULL) \|\| (primary == NULL)) {
	pcmk__config_err("Ignoring colocation '%s' because resource "
	"does not exist", id);
	return;
	}

	if (score == 0) {
	pcmk__rsc_trace(dependent,
	"Ignoring colocation '%s' (%s with %s) because score is 0",
	id, dependent->id, primary->id);
	return;
	}

	new_con = calloc(1, sizeof(pcmk__colocation_t));
	CRM_ASSERT(new_con != NULL);

	if (pcmk__str_eq(dependent_role, PCMK__ROLE_STARTED,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	dependent_role = PCMK__ROLE_UNKNOWN;
	}

	if (pcmk__str_eq(primary_role, PCMK__ROLE_STARTED,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	primary_role = PCMK__ROLE_UNKNOWN;
	}

	new_con->id = id;
	new_con->dependent = dependent;
	new_con->primary = primary;
	new_con->score = score;
	new_con->dependent_role = text2role(dependent_role);
	new_con->primary_role = text2role(primary_role);
	new_con->node_attribute = pcmk__s(node_attr, CRM_ATTR_UNAME);
	new_con->flags = flags;

	pcmk__add_this_with(&(dependent->rsc_cons), new_con, dependent);
	pcmk__add_with_this(&(primary->rsc_cons_lhs), new_con, primary);

	dependent->cluster->colocation_constraints = g_list_prepend(
	dependent->cluster->colocation_constraints, new_con);

	if (score <= -INFINITY) {
	anti_colocation_order(dependent, new_con->dependent_role, primary,
	new_con->primary_role);
	anti_colocation_order(primary, new_con->primary_role, dependent,
	new_con->dependent_role);
	}
	}

	/*!
	* \internal
	* \brief Return the boolean influence corresponding to configuration
	*
	* \param[in] coloc_id Colocation XML ID (for error logging)
	* \param[in] rsc Resource involved in constraint (for default)
	* \param[in] influence_s String value of influence option
	*
	* \return pcmk__coloc_influence if string evaluates true, or string is NULL or
	* invalid and resource's critical option evaluates true, otherwise
	* pcmk__coloc_none
	*/
	static uint32_t
	unpack_influence(const char coloc_id, const pcmk_resource_t rsc,
	const char *influence_s)
	{
	if (influence_s != NULL) {
	int influence_i = 0;

	if (crm_str_to_boolean(influence_s, &influence_i) < 0) {
	pcmk__config_err("Constraint '%s' has invalid value for "
	XML_COLOC_ATTR_INFLUENCE " (using default)",
	coloc_id);
	} else {
	return (influence_i == 0)? pcmk__coloc_none : pcmk__coloc_influence;
	}
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_critical)) {
	return pcmk__coloc_influence;
	}
	return pcmk__coloc_none;
	}

	static void
	unpack_colocation_set(xmlNode set, int score, const char coloc_id,
	const char influence_s, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_rsc = NULL;
	pcmk_resource_t *other = NULL;
	pcmk_resource_t *resource = NULL;
	const char *set_id = ID(set);
	const char *role = crm_element_value(set, "role");
	bool with_previous = false;
	int local_score = score;
	bool sequential = false;
	uint32_t flags = pcmk__coloc_none;
	const char *xml_rsc_id = NULL;
	const char *score_s = crm_element_value(set, XML_RULE_ATTR_SCORE);

	if (score_s) {
	local_score = char2score(score_s);
	}
	if (local_score == 0) {
	crm_trace("Ignoring colocation '%s' for set '%s' because score is 0",
	coloc_id, set_id);
	return;
	}

	/* @COMPAT The deprecated "ordering" attribute specifies whether resources
	* in a positive-score set are colocated with the previous or next resource.
	*/
	if (pcmk__str_eq(crm_element_value(set, "ordering"), "group",
	pcmk__str_null_matches\|pcmk__str_casei)) {
	with_previous = true;
	} else {
	pe_warn_once(pcmk__wo_set_ordering,
	"Support for 'ordering' other than 'group' in "
	XML_CONS_TAG_RSC_SET " (such as %s) is deprecated and "
	"will be removed in a future release", set_id);
	}

	if ((pcmk__xe_get_bool_attr(set, "sequential", &sequential) == pcmk_rc_ok)
	&& !sequential) {
	return;
	}

	if (local_score > 0) {
	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xml_rsc_id = ID(xml_rsc);
	resource = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (resource == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring %s and later resources in set %s: "
	"No such resource", xml_rsc_id, set_id);
	return;
	}
	if (other != NULL) {
	flags = pcmk__coloc_explicit
	\| unpack_influence(coloc_id, resource, influence_s);
	if (with_previous) {
	pcmk__rsc_trace(resource, "Colocating %s with %s in set %s",
	resource->id, other->id, set_id);
	pcmk__new_colocation(set_id, NULL, local_score, resource,
	other, role, role, flags);
	} else {
	pcmk__rsc_trace(resource, "Colocating %s with %s in set %s",
	other->id, resource->id, set_id);
	pcmk__new_colocation(set_id, NULL, local_score, other,
	resource, role, role, flags);
	}
	}
	other = resource;
	}

	} else {
	/* Anti-colocating with every prior resource is
	* the only way to ensure the intuitive result
	* (i.e. that no one in the set can run with anyone else in the set)
	*/

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xmlNode *xml_rsc_with = NULL;

	xml_rsc_id = ID(xml_rsc);
	resource = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (resource == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring %s and later resources in set %s: "
	"No such resource", xml_rsc_id, set_id);
	return;
	}
	flags = pcmk__coloc_explicit
	\| unpack_influence(coloc_id, resource, influence_s);
	for (xml_rsc_with = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc_with != NULL;
	xml_rsc_with = crm_next_same_xml(xml_rsc_with)) {

	xml_rsc_id = ID(xml_rsc_with);
	if (pcmk__str_eq(resource->id, xml_rsc_id, pcmk__str_none)) {
	break;
	}
	other = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	CRM_ASSERT(other != NULL); // We already processed it
	pcmk__new_colocation(set_id, NULL, local_score,
	resource, other, role, role, flags);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Colocate two resource sets relative to each other
	*
	* \param[in] id Colocation XML ID
	* \param[in] set1 Dependent set
	* \param[in] set2 Primary set
	* \param[in] score Colocation score
	* \param[in] influence_s Value of colocation's "influence" attribute
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	colocate_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	int score, const char *influence_s,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *xml_rsc = NULL;
	pcmk_resource_t *rsc_1 = NULL;
	pcmk_resource_t *rsc_2 = NULL;

	const char *xml_rsc_id = NULL;
	const char *role_1 = crm_element_value(set1, "role");
	const char *role_2 = crm_element_value(set2, "role");

	int rc = pcmk_rc_ok;
	bool sequential = false;
	uint32_t flags = pcmk__coloc_none;

	if (score == 0) {
	crm_trace("Ignoring colocation '%s' between sets %s and %s "
	"because score is 0", id, ID(set1), ID(set2));
	return;
	}

	rc = pcmk__xe_get_bool_attr(set1, "sequential", &sequential);
	if ((rc != pcmk_rc_ok) \|\| sequential) {
	// Get the first one
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	xml_rsc_id = ID(xml_rsc);
	rsc_1 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_1 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s with set %s "
	"because first resource %s not found",
	ID(set1), ID(set2), xml_rsc_id);
	return;
	}
	}
	}

	rc = pcmk__xe_get_bool_attr(set2, "sequential", &sequential);
	if ((rc != pcmk_rc_ok) \|\| sequential) {
	// Get the last one
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xml_rsc_id = ID(xml_rsc);
	}
	rsc_2 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_2 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s with set %s "
	"because last resource %s not found",
	ID(set1), ID(set2), xml_rsc_id);
	return;
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) { // Both sets are sequential
	flags = pcmk__coloc_explicit \| unpack_influence(id, rsc_1, influence_s);
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2,
	flags);

	} else if (rsc_1 != NULL) { // Only set1 is sequential
	flags = pcmk__coloc_explicit \| unpack_influence(id, rsc_1, influence_s);
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xml_rsc_id = ID(xml_rsc);
	rsc_2 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_2 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring set %s colocation with resource %s "
	"in set %s: No such resource",
	ID(set1), xml_rsc_id, ID(set2));
	continue;
	}
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	role_2, flags);
	}

	} else if (rsc_2 != NULL) { // Only set2 is sequential
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xml_rsc_id = ID(xml_rsc);
	rsc_1 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_1 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s resource %s "
	"with set %s: No such resource",
	ID(set1), xml_rsc_id, ID(set2));
	continue;
	}
	flags = pcmk__coloc_explicit
	\| unpack_influence(id, rsc_1, influence_s);
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	role_2, flags);
	}

	} else { // Neither set is sequential
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xmlNode *xml_rsc_2 = NULL;

	xml_rsc_id = ID(xml_rsc);
	rsc_1 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_1 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s resource %s "
	"with set %s: No such resource",
	ID(set1), xml_rsc_id, ID(set2));
	continue;
	}

	flags = pcmk__coloc_explicit
	\| unpack_influence(id, rsc_1, influence_s);
	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL;
	xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	xml_rsc_id = ID(xml_rsc_2);
	rsc_2 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_2 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s resource "
	"%s with set %s resource %s: No such "
	"resource", ID(set1), ID(xml_rsc),
	ID(set2), xml_rsc_id);
	continue;
	}
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2,
	role_1, role_2, flags);
	}
	}
	}
	}

	static void
	unpack_simple_colocation(xmlNode xml_obj, const char id,
	const char influence_s, pcmk_scheduler_t scheduler)
	{
	int score_i = 0;
	uint32_t flags = pcmk__coloc_none;

	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	const char *dependent_id = crm_element_value(xml_obj,
	XML_COLOC_ATTR_SOURCE);
	const char *primary_id = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	const char *dependent_role = crm_element_value(xml_obj,
	XML_COLOC_ATTR_SOURCE_ROLE);
	const char *primary_role = crm_element_value(xml_obj,
	XML_COLOC_ATTR_TARGET_ROLE);
	const char *attr = crm_element_value(xml_obj, XML_COLOC_ATTR_NODE_ATTR);

	const char *primary_instance = NULL;
	const char *dependent_instance = NULL;
	pcmk_resource_t *primary = NULL;
	pcmk_resource_t *dependent = NULL;

	primary = pcmk__find_constraint_resource(scheduler->resources, primary_id);
	dependent = pcmk__find_constraint_resource(scheduler->resources,
	dependent_id);

	// @COMPAT: Deprecated since 2.1.5
	primary_instance = crm_element_value(xml_obj,
	XML_COLOC_ATTR_TARGET_INSTANCE);
	dependent_instance = crm_element_value(xml_obj,
	XML_COLOC_ATTR_SOURCE_INSTANCE);
	if (dependent_instance != NULL) {
	pe_warn_once(pcmk__wo_coloc_inst,
	"Support for " XML_COLOC_ATTR_SOURCE_INSTANCE " is "
	"deprecated and will be removed in a future release.");
	}
	if (primary_instance != NULL) {
	pe_warn_once(pcmk__wo_coloc_inst,
	"Support for " XML_COLOC_ATTR_TARGET_INSTANCE " is "
	"deprecated and will be removed in a future release.");
	}

	if (dependent == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, dependent_id);
	return;

	} else if (primary == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, primary_id);
	return;

	} else if ((dependent_instance != NULL) && !pe_rsc_is_clone(dependent)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	id, dependent_id, dependent_instance);
	return;

	} else if ((primary_instance != NULL) && !pe_rsc_is_clone(primary)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	id, primary_id, primary_instance);
	return;
	}

	if (dependent_instance != NULL) {
	dependent = find_clone_instance(dependent, dependent_instance);
	if (dependent == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	id, dependent_id, dependent_instance);
	return;
	}
	}

	if (primary_instance != NULL) {
	primary = find_clone_instance(primary, primary_instance);
	if (primary == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", id, primary_id, primary_instance);
	return;
	}
	}

	if (pcmk__xe_attr_is_true(xml_obj, XML_CONS_ATTR_SYMMETRICAL)) {
	pcmk__config_warn("The colocation constraint '"
	XML_CONS_ATTR_SYMMETRICAL
	"' attribute has been removed");
	}

	if (score) {
	score_i = char2score(score);
	}

	flags = pcmk__coloc_explicit \| unpack_influence(id, dependent, influence_s);
	pcmk__new_colocation(id, attr, score_i, dependent, primary,
	dependent_role, primary_role, flags);
	}

	// \return Standard Pacemaker return code
	static int
	unpack_colocation_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	const char *id = NULL;
	const char *dependent_id = NULL;
	const char *primary_id = NULL;
	const char *dependent_role = NULL;
	const char *primary_role = NULL;

	pcmk_resource_t *dependent = NULL;
	pcmk_resource_t *primary = NULL;

	pcmk_tag_t *dependent_tag = NULL;
	pcmk_tag_t *primary_tag = NULL;

	xmlNode *dependent_set = NULL;
	xmlNode *primary_set = NULL;
	bool any_sets = false;

	*expanded_xml = NULL;

	CRM_CHECK(xml_obj != NULL, return EINVAL);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	return pcmk_rc_ok;
	}

	dependent_id = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	primary_id = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	if ((dependent_id == NULL) \|\| (primary_id == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, dependent_id, &dependent,
	&dependent_tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, dependent_id);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, primary_id, &primary,
	&primary_tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, primary_id);
	return pcmk_rc_unpack_error;
	}

	if ((dependent != NULL) && (primary != NULL)) {
	/* Neither side references any template/tag. */
	return pcmk_rc_ok;
	}

	if ((dependent_tag != NULL) && (primary_tag != NULL)) {
	// A colocation constraint between two templates/tags makes no sense
	pcmk__config_err("Ignoring constraint '%s' because two templates or "
	"tags cannot be colocated", id);
	return pcmk_rc_unpack_error;
	}

	dependent_role = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
	primary_role = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);

	*expanded_xml = copy_xml(xml_obj);

	// Convert dependent's template/tag reference into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &dependent_set, XML_COLOC_ATTR_SOURCE,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (dependent_set != NULL) {
	if (dependent_role != NULL) {
	// Move "rsc-role" into converted resource_set as "role"
	crm_xml_add(dependent_set, "role", dependent_role);
	xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_SOURCE_ROLE);
	}
	any_sets = true;
	}

	// Convert primary's template/tag reference into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &primary_set, XML_COLOC_ATTR_TARGET,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (primary_set != NULL) {
	if (primary_role != NULL) {
	// Move "with-rsc-role" into converted resource_set as "role"
	crm_xml_add(primary_set, "role", primary_role);
	xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_TARGET_ROLE);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Parse a colocation constraint from XML into scheduler data
	*
	* \param[in,out] xml_obj Colocation constraint XML to unpack
	* \param[in,out] scheduler Scheduler data to add constraint to
	*/
	void
	pcmk__unpack_colocation(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	int score_i = 0;
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *score = NULL;
	const char *influence_s = NULL;

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_DEPEND
	" without " CRM_ATTR_ID);
	return;
	}

	if (unpack_colocation_tags(xml_obj, &expanded_xml,
	scheduler) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	if (score != NULL) {
	score_i = char2score(score);
	}
	influence_s = crm_element_value(xml_obj, XML_COLOC_ATTR_INFLUENCE);

	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	set = crm_next_same_xml(set)) {

	set = expand_idref(set, scheduler->input);
	if (set == NULL) { // Configuration error, message already logged
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if (pcmk__str_empty(ID(set))) {
	pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_SET
	" without " CRM_ATTR_ID);
	continue;
	}
	unpack_colocation_set(set, score_i, id, influence_s, scheduler);

	if (last != NULL) {
	colocate_rsc_sets(id, last, set, score_i, influence_s, scheduler);
	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	if (last == NULL) {
	unpack_simple_colocation(xml_obj, id, influence_s, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Make actions of a given type unrunnable for a given resource
	*
	* \param[in,out] rsc Resource whose actions should be blocked
	* \param[in] task Name of action to block
	* \param[in] reason Unrunnable start action causing the block
	*/
	static void
	mark_action_blocked(pcmk_resource_t rsc, const char task,
	const pcmk_resource_t *reason)
	{
	GList *iter = NULL;
	char *reason_text = crm_strdup_printf("colocation with %s", reason->id);

	for (iter = rsc->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t *action = iter->data;

	if (pcmk_is_set(action->flags, pcmk_action_runnable)
	&& pcmk__str_eq(action->task, task, pcmk__str_none)) {

	pe__clear_action_flags(action, pcmk_action_runnable);
	pe_action_set_reason(action, reason_text, false);
	pcmk__block_colocation_dependents(action);
	pcmk__update_action_for_orderings(action, rsc->cluster);
	}
	}

	// If parent resource can't perform an action, neither can any children
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	mark_action_blocked((pcmk_resource_t *) (iter->data), task, reason);
	}
	free(reason_text);
	}

	/*!
	* \internal
	* \brief If an action is unrunnable, block any relevant dependent actions
	*
	* If a given action is an unrunnable start or promote, block the start or
	* promote actions of resources colocated with it, as appropriate to the
	* colocations' configured roles.
	*
	* \param[in,out] action Action to check
	*/
	void
	pcmk__block_colocation_dependents(pcmk_action_t *action)
	{
	GList *iter = NULL;
	GList *colocations = NULL;
	pcmk_resource_t *rsc = NULL;
	bool is_start = false;

	if (pcmk_is_set(action->flags, pcmk_action_runnable)) {
	return; // Only unrunnable actions block dependents
	}

	is_start = pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_none);
	if (!is_start
	&& !pcmk__str_eq(action->task, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return; // Only unrunnable starts and promotes block dependents
	}

	CRM_ASSERT(action->rsc != NULL); // Start and promote are resource actions

	/* If this resource is part of a collective resource, dependents are blocked
	* only if all instances of the collective are unrunnable, so check the
	* collective resource.
	*/
	rsc = uber_parent(action->rsc);
	if (rsc->parent != NULL) {
	rsc = rsc->parent; // Bundle
	}

	// Colocation fails only if entire primary can't reach desired role
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t *child = iter->data;
	pcmk_action_t *child_action = find_first_action(child->actions, NULL,
	action->task, NULL);

	if ((child_action == NULL)
	\|\| pcmk_is_set(child_action->flags, pcmk_action_runnable)) {
	crm_trace("Not blocking %s colocation dependents because "
	"at least %s has runnable %s",
	rsc->id, child->id, action->task);
	return; // At least one child can reach desired role
	}
	}

	crm_trace("Blocking %s colocation dependents due to unrunnable %s %s",
	rsc->id, action->rsc->id, action->task);

	// Check each colocation where this resource is primary
	colocations = pcmk__with_this_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (colocation->score < INFINITY) {
	continue; // Only mandatory colocations block dependent
	}

	/* If the primary can't start, the dependent can't reach its colocated
	* role, regardless of what the primary or dependent colocation role is.
	*
	* If the primary can't be promoted, the dependent can't reach its
	* colocated role if the primary's colocation role is promoted.
	*/
	if (!is_start && (colocation->primary_role != pcmk_role_promoted)) {
	continue;
	}

	// Block the dependent from reaching its colocated role
	if (colocation->dependent_role == pcmk_role_promoted) {
	mark_action_blocked(colocation->dependent, PCMK_ACTION_PROMOTE,
	action->rsc);
	} else {
	mark_action_blocked(colocation->dependent, PCMK_ACTION_START,
	action->rsc);
	}
	}
	g_list_free(colocations);
	}

	/*!
	* \internal
	* \brief Get the resource to use for role comparisons
	*
	* A bundle replica includes a container and possibly an instance of the bundled
	* resource. The dependent in a "with bundle" colocation is colocated with a
	* particular bundle container. However, if the colocation includes a role, then
	* the role must be checked on the bundled resource instance inside the
	* container. The container itself will never be promoted; the bundled resource
	* may be.
	*
	* If the given resource is a bundle replica container, return the resource
	* inside it, if any. Otherwise, return the resource itself.
	*
	* \param[in] rsc Resource to check
	*
	* \return Resource to use for role comparisons
	*/
	static const pcmk_resource_t *
	get_resource_for_role(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_replica_container)) {
	const pcmk_resource_t *child = pe__get_rsc_in_container(rsc);

	if (child != NULL) {
	return child;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine how a colocation constraint should affect a resource
	*
	* Colocation constraints have different effects at different points in the
	* scheduler sequence. Initially, they affect a resource's location; once that
	* is determined, then for promotable clones they can affect a resource
	* instance's role; after both are determined, the constraints no longer matter.
	* Given a specific colocation constraint, check what has been done so far to
	* determine what should be affected at the current point in the scheduler.
	*
	* \param[in] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint
	* \param[in] preview If true, pretend resources have already been assigned
	*
	* \return How colocation constraint should be applied at this point
	*/
	enum pcmk__coloc_affects
	pcmk__colocation_affects(const pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation, bool preview)
	{
	const pcmk_resource_t *dependent_role_rsc = NULL;
	const pcmk_resource_t *primary_role_rsc = NULL;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (!preview && pcmk_is_set(primary->flags, pcmk_rsc_unassigned)) {
	// Primary resource has not been assigned yet, so we can't do anything
	return pcmk__coloc_affects_nothing;
	}

	dependent_role_rsc = get_resource_for_role(dependent);
	primary_role_rsc = get_resource_for_role(primary);

	if ((colocation->dependent_role >= pcmk_role_unpromoted)
	&& (dependent_role_rsc->parent != NULL)
	&& pcmk_is_set(dependent_role_rsc->parent->flags, pcmk_rsc_promotable)
	&& !pcmk_is_set(dependent_role_rsc->flags, pcmk_rsc_unassigned)) {

	/* This is a colocation by role, and the dependent is a promotable clone
	* that has already been assigned, so the colocation should now affect
	* the role.
	*/
	return pcmk__coloc_affects_role;
	}

	if (!preview && !pcmk_is_set(dependent->flags, pcmk_rsc_unassigned)) {
	/* The dependent resource has already been through assignment, so the
	* constraint no longer has any effect. Log an error if a mandatory
	* colocation constraint has been violated.
	*/

	const pcmk_node_t *primary_node = primary->allocated_to;

	if (dependent->allocated_to == NULL) {
	crm_trace("Skipping colocation '%s': %s will not run anywhere",
	colocation->id, dependent->id);

	} else if (colocation->score >= INFINITY) {
	// Dependent resource must colocate with primary resource

	if (!pe__same_node(primary_node, dependent->allocated_to)) {
	- crm_err("%s must be colocated with %s but is not (%s vs. %s)",
	- dependent->id, primary->id,
	- pe__node_name(dependent->allocated_to),
	- pe__node_name(primary_node));
	+ pcmk__sched_err("%s must be colocated with %s but is not "
	+ "(%s vs. %s)",
	+ dependent->id, primary->id,
	+ pe__node_name(dependent->allocated_to),
	+ pe__node_name(primary_node));
	}

	} else if (colocation->score <= -CRM_SCORE_INFINITY) {
	// Dependent resource must anti-colocate with primary resource

	if (pe__same_node(dependent->allocated_to, primary_node)) {
	- crm_err("%s and %s must be anti-colocated but are assigned "
	- "to the same node (%s)",
	- dependent->id, primary->id,
	- pe__node_name(primary_node));
	+ pcmk__sched_err("%s and %s must be anti-colocated but are "
	+ "assigned to the same node (%s)",
	+ dependent->id, primary->id,
	+ pe__node_name(primary_node));
	}
	}
	return pcmk__coloc_affects_nothing;
	}

	if ((colocation->dependent_role != pcmk_role_unknown)
	&& (colocation->dependent_role != dependent_role_rsc->next_role)) {
	crm_trace("Skipping %scolocation '%s': dependent limited to %s role "

	"but %s next role is %s",
	((colocation->score < 0)? "anti-" : ""),
	colocation->id, role2text(colocation->dependent_role),
	dependent_role_rsc->id,
	role2text(dependent_role_rsc->next_role));
	return pcmk__coloc_affects_nothing;
	}

	if ((colocation->primary_role != pcmk_role_unknown)
	&& (colocation->primary_role != primary_role_rsc->next_role)) {
	crm_trace("Skipping %scolocation '%s': primary limited to %s role "
	"but %s next role is %s",
	((colocation->score < 0)? "anti-" : ""),
	colocation->id, role2text(colocation->primary_role),
	primary_role_rsc->id, role2text(primary_role_rsc->next_role));
	return pcmk__coloc_affects_nothing;
	}

	return pcmk__coloc_affects_location;
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent for assignment purposes
	*
	* Update the allowed node scores of the dependent resource in a colocation,
	* for the purposes of assigning it to a node.
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint
	*/
	void
	pcmk__apply_coloc_to_scores(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation)
	{
	const char *attr = colocation->node_attribute;
	const char *value = NULL;
	GHashTable *work = NULL;
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	if (primary->allocated_to != NULL) {
	value = pcmk__colocation_node_attr(primary->allocated_to, attr,
	primary);

	} else if (colocation->score < 0) {
	// Nothing to do (anti-colocation with something that is not running)
	return;
	}

	work = pcmk__copy_node_table(dependent->allowed_nodes);

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (primary->allocated_to == NULL) {
	node->weight = pcmk__add_scores(-colocation->score, node->weight);
	pcmk__rsc_trace(dependent,
	"Applied %s to %s score on %s (now %s after "
	"subtracting %s because primary %s inactive)",
	colocation->id, dependent->id, pe__node_name(node),
	pcmk_readable_score(node->weight),
	pcmk_readable_score(colocation->score), primary->id);
	continue;
	}

	if (pcmk__str_eq(pcmk__colocation_node_attr(node, attr, dependent),
	value, pcmk__str_casei)) {

	/* Add colocation score only if optional (or minus infinity). A
	* mandatory colocation is a requirement rather than a preference,
	* so we don't need to consider it for relative assignment purposes.
	* The resource will simply be forbidden from running on the node if
	* the primary isn't active there (via the condition above).
	*/
	if (colocation->score < CRM_SCORE_INFINITY) {
	node->weight = pcmk__add_scores(colocation->score,
	node->weight);
	pcmk__rsc_trace(dependent,
	"Applied %s to %s score on %s (now %s after "
	"adding %s)",
	colocation->id, dependent->id,
	pe__node_name(node),
	pcmk_readable_score(node->weight),
	pcmk_readable_score(colocation->score));
	}
	continue;
	}

	if (colocation->score >= CRM_SCORE_INFINITY) {
	/* Only mandatory colocations are relevant when the colocation
	* attribute doesn't match, because an attribute not matching is not
	* a negative preference -- the colocation is simply relevant only
	* where it matches.
	*/
	node->weight = -CRM_SCORE_INFINITY;
	pcmk__rsc_trace(dependent,
	"Banned %s from %s because colocation %s attribute %s "
	"does not match",
	dependent->id, pe__node_name(node), colocation->id,
	attr);
	}
	}

	if ((colocation->score <= -INFINITY) \|\| (colocation->score >= INFINITY)
	\|\| pcmk__any_node_available(work)) {

	g_hash_table_destroy(dependent->allowed_nodes);
	dependent->allowed_nodes = work;
	work = NULL;

	} else {
	pcmk__rsc_info(dependent,
	"%s: Rolling back scores from %s (no available nodes)",
	dependent->id, primary->id);
	}

	if (work != NULL) {
	g_hash_table_destroy(work);
	}
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent for role purposes
	*
	* Update the priority of the dependent resource in a colocation, for the
	* purposes of selecting its role
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint
	*/
	void
	pcmk__apply_coloc_to_priority(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation)
	{
	const char *dependent_value = NULL;
	const char *primary_value = NULL;
	const char *attr = colocation->node_attribute;
	int score_multiplier = 1;

	const pcmk_resource_t *primary_role_rsc = NULL;

	CRM_ASSERT((dependent != NULL) && (primary != NULL) &&
	(colocation != NULL));

	if ((primary->allocated_to == NULL) \|\| (dependent->allocated_to == NULL)) {
	return;
	}

	dependent_value = pcmk__colocation_node_attr(dependent->allocated_to, attr,
	dependent);
	primary_value = pcmk__colocation_node_attr(primary->allocated_to, attr,
	primary);

	primary_role_rsc = get_resource_for_role(primary);

	if (!pcmk__str_eq(dependent_value, primary_value, pcmk__str_casei)) {
	if ((colocation->score == INFINITY)
	&& (colocation->dependent_role == pcmk_role_promoted)) {
	dependent->priority = -INFINITY;
	}
	return;
	}

	if ((colocation->primary_role != pcmk_role_unknown)
	&& (colocation->primary_role != primary_role_rsc->next_role)) {
	return;
	}

	if (colocation->dependent_role == pcmk_role_unpromoted) {
	score_multiplier = -1;
	}

	dependent->priority = pcmk__add_scores(score_multiplier * colocation->score,
	dependent->priority);
	pcmk__rsc_trace(dependent,
	"Applied %s to %s promotion priority (now %s after %s %s)",
	colocation->id, dependent->id,
	pcmk_readable_score(dependent->priority),
	((score_multiplier == 1)? "adding" : "subtracting"),
	pcmk_readable_score(colocation->score));
	}

	/*!
	* \internal
	* \brief Find score of highest-scored node that matches colocation attribute
	*
	* \param[in] rsc Resource whose allowed nodes should be searched
	* \param[in] attr Colocation attribute name (must not be NULL)
	* \param[in] value Colocation attribute value to require
	*/
	static int
	best_node_score_matching_attr(const pcmk_resource_t rsc, const char attr,
	const char *value)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	int best_score = -INFINITY;
	const char *best_node = NULL;

	// Find best allowed node with matching attribute
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {

	if ((node->weight > best_score)
	&& pcmk__node_available(node, false, false)
	&& pcmk__str_eq(value, pcmk__colocation_node_attr(node, attr, rsc),
	pcmk__str_casei)) {

	best_score = node->weight;
	best_node = node->details->uname;
	}
	}

	if (!pcmk__str_eq(attr, CRM_ATTR_UNAME, pcmk__str_none)) {
	if (best_node == NULL) {
	crm_info("No allowed node for %s matches node attribute %s=%s",
	rsc->id, attr, value);
	} else {
	crm_info("Allowed node %s for %s had best score (%d) "
	"of those matching node attribute %s=%s",
	best_node, rsc->id, best_score, attr, value);
	}
	}
	return best_score;
	}

	/*!
	* \internal
	* \brief Check whether a resource is allowed only on a single node
	*
	* \param[in] rsc Resource to check
	*
	* \return \c true if \p rsc is allowed only on one node, otherwise \c false
	*/
	static bool
	allowed_on_one(const pcmk_resource_t *rsc)
	{
	GHashTableIter iter;
	pcmk_node_t *allowed_node = NULL;
	int allowed_nodes = 0;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &allowed_node)) {
	if ((allowed_node->weight >= 0) && (++allowed_nodes > 1)) {
	pcmk__rsc_trace(rsc, "%s is allowed on multiple nodes", rsc->id);
	return false;
	}
	}
	pcmk__rsc_trace(rsc, "%s is allowed %s", rsc->id,
	((allowed_nodes == 1)? "on a single node" : "nowhere"));
	return (allowed_nodes == 1);
	}

	/*!
	* \internal
	* \brief Add resource's colocation matches to current node assignment scores
	*
	* For each node in a given table, if any of a given resource's allowed nodes
	* have a matching value for the colocation attribute, add the highest of those
	* nodes' scores to the node's score.
	*
	* \param[in,out] nodes Table of nodes with assignment scores so far
	* \param[in] source_rsc Resource whose node scores to add
	* \param[in] target_rsc Resource on whose behalf to update \p nodes
	* \param[in] colocation Original colocation constraint (used to get
	* configured primary resource's stickiness, and
	* to get colocation node attribute; pass NULL to
	* ignore stickiness and use default attribute)
	* \param[in] factor Factor by which to multiply scores being added
	* \param[in] only_positive Whether to add only positive scores
	*/
	static void
	add_node_scores_matching_attr(GHashTable *nodes,
	const pcmk_resource_t *source_rsc,
	const pcmk_resource_t *target_rsc,
	const pcmk__colocation_t *colocation,
	float factor, bool only_positive)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	const char *attr = colocation->node_attribute;

	// Iterate through each node
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	float delta_f = 0;
	int delta = 0;
	int score = 0;
	int new_score = 0;
	const char *value = pcmk__colocation_node_attr(node, attr, target_rsc);

	score = best_node_score_matching_attr(source_rsc, attr, value);

	if ((factor < 0) && (score < 0)) {
	/* If the dependent is anti-colocated, we generally don't want the
	* primary to prefer nodes that the dependent avoids. That could
	* lead to unnecessary shuffling of the primary when the dependent
	* hits its migration threshold somewhere, for example.
	*
	* However, there are cases when it is desirable. If the dependent
	* can't run anywhere but where the primary is, it would be
	* worthwhile to move the primary for the sake of keeping the
	* dependent active.
	*
	* We can't know that exactly at this point since we don't know
	* where the primary will be assigned, but we can limit considering
	* the preference to when the dependent is allowed only on one node.
	* This is less than ideal for multiple reasons:
	*
	* - the dependent could be allowed on more than one node but have
	* anti-colocation primaries on each;
	* - the dependent could be a clone or bundle with multiple
	* instances, and the dependent as a whole is allowed on multiple
	* nodes but some instance still can't run
	* - the dependent has considered node-specific criteria such as
	* location constraints and stickiness by this point, but might
	* have other factors that end up disallowing a node
	*
	* but the alternative is making the primary move when it doesn't
	* need to.
	*
	* We also consider the primary's stickiness and influence, so the
	* user has some say in the matter. (This is the configured primary,
	* not a particular instance of the primary, but that doesn't matter
	* unless stickiness uses a rule to vary by node, and that seems
	* acceptable to ignore.)
	*/
	if ((colocation->primary->stickiness >= -score)
	\|\| !pcmk__colocation_has_influence(colocation, NULL)
	\|\| !allowed_on_one(colocation->dependent)) {
	crm_trace("%s: Filtering %d + %f * %d "
	"(double negative disallowed)",
	pe__node_name(node), node->weight, factor, score);
	continue;
	}
	}

	if (node->weight == INFINITY_HACK) {
	crm_trace("%s: Filtering %d + %f * %d (node was marked unusable)",
	pe__node_name(node), node->weight, factor, score);
	continue;
	}

	delta_f = factor * score;

	// Round the number; see http://c-faq.com/fp/round.html
	delta = (int) ((delta_f < 0)? (delta_f - 0.5) : (delta_f + 0.5));

	/* Small factors can obliterate the small scores that are often actually
	* used in configurations. If the score and factor are nonzero, ensure
	* that the result is nonzero as well.
	*/
	if ((delta == 0) && (score != 0)) {
	if (factor > 0.0) {
	delta = 1;
	} else if (factor < 0.0) {
	delta = -1;
	}
	}

	new_score = pcmk__add_scores(delta, node->weight);

	if (only_positive && (new_score < 0) && (node->weight > 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d "
	"(negative disallowed, marking node unusable)",
	pe__node_name(node), node->weight, factor, score,
	new_score);
	node->weight = INFINITY_HACK;
	continue;
	}

	if (only_positive && (new_score < 0) && (node->weight == 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d (negative disallowed)",
	pe__node_name(node), node->weight, factor, score,
	new_score);
	continue;
	}

	crm_trace("%s: %d + %f * %d = %d", pe__node_name(node),
	node->weight, factor, score, new_score);
	node->weight = new_score;
	}
	}

	/*!
	* \internal
	* \brief Update nodes with scores of colocated resources' nodes
	*
	* Given a table of nodes and a resource, update the nodes' scores with the
	* scores of the best nodes matching the attribute used for each of the
	* resource's relevant colocations.
	*
	* \param[in,out] source_rsc Resource whose node scores to add
	* \param[in] target_rsc Resource on whose behalf to update \p *nodes
	* \param[in] log_id Resource ID for logs (if \c NULL, use
	* \p source_rsc ID)
	* \param[in,out] nodes Nodes to update (set initial contents to \c NULL
	* to copy allowed nodes from \p source_rsc)
	* \param[in] colocation Original colocation constraint (used to get
	* configured primary resource's stickiness, and
	* to get colocation node attribute; if \c NULL,
	* <tt>source_rsc</tt>'s own matching node scores
	* will not be added, and \p *nodes must be \c NULL
	* as well)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pcmk__coloc_select values
	*
	* \note \c NULL \p target_rsc, \c NULL \p *nodes, \c NULL \p colocation, and
	* the \c pcmk__coloc_select_this_with flag are used together (and only by
	* \c cmp_resources()).
	* \note The caller remains responsible for freeing \p *nodes.
	* \note This is the shared implementation of
	* \c pcmk_assignment_methods_t:add_colocated_node_scores().
	*/
	void
	pcmk__add_colocated_node_scores(pcmk_resource_t *source_rsc,
	const pcmk_resource_t *target_rsc,
	const char *log_id,
	GHashTable **nodes,
	const pcmk__colocation_t *colocation,
	float factor, uint32_t flags)
	{
	GHashTable *work = NULL;

	CRM_ASSERT((source_rsc != NULL) && (nodes != NULL)
	&& ((colocation != NULL)
	\|\| ((target_rsc == NULL) && (*nodes == NULL))));

	if (log_id == NULL) {
	log_id = source_rsc->id;
	}

	// Avoid infinite recursion
	if (pcmk_is_set(source_rsc->flags, pcmk_rsc_updating_nodes)) {
	pcmk__rsc_info(source_rsc, "%s: Breaking dependency loop at %s",
	log_id, source_rsc->id);
	return;
	}
	pe__set_resource_flags(source_rsc, pcmk_rsc_updating_nodes);

	if (*nodes == NULL) {
	work = pcmk__copy_node_table(source_rsc->allowed_nodes);
	target_rsc = source_rsc;
	} else {
	const bool pos = pcmk_is_set(flags, pcmk__coloc_select_nonnegative);

	pcmk__rsc_trace(source_rsc, "%s: Merging %s scores from %s (at %.6f)",
	log_id, (pos? "positive" : "all"), source_rsc->id, factor);
	work = pcmk__copy_node_table(*nodes);
	add_node_scores_matching_attr(work, source_rsc, target_rsc, colocation,
	factor, pos);
	}

	if (work == NULL) {
	pe__clear_resource_flags(source_rsc, pcmk_rsc_updating_nodes);
	return;
	}

	if (pcmk__any_node_available(work)) {
	GList *colocations = NULL;

	if (pcmk_is_set(flags, pcmk__coloc_select_this_with)) {
	colocations = pcmk__this_with_colocations(source_rsc);
	pcmk__rsc_trace(source_rsc,
	"Checking additional %d optional '%s with' "
	"constraints",
	g_list_length(colocations), source_rsc->id);
	} else {
	colocations = pcmk__with_this_colocations(source_rsc);
	pcmk__rsc_trace(source_rsc,
	"Checking additional %d optional 'with %s' "
	"constraints",
	g_list_length(colocations), source_rsc->id);
	}
	flags \|= pcmk__coloc_select_active;

	for (GList *iter = colocations; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *constraint = iter->data;

	pcmk_resource_t *other = NULL;
	float other_factor = factor * constraint->score / (float) INFINITY;

	if (pcmk_is_set(flags, pcmk__coloc_select_this_with)) {
	other = constraint->primary;
	} else if (!pcmk__colocation_has_influence(constraint, NULL)) {
	continue;
	} else {
	other = constraint->dependent;
	}

	pcmk__rsc_trace(source_rsc,
	"Optionally merging score of '%s' constraint "
	"(%s with %s)",
	constraint->id, constraint->dependent->id,
	constraint->primary->id);
	other->cmds->add_colocated_node_scores(other, target_rsc, log_id,
	&work, constraint,
	other_factor, flags);
	pe__show_node_scores(true, NULL, log_id, work, source_rsc->cluster);
	}
	g_list_free(colocations);

	} else if (pcmk_is_set(flags, pcmk__coloc_select_active)) {
	pcmk__rsc_info(source_rsc, "%s: Rolling back optional scores from %s",
	log_id, source_rsc->id);
	g_hash_table_destroy(work);
	pe__clear_resource_flags(source_rsc, pcmk_rsc_updating_nodes);
	return;
	}


	if (pcmk_is_set(flags, pcmk__coloc_select_nonnegative)) {
	pcmk_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->weight == INFINITY_HACK) {
	node->weight = 1;
	}
	}
	}

	if (*nodes != NULL) {
	g_hash_table_destroy(*nodes);
	}
	*nodes = work;

	pe__clear_resource_flags(source_rsc, pcmk_rsc_updating_nodes);
	}

	/*!
	* \internal
	* \brief Apply a "with this" colocation to a resource's allowed node scores
	*
	* \param[in,out] data Colocation to apply
	* \param[in,out] user_data Resource being assigned
	*/
	void
	pcmk__add_dependent_scores(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t *colocation = data;
	pcmk_resource_t *target_rsc = user_data;

	pcmk_resource_t *source_rsc = colocation->dependent;
	const float factor = colocation->score / (float) INFINITY;
	uint32_t flags = pcmk__coloc_select_active;

	if (!pcmk__colocation_has_influence(colocation, NULL)) {
	return;
	}
	if (target_rsc->variant == pcmk_rsc_variant_clone) {
	flags \|= pcmk__coloc_select_nonnegative;
	}
	pcmk__rsc_trace(target_rsc,
	"%s: Incorporating attenuated %s assignment scores due "
	"to colocation %s",
	target_rsc->id, source_rsc->id, colocation->id);
	source_rsc->cmds->add_colocated_node_scores(source_rsc, target_rsc,
	source_rsc->id,
	&target_rsc->allowed_nodes,
	colocation, factor, flags);
	}

	/*!
	* \internal
	* \brief Exclude nodes from a dependent's node table if not in a given list
	*
	* Given a dependent resource in a colocation and a list of nodes where the
	* primary resource will run, set a node's score to \c -INFINITY in the
	* dependent's node table if not found in the primary nodes list.
	*
	* \param[in,out] dependent Dependent resource
	* \param[in] primary Primary resource (for logging only)
	* \param[in] colocation Colocation constraint (for logging only)
	* \param[in] primary_nodes List of nodes where the primary will have
	* unblocked instances in a suitable role
	* \param[in] merge_scores If \c true and a node is found in both \p table
	* and \p list, add the node's score in \p list to
	* the node's score in \p table
	*/
	void
	pcmk__colocation_intersect_nodes(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	const GList *primary_nodes, bool merge_scores)
	{
	GHashTableIter iter;
	pcmk_node_t *dependent_node = NULL;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	g_hash_table_iter_init(&iter, dependent->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &dependent_node)) {
	const pcmk_node_t *primary_node = NULL;

	primary_node = pe_find_node_id(primary_nodes,
	dependent_node->details->id);
	if (primary_node == NULL) {
	dependent_node->weight = -INFINITY;
	pcmk__rsc_trace(dependent,
	"Banning %s from %s (no primary instance) for %s",
	dependent->id, pe__node_name(dependent_node),
	colocation->id);

	} else if (merge_scores) {
	dependent_node->weight = pcmk__add_scores(dependent_node->weight,
	primary_node->weight);
	pcmk__rsc_trace(dependent,
	"Added %s's score %s to %s's score for %s (now %s) "
	"for colocation %s",
	primary->id, pcmk_readable_score(primary_node->weight),
	dependent->id, pe__node_name(dependent_node),
	pcmk_readable_score(dependent_node->weight),
	colocation->id);
	}
	}
	}

	/*!
	* \internal
	* \brief Get all colocations affecting a resource as the primary
	*
	* \param[in] rsc Resource to get colocations for
	*
	* \return Newly allocated list of colocations affecting \p rsc as primary
	*
	* \note This is a convenience wrapper for the with_this_colocations() method.
	*/
	GList *
	pcmk__with_this_colocations(const pcmk_resource_t *rsc)
	{
	GList *list = NULL;

	rsc->cmds->with_this_colocations(rsc, rsc, &list);
	return list;
	}

	/*!
	* \internal
	* \brief Get all colocations affecting a resource as the dependent
	*
	* \param[in] rsc Resource to get colocations for
	*
	* \return Newly allocated list of colocations affecting \p rsc as dependent
	*
	* \note This is a convenience wrapper for the this_with_colocations() method.
	*/
	GList *
	pcmk__this_with_colocations(const pcmk_resource_t *rsc)
	{
	GList *list = NULL;

	rsc->cmds->this_with_colocations(rsc, rsc, &list);
	return list;
	}
	diff --git a/lib/pacemaker/pcmk_sched_constraints.c b/lib/pacemaker/pcmk_sched_constraints.c
	index 1177f9f3d2..121c1aef92 100644
	--- a/lib/pacemaker/pcmk_sched_constraints.c
	+++ b/lib/pacemaker/pcmk_sched_constraints.c
	@@ -1,424 +1,424 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <stdbool.h>
	#include <regex.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/pengine/rules.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	static bool
	evaluate_lifetime(xmlNode lifetime, pcmk_scheduler_t scheduler)
	{
	bool result = FALSE;
	crm_time_t *next_change = crm_time_new_undefined();

	result = pe_evaluate_rules(lifetime, NULL, scheduler->now, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, scheduler, "constraint lifetime");
	}
	crm_time_free(next_change);
	return result;
	}

	/*!
	* \internal
	* \brief Unpack constraints from XML
	*
	* Given scheduler data, unpack all constraints from its input XML into
	* data structures.
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__unpack_constraints(pcmk_scheduler_t *scheduler)
	{
	xmlNode *xml_constraints = pcmk_find_cib_element(scheduler->input,
	XML_CIB_TAG_CONSTRAINTS);

	for (xmlNode *xml_obj = pcmk__xe_first_child(xml_constraints);
	xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj)) {

	xmlNode *lifetime = NULL;
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char tag = (const char ) xml_obj->name;

	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without "
	XML_ATTR_ID, tag);
	continue;
	}

	crm_trace("Unpacking %s constraint '%s'", tag, id);

	lifetime = first_named_child(xml_obj, "lifetime");
	if (lifetime != NULL) {
	pcmk__config_warn("Support for 'lifetime' attribute (in %s) is "
	"deprecated (the rules it contains should "
	"instead be direct descendants of the "
	"constraint object)", id);
	}

	if ((lifetime != NULL) && !evaluate_lifetime(lifetime, scheduler)) {
	crm_info("Constraint %s %s is not active", tag, id);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_ORDER, tag, pcmk__str_none)) {
	pcmk__unpack_ordering(xml_obj, scheduler);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_DEPEND, tag, pcmk__str_none)) {
	pcmk__unpack_colocation(xml_obj, scheduler);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_LOCATION, tag,
	pcmk__str_none)) {
	pcmk__unpack_location(xml_obj, scheduler);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_TICKET, tag, pcmk__str_none)) {
	pcmk__unpack_rsc_ticket(xml_obj, scheduler);

	} else {
	pcmk__config_err("Unsupported constraint type: %s", tag);
	}
	}
	}

	pcmk_resource_t *
	pcmk__find_constraint_resource(GList rsc_list, const char id)
	{
	if (id == NULL) {
	return NULL;
	}
	for (GList *iter = rsc_list; iter != NULL; iter = iter->next) {
	pcmk_resource_t *parent = iter->data;
	pcmk_resource_t *match = parent->fns->find_rsc(parent, id, NULL,
	pcmk_rsc_match_history);

	if (match != NULL) {
	if (!pcmk__str_eq(match->id, id, pcmk__str_none)) {
	/* We found an instance of a clone instead */
	match = uber_parent(match);
	crm_debug("Found %s for %s", match->id, id);
	}
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether an ID references a resource tag
	*
	* \param[in] scheduler Scheduler data
	* \param[in] id Tag ID to search for
	* \param[out] tag Where to store tag, if found
	*
	* \return true if ID refers to a tagged resource or resource set template,
	* otherwise false
	*/
	static bool
	find_constraint_tag(const pcmk_scheduler_t scheduler, const char id,
	pcmk_tag_t **tag)
	{
	*tag = NULL;

	// Check whether id refers to a resource set template
	if (g_hash_table_lookup_extended(scheduler->template_rsc_sets, id,
	NULL, (gpointer *) tag)) {
	if (*tag == NULL) {
	- crm_warn("No resource is derived from template '%s'", id);
	+ crm_notice("No resource is derived from template '%s'", id);
	return false;
	}
	return true;
	}

	// If not, check whether id refers to a tag
	if (g_hash_table_lookup_extended(scheduler->tags, id,
	NULL, (gpointer *) tag)) {
	if (*tag == NULL) {
	- crm_warn("No resource is tagged with '%s'", id);
	+ crm_notice("No resource is tagged with '%s'", id);
	return false;
	}
	return true;
	}

	- crm_warn("No template or tag named '%s'", id);
	+ pcmk__config_warn("No resource, template, or tag named '%s'", id);
	return false;
	}

	/*!
	* \brief
	* \internal Check whether an ID refers to a valid resource or tag
	*
	* \param[in] scheduler Scheduler data
	* \param[in] id ID to search for
	* \param[out] rsc Where to store resource, if found
	* (or NULL to skip searching resources)
	* \param[out] tag Where to store tag, if found
	* (or NULL to skip searching tags)
	*
	* \return true if id refers to a resource (possibly indirectly via a tag)
	*/
	bool
	pcmk__valid_resource_or_tag(const pcmk_scheduler_t scheduler, const char id,
	pcmk_resource_t rsc, pcmk_tag_t tag)
	{
	if (rsc != NULL) {
	*rsc = pcmk__find_constraint_resource(scheduler->resources, id);
	if (*rsc != NULL) {
	return true;
	}
	}

	if ((tag != NULL) && find_constraint_tag(scheduler, id, tag)) {
	return true;
	}

	return false;
	}

	/*!
	* \internal
	* \brief Replace any resource tags with equivalent resource_ref entries
	*
	* If a given constraint has resource sets, check each set for resource_ref
	* entries that list tags rather than resource IDs, and replace any found with
	* resource_ref entries for the corresponding resource IDs.
	*
	* \param[in,out] xml_obj Constraint XML
	* \param[in] scheduler Scheduler data
	*
	* \return Equivalent XML with resource tags replaced (or NULL if none)
	* \note It is the caller's responsibility to free the result with free_xml().
	*/
	xmlNode *
	pcmk__expand_tags_in_sets(xmlNode xml_obj, const pcmk_scheduler_t scheduler)
	{
	xmlNode *new_xml = NULL;
	bool any_refs = false;

	// Short-circuit if there are no sets
	if (first_named_child(xml_obj, XML_CONS_TAG_RSC_SET) == NULL) {
	return NULL;
	}

	new_xml = copy_xml(xml_obj);

	for (xmlNode *set = first_named_child(new_xml, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	GList *tag_refs = NULL;
	GList *iter = NULL;

	for (xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	pcmk_resource_t *rsc = NULL;
	pcmk_tag_t *tag = NULL;

	if (!pcmk__valid_resource_or_tag(scheduler, ID(xml_rsc), &rsc,
	&tag)) {
	pcmk__config_err("Ignoring resource sets for constraint '%s' "
	"because '%s' is not a valid resource or tag",
	ID(xml_obj), ID(xml_rsc));
	free_xml(new_xml);
	return NULL;

	} else if (rsc) {
	continue;

	} else if (tag) {
	// resource_ref under resource_set references template or tag
	xmlNode *last_ref = xml_rsc;

	/* For example, given the original XML:
	*
	* <resource_set id="tag1-colocation-0" sequential="true">
	* <resource_ref id="rsc1"/>
	* <resource_ref id="tag1"/>
	* <resource_ref id="rsc4"/>
	* </resource_set>
	*
	* If rsc2 and rsc3 are tagged with tag1, we add them after it:
	*
	* <resource_set id="tag1-colocation-0" sequential="true">
	* <resource_ref id="rsc1"/>
	* <resource_ref id="tag1"/>
	* <resource_ref id="rsc2"/>
	* <resource_ref id="rsc3"/>
	* <resource_ref id="rsc4"/>
	* </resource_set>
	*/

	for (iter = tag->refs; iter != NULL; iter = iter->next) {
	const char *obj_ref = iter->data;
	xmlNode *new_rsc_ref = NULL;

	new_rsc_ref = xmlNewDocRawNode(set->doc, NULL,
	(pcmkXmlStr)
	XML_TAG_RESOURCE_REF,
	NULL);
	crm_xml_add(new_rsc_ref, XML_ATTR_ID, obj_ref);
	xmlAddNextSibling(last_ref, new_rsc_ref);

	last_ref = new_rsc_ref;
	}

	any_refs = true;

	/* Freeing the resource_ref now would break the XML child
	* iteration, so just remember it for freeing later.
	*/
	tag_refs = g_list_append(tag_refs, xml_rsc);
	}
	}

	/* Now free '<resource_ref id="tag1"/>', and finally get:

	<resource_set id="tag1-colocation-0" sequential="true">
	<resource_ref id="rsc1"/>
	<resource_ref id="rsc2"/>
	<resource_ref id="rsc3"/>
	<resource_ref id="rsc4"/>
	</resource_set>

	*/
	for (iter = tag_refs; iter != NULL; iter = iter->next) {
	xmlNode *tag_ref = iter->data;

	free_xml(tag_ref);
	}
	g_list_free(tag_refs);
	}

	if (!any_refs) {
	free_xml(new_xml);
	new_xml = NULL;
	}
	return new_xml;
	}

	/*!
	* \internal
	* \brief Convert a tag into a resource set of tagged resources
	*
	* \param[in,out] xml_obj Constraint XML
	* \param[out] rsc_set Where to store resource set XML
	* \param[in] attr Name of XML attribute with resource or tag ID
	* \param[in] convert_rsc If true, convert to set even if \p attr
	* references a resource
	* \param[in] scheduler Scheduler data
	*/
	bool
	pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	bool convert_rsc, const pcmk_scheduler_t *scheduler)
	{
	const char *cons_id = NULL;
	const char *id = NULL;

	pcmk_resource_t *rsc = NULL;
	pcmk_tag_t *tag = NULL;

	*rsc_set = NULL;

	CRM_CHECK((xml_obj != NULL) && (attr != NULL), return false);

	cons_id = ID(xml_obj);
	if (cons_id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	xml_obj->name);
	return false;
	}

	id = crm_element_value(xml_obj, attr);
	if (id == NULL) {
	return true;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id, &rsc, &tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", cons_id, id);
	return false;

	} else if (tag) {
	/* The "attr" attribute (for a resource in a constraint) specifies a
	* template or tag. Add the corresponding resource_set containing the
	* resources derived from or tagged with it.
	*/
	*rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
	crm_xml_add(*rsc_set, XML_ATTR_ID, id);

	for (GList *iter = tag->refs; iter != NULL; iter = iter->next) {
	const char *obj_ref = iter->data;
	xmlNode *rsc_ref = NULL;

	rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
	crm_xml_add(rsc_ref, XML_ATTR_ID, obj_ref);
	}

	/* Set sequential="false" for the resource_set */
	pcmk__xe_set_bool_attr(*rsc_set, "sequential", false);

	} else if ((rsc != NULL) && convert_rsc) {
	/* Even if a regular resource is referenced by "attr", convert it into a
	* resource_set, because the other resource reference in the constraint
	* could be a template or tag.
	*/
	xmlNode *rsc_ref = NULL;

	*rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
	crm_xml_add(*rsc_set, XML_ATTR_ID, id);

	rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
	crm_xml_add(rsc_ref, XML_ATTR_ID, id);

	} else {
	return true;
	}

	/* Remove the "attr" attribute referencing the template/tag */
	if (*rsc_set != NULL) {
	xml_remove_prop(xml_obj, attr);
	}

	return true;
	}

	/*!
	* \internal
	* \brief Create constraints inherent to resource types
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__create_internal_constraints(pcmk_scheduler_t *scheduler)
	{
	crm_trace("Create internal constraints");
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->internal_constraints(rsc);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c
	index 667f42ab7e..2899b37d09 100644
	--- a/lib/pacemaker/pcmk_sched_ordering.c
	+++ b/lib/pacemaker/pcmk_sched_ordering.c
	@@ -1,1503 +1,1502 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <inttypes.h> // PRIx32
	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	enum pe_order_kind {
	pe_order_kind_optional,
	pe_order_kind_mandatory,
	pe_order_kind_serialize,
	};

	enum ordering_symmetry {
	ordering_asymmetric, // the only relation in an asymmetric ordering
	ordering_symmetric, // the normal relation in a symmetric ordering
	ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
	};

	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	__rsc = pcmk__find_constraint_resource(scheduler->resources, \
	__name); \
	if (__rsc == NULL) { \
	pcmk__config_err("%s: No resource found for %s", __set, __name);\
	return pcmk_rc_unpack_error; \
	} \
	} while (0)

	static const char *
	invert_action(const char *action)
	{
	if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
	return PCMK_ACTION_STOP;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
	return PCMK_ACTION_START;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
	return PCMK_ACTION_STOPPED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
	return PCMK_ACTION_RUNNING;
	}
	- crm_warn("Unknown action '%s' specified in order constraint", action);
	+ pcmk__config_warn("Unknown action '%s' specified in order constraint",
	+ action);
	return NULL;
	}

	static enum pe_order_kind
	get_ordering_type(const xmlNode *xml_obj)
	{
	enum pe_order_kind kind_e = pe_order_kind_mandatory;
	const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);

	if (kind == NULL) {
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);

	kind_e = pe_order_kind_mandatory;

	if (score) {
	// @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
	int score_i = char2score(score);

	if (score_i == 0) {
	kind_e = pe_order_kind_optional;
	}
	pe_warn_once(pcmk__wo_order_score,
	"Support for 'score' in rsc_order is deprecated "
	"and will be removed in a future release "
	"(use 'kind' instead)");
	}

	} else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_none)) {
	kind_e = pe_order_kind_mandatory;

	} else if (pcmk__str_eq(kind, "Optional", pcmk__str_none)) {
	kind_e = pe_order_kind_optional;

	} else if (pcmk__str_eq(kind, "Serialize", pcmk__str_none)) {
	kind_e = pe_order_kind_serialize;

	} else {
	pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
	"%s to 'Mandatory' because '%s' is not valid",
	pcmk__s(ID(xml_obj), "missing ID"), kind);
	}
	return kind_e;
	}

	/*!
	* \internal
	* \brief Get ordering symmetry from XML
	*
	* \param[in] xml_obj Ordering XML
	* \param[in] parent_kind Default ordering kind
	* \param[in] parent_symmetrical_s Parent element's symmetrical setting, if any
	*
	* \retval ordering_symmetric Ordering is symmetric
	* \retval ordering_asymmetric Ordering is asymmetric
	*/
	static enum ordering_symmetry
	get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
	const char *parent_symmetrical_s)
	{
	int rc = pcmk_rc_ok;
	bool symmetric = false;
	enum pe_order_kind kind = parent_kind; // Default to parent's kind

	// Check ordering XML for explicit kind
	if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL)
	\|\| (crm_element_value(xml_obj, XML_RULE_ATTR_SCORE) != NULL)) {
	kind = get_ordering_type(xml_obj);
	}

	// Check ordering XML (and parent) for explicit symmetrical setting
	rc = pcmk__xe_get_bool_attr(xml_obj, XML_CONS_ATTR_SYMMETRICAL, &symmetric);

	if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
	symmetric = crm_is_true(parent_symmetrical_s);
	rc = pcmk_rc_ok;
	}

	if (rc == pcmk_rc_ok) {
	if (symmetric) {
	if (kind == pe_order_kind_serialize) {
	pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
	" for '%s' because not valid with "
	XML_ORDER_ATTR_KIND " of 'Serialize'",
	ID(xml_obj));
	} else {
	return ordering_symmetric;
	}
	}
	return ordering_asymmetric;
	}

	// Use default symmetry
	if (kind == pe_order_kind_serialize) {
	return ordering_asymmetric;
	}
	return ordering_symmetric;
	}

	/*!
	* \internal
	* \brief Get ordering flags appropriate to ordering kind
	*
	* \param[in] kind Ordering kind
	* \param[in] first Action name for 'first' action
	* \param[in] symmetry This ordering's symmetry role
	*
	* \return Minimal ordering flags appropriate to \p kind
	*/
	static uint32_t
	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
	enum ordering_symmetry symmetry)
	{
	uint32_t flags = pcmk__ar_none; // so we trace-log all flags set

	switch (kind) {
	case pe_order_kind_optional:
	pe__set_order_flags(flags, pcmk__ar_ordered);
	break;

	case pe_order_kind_serialize:
	/* This flag is not used anywhere directly but means the relation
	* will not match an equality comparison against pcmk__ar_none or
	* pcmk__ar_ordered.
	*/
	pe__set_order_flags(flags, pcmk__ar_serialize);
	break;

	case pe_order_kind_mandatory:
	pe__set_order_flags(flags, pcmk__ar_ordered);
	switch (symmetry) {
	case ordering_asymmetric:
	pe__set_order_flags(flags, pcmk__ar_asymmetric);
	break;

	case ordering_symmetric:
	pe__set_order_flags(flags, pcmk__ar_first_implies_then);
	if (pcmk__strcase_any_of(first, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	pe__set_order_flags(flags,
	pcmk__ar_unrunnable_first_blocks);
	}
	break;

	case ordering_symmetric_inverse:
	pe__set_order_flags(flags, pcmk__ar_then_implies_first);
	break;
	}
	break;
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Find resource corresponding to ID specified in ordering
	*
	* \param[in] xml Ordering XML
	* \param[in] resource_attr XML attribute name for resource ID
	* \param[in] instance_attr XML attribute name for instance number.
	* This option is deprecated and will be removed in a
	* future release.
	* \param[in] scheduler Scheduler data
	*
	* \return Resource corresponding to \p id, or NULL if none
	*/
	static pcmk_resource_t *
	get_ordering_resource(const xmlNode xml, const char resource_attr,
	const char *instance_attr,
	const pcmk_scheduler_t *scheduler)
	{
	// @COMPAT: instance_attr and instance_id variables deprecated since 2.1.5
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(xml, resource_attr);
	const char *instance_id = crm_element_value(xml, instance_attr);

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without %s",
	ID(xml), resource_attr);
	return NULL;
	}

	rsc = pcmk__find_constraint_resource(scheduler->resources, rsc_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", ID(xml), rsc_id);
	return NULL;
	}

	if (instance_id != NULL) {
	pe_warn_once(pcmk__wo_order_inst,
	"Support for " XML_ORDER_ATTR_FIRST_INSTANCE " and "
	XML_ORDER_ATTR_THEN_INSTANCE " is deprecated and will be "
	"removed in a future release.");

	if (!pe_rsc_is_clone(rsc)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	ID(xml), rsc_id, instance_id);
	return NULL;
	}
	rsc = find_clone_instance(rsc, instance_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", ID(xml), rsc_id, instance_id);
	return NULL;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine minimum number of 'first' instances required in ordering
	*
	* \param[in] rsc 'First' resource in ordering
	* \param[in] xml Ordering XML
	*
	* \return Minimum 'first' instances required (or 0 if not applicable)
	*/
	static int
	get_minimum_first_instances(const pcmk_resource_t rsc, const xmlNode xml)
	{
	const char *clone_min = NULL;
	bool require_all = false;

	if (!pe_rsc_is_clone(rsc)) {
	return 0;
	}

	clone_min = g_hash_table_lookup(rsc->meta, PCMK_META_CLONE_MIN);
	if (clone_min != NULL) {
	int clone_min_int = 0;

	pcmk__scan_min_int(clone_min, &clone_min_int, 0);
	return clone_min_int;
	}

	/* @COMPAT 1.1.13:
	* require-all=false is deprecated equivalent of clone-min=1
	*/
	if (pcmk__xe_get_bool_attr(xml, "require-all", &require_all) != ENODATA) {
	pe_warn_once(pcmk__wo_require_all,
	"Support for require-all in ordering constraints "
	"is deprecated and will be removed in a future release"
	" (use clone-min clone meta-attribute instead)");
	if (!require_all) {
	return 1;
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Create orderings for a constraint with clone-min > 0
	*
	* \param[in] id Ordering ID
	* \param[in,out] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	* \param[in] flags Ordering flags
	* \param[in] clone_min Minimum required instances of 'first'
	*/
	static void
	clone_min_ordering(const char *id,
	pcmk_resource_t rsc_first, const char action_first,
	pcmk_resource_t rsc_then, const char action_then,
	uint32_t flags, int clone_min)
	{
	// Create a pseudo-action for when the minimum instances are active
	char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
	pcmk_action_t *clone_min_met = get_pseudo_op(task, rsc_first->cluster);

	free(task);

	/* Require the pseudo-action to have the required number of actions to be
	* considered runnable before allowing the pseudo-action to be runnable.
	*/
	clone_min_met->required_runnable_before = clone_min;
	pe__set_action_flags(clone_min_met, pcmk_action_min_runnable);

	// Order the actions for each clone instance before the pseudo-action
	for (GList *iter = rsc_first->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t *child = iter->data;

	pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
	NULL, NULL, NULL, clone_min_met,
	pcmk__ar_min_runnable
	\|pcmk__ar_first_implies_then_graphed,
	rsc_first->cluster);
	}

	// Order "then" action after the pseudo-action (if runnable)
	pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
	pcmk__op_key(rsc_then->id, action_then, 0),
	NULL, flags\|pcmk__ar_unrunnable_first_blocks,
	rsc_first->cluster);
	}

	/*!
	* \internal
	* \brief Update ordering flags for restart-type=restart
	*
	* \param[in] rsc 'Then' resource in ordering
	* \param[in] kind Ordering kind
	* \param[in] flag Ordering flag to set (when applicable)
	* \param[in,out] flags Ordering flag set to update
	*
	* \compat The restart-type resource meta-attribute is deprecated. Eventually,
	* it will be removed, and pe_restart_ignore will be the only behavior,
	* at which time this can just be removed entirely.
	*/
	#define handle_restart_type(rsc, kind, flag, flags) do { \
	if (((kind) == pe_order_kind_optional) \
	&& ((rsc)->restart_type == pe_restart_restart)) { \
	pe__set_order_flags((flags), (flag)); \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Create new ordering for inverse of symmetric constraint
	*
	* \param[in] id Ordering ID (for logging only)
	* \param[in] kind Ordering kind
	* \param[in] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in,out] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	*/
	static void
	inverse_ordering(const char *id, enum pe_order_kind kind,
	pcmk_resource_t rsc_first, const char action_first,
	pcmk_resource_t rsc_then, const char action_then)
	{
	action_then = invert_action(action_then);
	action_first = invert_action(action_first);
	if ((action_then == NULL) \|\| (action_first == NULL)) {
	pcmk__config_warn("Cannot invert constraint '%s' "
	"(please specify inverse manually)", id);
	} else {
	uint32_t flags = ordering_flags_for_kind(kind, action_first,
	ordering_symmetric_inverse);

	handle_restart_type(rsc_then, kind, pcmk__ar_then_implies_first, flags);
	pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
	action_first, flags);
	}
	}

	static void
	unpack_simple_rsc_order(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *rsc_then = NULL;
	pcmk_resource_t *rsc_first = NULL;
	int min_required_before = 0;
	enum pe_order_kind kind = pe_order_kind_mandatory;
	uint32_t flags = pcmk__ar_none;
	enum ordering_symmetry symmetry;

	const char *action_then = NULL;
	const char *action_first = NULL;
	const char *id = NULL;

	CRM_CHECK(xml_obj != NULL, return);

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	xml_obj->name);
	return;
	}

	rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST,
	XML_ORDER_ATTR_FIRST_INSTANCE,
	scheduler);
	if (rsc_first == NULL) {
	return;
	}

	rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN,
	XML_ORDER_ATTR_THEN_INSTANCE,
	scheduler);
	if (rsc_then == NULL) {
	return;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	if (action_first == NULL) {
	action_first = PCMK_ACTION_START;
	}

	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
	if (action_then == NULL) {
	action_then = action_first;
	}

	kind = get_ordering_type(xml_obj);

	symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
	flags = ordering_flags_for_kind(kind, action_first, symmetry);

	handle_restart_type(rsc_then, kind, pcmk__ar_first_implies_then, flags);

	/* If there is a minimum number of instances that must be runnable before
	* the 'then' action is runnable, we use a pseudo-action for convenience:
	* minimum number of clone instances have runnable actions ->
	* pseudo-action is runnable -> dependency is runnable.
	*/
	min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
	if (min_required_before > 0) {
	clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
	flags, min_required_before);
	} else {
	pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
	action_then, flags);
	}

	if (symmetry == ordering_symmetric) {
	inverse_ordering(id, kind, rsc_first, action_first,
	rsc_then, action_then);
	}
	}

	/*!
	* \internal
	* \brief Create a new ordering between two actions
	*
	* \param[in,out] first_rsc Resource for 'first' action (if NULL and
	* \p first_action is a resource action, that
	* resource will be used)
	* \param[in,out] first_action_task Action key for 'first' action (if NULL and
	* \p first_action is not NULL, its UUID will
	* be used)
	* \param[in,out] first_action 'first' action (if NULL, \p first_rsc and
	* \p first_action_task must be set)
	*
	* \param[in] then_rsc Resource for 'then' action (if NULL and
	* \p then_action is a resource action, that
	* resource will be used)
	* \param[in,out] then_action_task Action key for 'then' action (if NULL and
	* \p then_action is not NULL, its UUID will
	* be used)
	* \param[in] then_action 'then' action (if NULL, \p then_rsc and
	* \p then_action_task must be set)
	*
	* \param[in] flags Group of enum pcmk__action_relation_flags
	* \param[in,out] sched Scheduler data to add ordering to
	*
	* \note This function takes ownership of first_action_task and
	* then_action_task, which do not need to be freed by the caller.
	*/
	void
	pcmk__new_ordering(pcmk_resource_t first_rsc, char first_action_task,
	pcmk_action_t first_action, pcmk_resource_t then_rsc,
	char then_action_task, pcmk_action_t then_action,
	uint32_t flags, pcmk_scheduler_t *sched)
	{
	pcmk__action_relation_t *order = NULL;

	// One of action or resource must be specified for each side
	CRM_CHECK(((first_action != NULL) \|\| (first_rsc != NULL))
	&& ((then_action != NULL) \|\| (then_rsc != NULL)),
	free(first_action_task); free(then_action_task); return);

	if ((first_rsc == NULL) && (first_action != NULL)) {
	first_rsc = first_action->rsc;
	}
	if ((then_rsc == NULL) && (then_action != NULL)) {
	then_rsc = then_action->rsc;
	}

	order = calloc(1, sizeof(pcmk__action_relation_t));
	CRM_ASSERT(order != NULL);

	order->id = sched->order_id++;
	order->flags = flags;
	order->rsc1 = first_rsc;
	order->rsc2 = then_rsc;
	order->action1 = first_action;
	order->action2 = then_action;
	order->task1 = first_action_task;
	order->task2 = then_action_task;

	if ((order->task1 == NULL) && (first_action != NULL)) {
	order->task1 = strdup(first_action->uuid);
	}

	if ((order->task2 == NULL) && (then_action != NULL)) {
	order->task2 = strdup(then_action->uuid);
	}

	if ((order->rsc1 == NULL) && (first_action != NULL)) {
	order->rsc1 = first_action->rsc;
	}

	if ((order->rsc2 == NULL) && (then_action != NULL)) {
	order->rsc2 = then_action->rsc;
	}

	pcmk__rsc_trace(first_rsc, "Created ordering %d for %s then %s",
	(sched->order_id - 1),
	pcmk__s(order->task1, "an underspecified action"),
	pcmk__s(order->task2, "an underspecified action"));

	sched->ordering_constraints = g_list_prepend(sched->ordering_constraints,
	order);
	pcmk__order_migration_equivalents(order);
	}

	/*!
	* \brief Unpack a set in an ordering constraint
	*
	* \param[in] set Set XML to unpack
	* \param[in] parent_kind rsc_order XML "kind" attribute
	* \param[in] parent_symmetrical_s rsc_order XML "symmetrical" attribute
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
	const char parent_symmetrical_s, pcmk_scheduler_t scheduler)
	{
	GList *set_iter = NULL;
	GList *resources = NULL;

	pcmk_resource_t *last = NULL;
	pcmk_resource_t *resource = NULL;

	int local_kind = parent_kind;
	bool sequential = false;
	uint32_t flags = pcmk__ar_ordered;
	enum ordering_symmetry symmetry;

	char *key = NULL;
	const char *id = ID(set);
	const char *action = crm_element_value(set, "action");
	const char *sequential_s = crm_element_value(set, "sequential");
	const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);

	if (action == NULL) {
	action = PCMK_ACTION_START;
	}

	if (kind_s) {
	local_kind = get_ordering_type(set);
	}
	if (sequential_s == NULL) {
	sequential_s = "1";
	}

	sequential = crm_is_true(sequential_s);

	symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
	flags = ordering_flags_for_kind(local_kind, action, symmetry);

	for (const xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
	resources = g_list_append(resources, resource);
	}

	if (pcmk__list_of_1(resources)) {
	crm_trace("Single set: %s", id);
	goto done;
	}

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pcmk_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = pcmk__op_key(resource->id, action, 0);

	if (local_kind == pe_order_kind_serialize) {
	/* Serialize before everything that comes after */

	for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
	pcmk_resource_t *then_rsc = iter->data;
	char *then_key = pcmk__op_key(then_rsc->id, action, 0);

	pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
	then_key, NULL, flags, scheduler);
	}

	} else if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(last, action, resource, action,
	flags);
	}
	last = resource;
	}
	free(key);
	}

	if (symmetry == ordering_asymmetric) {
	goto done;
	}

	last = NULL;
	action = invert_action(action);

	flags = ordering_flags_for_kind(local_kind, action,
	ordering_symmetric_inverse);

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pcmk_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(resource, action, last, action,
	flags);
	}
	last = resource;
	}
	}

	done:
	g_list_free(resources);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Order two resource sets relative to each other
	*
	* \param[in] id Ordering ID (for logging)
	* \param[in] set1 First listed set
	* \param[in] set2 Second listed set
	* \param[in] kind Ordering kind
	* \param[in,out] scheduler Scheduler data
	* \param[in] symmetry Which ordering symmetry applies to this relation
	*
	* \return Standard Pacemaker return code
	*/
	static int
	order_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	enum pe_order_kind kind, pcmk_scheduler_t *scheduler,
	enum ordering_symmetry symmetry)
	{

	const xmlNode *xml_rsc = NULL;
	const xmlNode *xml_rsc_2 = NULL;

	pcmk_resource_t *rsc_1 = NULL;
	pcmk_resource_t *rsc_2 = NULL;

	const char *action_1 = crm_element_value(set1, "action");
	const char *action_2 = crm_element_value(set2, "action");

	uint32_t flags = pcmk__ar_none;

	bool require_all = true;

	(void) pcmk__xe_get_bool_attr(set1, "require-all", &require_all);

	if (action_1 == NULL) {
	action_1 = PCMK_ACTION_START;
	}

	if (action_2 == NULL) {
	action_2 = PCMK_ACTION_START;
	}

	if (symmetry == ordering_symmetric_inverse) {
	action_1 = invert_action(action_1);
	action_2 = invert_action(action_2);
	}

	if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
	\|\| pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
	/* Assuming: A -> ( B \|\| C) -> D
	* The one-or-more logic only applies during the start/promote phase.
	* During shutdown neither B nor can shutdown until D is down, so simply
	* turn require_all back on.
	*/
	require_all = true;
	}

	flags = ordering_flags_for_kind(kind, action_1, symmetry);

	/* If we have an unordered set1, whether it is sequential or not is
	* irrelevant in regards to set2.
	*/
	if (!require_all) {
	char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s", ID(set1));
	pcmk_action_t *unordered_action = get_pseudo_op(task, scheduler);

	free(task);
	pe__set_action_flags(unordered_action, pcmk_action_min_runnable);

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	/* Add an ordering constraint between every element in set1 and the
	* pseudo action. If any action in set1 is runnable the pseudo
	* action will be runnable.
	*/
	pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
	NULL, NULL, NULL, unordered_action,
	pcmk__ar_min_runnable
	\|pcmk__ar_first_implies_then_graphed,
	scheduler);
	}
	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));

	/* Add an ordering constraint between the pseudo-action and every
	* element in set2. If the pseudo-action is runnable, every action
	* in set2 will be runnable.
	*/
	pcmk__new_ordering(NULL, NULL, unordered_action,
	rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
	NULL, flags\|pcmk__ar_unrunnable_first_blocks,
	scheduler);
	}

	return pcmk_rc_ok;
	}

	if (pcmk__xe_attr_is_true(set1, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the first one
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	}

	} else {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
	}
	}

	if (pcmk__xe_attr_is_true(set2, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);

	} else {
	// Get the first one
	xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	}
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);

	} else if (rsc_1 != NULL) {
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else if (rsc_2 != NULL) {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	for (xmlNode *xml_rsc_2 = first_named_child(set2,
	XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
	action_2, flags);
	}
	}
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief If an ordering constraint uses resource tags, expand them
	*
	* \param[in,out] xml_obj Ordering constraint XML
	* \param[out] expanded_xml Equivalent XML with tags expanded
	* \param[in] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
	* and pcmk_rc_unpack_error on invalid configuration)
	*/
	static int
	unpack_order_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	const pcmk_scheduler_t *scheduler)
	{
	const char *id_first = NULL;
	const char *id_then = NULL;
	const char *action_first = NULL;
	const char *action_then = NULL;

	pcmk_resource_t *rsc_first = NULL;
	pcmk_resource_t *rsc_then = NULL;
	pcmk_tag_t *tag_first = NULL;
	pcmk_tag_t *tag_then = NULL;

	xmlNode *rsc_set_first = NULL;
	xmlNode *rsc_set_then = NULL;
	bool any_sets = false;

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	return pcmk_rc_ok;
	}

	id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
	id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
	if ((id_first == NULL) \|\| (id_then == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id_first, &rsc_first,
	&tag_first)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_first);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id_then, &rsc_then,
	&tag_then)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_then);
	return pcmk_rc_unpack_error;
	}

	if ((rsc_first != NULL) && (rsc_then != NULL)) {
	// Neither side references a template or tag
	return pcmk_rc_ok;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);

	*expanded_xml = copy_xml(xml_obj);

	// Convert template/tag reference in "first" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_first != NULL) {
	if (action_first != NULL) {
	// Move "first-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_first, "action", action_first);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION);
	}
	any_sets = true;
	}

	// Convert template/tag reference in "then" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_then != NULL) {
	if (action_then != NULL) {
	// Move "then-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_then, "action", action_then);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Unpack ordering constraint XML
	*
	* \param[in,out] xml_obj Ordering constraint XML to unpack
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__unpack_ordering(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	enum pe_order_kind kind = get_ordering_type(xml_obj);

	enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
	NULL);

	// Expand any resource tags in the constraint XML
	if (unpack_order_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	// If the constraint has resource sets, unpack them
	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	set = expand_idref(set, scheduler->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_order_set(set, kind, invert, scheduler) != pcmk_rc_ok)) {

	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if (last != NULL) {

	if (order_rsc_sets(id, last, set, kind, scheduler,
	symmetry) != pcmk_rc_ok) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if ((symmetry == ordering_symmetric)
	&& (order_rsc_sets(id, set, last, kind, scheduler,
	ordering_symmetric_inverse) != pcmk_rc_ok)) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	// If the constraint has no resource sets, unpack it as a simple ordering
	if (last == NULL) {
	return unpack_simple_rsc_order(xml_obj, scheduler);
	}
	}

	static bool
	ordering_is_invalid(pcmk_action_t action, pcmk__related_action_t input)
	{
	/* Prevent user-defined ordering constraints between resources
	* running in a guest node and the resource that defines that node.
	*/
	if (!pcmk_is_set(input->type, pcmk__ar_guest_allowed)
	&& (input->action->rsc != NULL)
	&& pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {

	- crm_warn("Invalid ordering constraint between %s and %s",
	- input->action->rsc->id, action->rsc->id);
	+ pcmk__config_warn("Invalid ordering constraint between %s and %s",
	+ input->action->rsc->id, action->rsc->id);
	return true;
	}

	/* If there's an order like
	* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
	*
	* then rscA is being migrated from node1 to node2, while rscB is being
	* migrated from node2 to node1. If there would be a graph loop,
	* break the order "load_stopped_node2" -> "rscA_migrate_to node1".
	*/
	if (((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target)
	&& (action->rsc != NULL)
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
	&& pcmk__graph_has_loop(action, action, input)) {
	return true;
	}

	return false;
	}

	void
	pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;
	pcmk__related_action_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = input_iter->data;
	if (ordering_is_invalid(action, input)) {
	input->type = (enum pe_ordering) pcmk__ar_none;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Order stops on a node before the node's shutdown
	*
	* \param[in,out] node Node being shut down
	* \param[in] shutdown_op Shutdown action for node
	*/
	void
	pcmk__order_stops_before_shutdown(pcmk_node_t node, pcmk_action_t shutdown_op)
	{
	for (GList *iter = node->details->data_set->actions;
	iter != NULL; iter = iter->next) {

	pcmk_action_t action = (pcmk_action_t ) iter->data;

	// Only stops on the node shutting down are relevant
	if (!pe__same_node(action->node, node)
	\|\| !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	continue;
	}

	// Resources and nodes in maintenance mode won't be touched

	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)) {
	pcmk__rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;

	} else if (node->details->maintenance) {
	pcmk__rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"node in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;
	}

	/* Don't touch a resource that is unmanaged or blocked, to avoid
	* blocking the shutdown (though if another action depends on this one,
	* we may still end up blocking)
	*/
	if (!pcmk_any_flags_set(action->rsc->flags,
	pcmk_rsc_managed\|pcmk_rsc_blocked)) {
	pcmk__rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource is unmanaged or blocked",
	action->uuid, pe__node_name(node));
	continue;
	}

	pcmk__rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
	action->uuid, pe__node_name(node));
	pe__clear_action_flags(action, pcmk_action_optional);
	pcmk__new_ordering(action->rsc, NULL, action, NULL,
	strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
	pcmk__ar_ordered\|pcmk__ar_unrunnable_first_blocks,
	node->details->data_set);
	}
	}

	/*!
	* \brief Find resource actions matching directly or as child
	*
	* \param[in] rsc Resource to check
	* \param[in] original_key Action key to search for (possibly referencing
	* parent of \rsc)
	*
	* \return Newly allocated list of matching actions
	* \note It is the caller's responsibility to free the result with g_list_free()
	*/
	static GList *
	find_actions_by_task(const pcmk_resource_t rsc, const char original_key)
	{
	// Search under given task key directly
	GList *list = find_actions(rsc->actions, original_key, NULL);

	if (list == NULL) {
	// Search again using this resource's ID
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	- if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
	- key = pcmk__op_key(rsc->id, task, interval_ms);
	- list = find_actions(rsc->actions, key, NULL);
	- free(key);
	- free(task);
	- } else {
	- crm_err("Invalid operation key (bug?): %s", original_key);
	- }
	+ CRM_CHECK(parse_op_key(original_key, NULL, &task, &interval_ms),
	+ return NULL);
	+ key = pcmk__op_key(rsc->id, task, interval_ms);
	+ list = find_actions(rsc->actions, key, NULL);
	+ free(key);
	+ free(task);
	}
	return list;
	}

	/*!
	* \internal
	* \brief Order relevant resource actions after a given action
	*
	* \param[in,out] first_action Action to order after (or NULL if none runnable)
	* \param[in] rsc Resource whose actions should be ordered
	* \param[in,out] order Ordering constraint being applied
	*/
	static void
	order_resource_actions_after(pcmk_action_t *first_action,
	const pcmk_resource_t *rsc,
	pcmk__action_relation_t *order)
	{
	GList *then_actions = NULL;
	uint32_t flags = pcmk__ar_none;

	CRM_CHECK((rsc != NULL) && (order != NULL), return);

	flags = order->flags;
	pcmk__rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
	order->id, rsc->id);

	if (order->action2 != NULL) {
	then_actions = g_list_prepend(NULL, order->action2);

	} else {
	then_actions = find_actions_by_task(rsc, order->task2);
	}

	if (then_actions == NULL) {
	pcmk__rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
	order->id, order->task2, rsc->id);
	return;
	}

	if ((first_action != NULL) && (first_action->rsc == rsc)
	&& pcmk_is_set(first_action->flags, pcmk_action_migration_abort)) {

	pcmk__rsc_trace(rsc,
	"Detected dangling migration ordering (%s then %s %s)",
	first_action->uuid, order->task2, rsc->id);
	pe__clear_order_flags(flags, pcmk__ar_first_implies_then);
	}

	if ((first_action == NULL)
	&& !pcmk_is_set(flags, pcmk__ar_first_implies_then)) {

	pcmk__rsc_debug(rsc,
	"Ignoring ordering %d for %s: No first action found",
	order->id, rsc->id);
	g_list_free(then_actions);
	return;
	}

	for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
	pcmk_action_t then_action_iter = (pcmk_action_t ) iter->data;

	if (first_action != NULL) {
	order_actions(first_action, then_action_iter, flags);
	} else {
	pe__clear_action_flags(then_action_iter, pcmk_action_runnable);
	crm_warn("%s of %s is unrunnable because there is no %s of %s "
	"to order it after", then_action_iter->task, rsc->id,
	order->task1, order->rsc1->id);
	}
	}

	g_list_free(then_actions);
	}

	static void
	rsc_order_first(pcmk_resource_t first_rsc, pcmk__action_relation_t order)
	{
	GList *first_actions = NULL;
	pcmk_action_t *first_action = order->action1;
	pcmk_resource_t *then_rsc = order->rsc2;

	CRM_ASSERT(first_rsc != NULL);
	pcmk__rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
	order->id, first_rsc->id);

	if (first_action != NULL) {
	first_actions = g_list_prepend(NULL, first_action);

	} else {
	first_actions = find_actions_by_task(first_rsc, order->task1);
	}

	if ((first_actions == NULL) && (first_rsc == then_rsc)) {
	pcmk__rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->task1, first_rsc->id);

	} else if (first_actions == NULL) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;

	parse_op_key(order->task1, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(first_rsc->id, op_type, interval_ms);

	if ((first_rsc->fns->state(first_rsc, TRUE) == pcmk_role_stopped)
	&& pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
	free(key);
	pcmk__rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) "
	"not found",
	order->id, order->task1, first_rsc->id);

	} else if ((first_rsc->fns->state(first_rsc,
	TRUE) == pcmk_role_unpromoted)
	&& pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	free(key);
	pcmk__rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) "
	"not found",
	order->id, order->task1, first_rsc->id);

	} else {
	pcmk__rsc_trace(first_rsc,
	"Creating first (%s for %s) for constraint %d ",
	order->task1, first_rsc->id, order->id);
	first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
	first_rsc->cluster);
	first_actions = g_list_prepend(NULL, first_action);
	}

	free(op_type);
	}

	if (then_rsc == NULL) {
	if (order->action2 == NULL) {
	pcmk__rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
	order->id);
	return;
	}
	then_rsc = order->action2->rsc;
	}
	for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
	first_action = iter->data;

	if (then_rsc == NULL) {
	order_actions(first_action, order->action2, order->flags);

	} else {
	order_resource_actions_after(first_action, then_rsc, order);
	}
	}

	g_list_free(first_actions);
	}

	// GFunc to call pcmk__block_colocation_dependents()
	static void
	block_colocation_dependents(gpointer data, gpointer user_data)
	{
	pcmk__block_colocation_dependents(data);
	}

	// GFunc to call pcmk__update_action_for_orderings()
	static void
	update_action_for_orderings(gpointer data, gpointer user_data)
	{
	pcmk__update_action_for_orderings((pcmk_action_t *) data,
	(pcmk_scheduler_t *) user_data);
	}

	/*!
	* \internal
	* \brief Apply all ordering constraints
	*
	* \param[in,out] sched Scheduler data
	*/
	void
	pcmk__apply_orderings(pcmk_scheduler_t *sched)
	{
	crm_trace("Applying ordering constraints");

	/* Ordering constraints need to be processed in the order they were created.
	* rsc_order_first() and order_resource_actions_after() require the relevant
	* actions to already exist in some cases, but rsc_order_first() will create
	* the 'first' action in certain cases. Thus calling rsc_order_first() can
	* change the behavior of later-created orderings.
	*
	* Also, g_list_append() should be avoided for performance reasons, so we
	* prepend orderings when creating them and reverse the list here.
	*
	* @TODO This is brittle and should be carefully redesigned so that the
	* order of creation doesn't matter, and the reverse becomes unneeded.
	*/
	sched->ordering_constraints = g_list_reverse(sched->ordering_constraints);

	for (GList *iter = sched->ordering_constraints;
	iter != NULL; iter = iter->next) {

	pcmk__action_relation_t *order = iter->data;
	pcmk_resource_t *rsc = order->rsc1;

	if (rsc != NULL) {
	rsc_order_first(rsc, order);
	continue;
	}

	rsc = order->rsc2;
	if (rsc != NULL) {
	order_resource_actions_after(order->action1, rsc, order);

	} else {
	crm_trace("Applying ordering constraint %d (non-resource actions)",
	order->id);
	order_actions(order->action1, order->action2, order->flags);
	}
	}

	g_list_foreach(sched->actions, block_colocation_dependents, NULL);

	crm_trace("Ordering probes");
	pcmk__order_probes(sched);

	crm_trace("Updating %d actions", g_list_length(sched->actions));
	g_list_foreach(sched->actions, update_action_for_orderings, sched);

	pcmk__disable_invalid_orderings(sched);
	}

	/*!
	* \internal
	* \brief Order a given action after each action in a given list
	*
	* \param[in,out] after "After" action
	* \param[in,out] list List of "before" actions
	*/
	void
	pcmk__order_after_each(pcmk_action_t after, GList list)
	{
	const char *after_desc = (after->task == NULL)? after->uuid : after->task;

	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk_action_t before = (pcmk_action_t ) iter->data;
	const char *before_desc = before->task? before->task : before->uuid;

	crm_debug("Ordering %s on %s before %s on %s",
	before_desc, pe__node_name(before->node),
	after_desc, pe__node_name(after->node));
	order_actions(before, after, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Order promotions and demotions for restarts of a clone or bundle
	*
	* \param[in,out] rsc Clone or bundle to order
	*/
	void
	pcmk__promotable_restart_ordering(pcmk_resource_t *rsc)
	{
	// Order start and promote after all instances are stopped
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order stop, start, and promote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order promote after all instances are started
	pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order demote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
	rsc, PCMK_ACTION_DEMOTED,
	pcmk__ar_ordered);
	}
	diff --git a/lib/pacemaker/pcmk_sched_probes.c b/lib/pacemaker/pcmk_sched_probes.c
	index d9c6536794..4df13d98d8 100644
	--- a/lib/pacemaker/pcmk_sched_probes.c
	+++ b/lib/pacemaker/pcmk_sched_probes.c
	@@ -1,903 +1,903 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add the expected result to a newly created probe
	*
	* \param[in,out] probe Probe action to add expected result to
	* \param[in] rsc Resource that probe is for
	* \param[in] node Node that probe will run on
	*/
	static void
	add_expected_result(pcmk_action_t probe, const pcmk_resource_t rsc,
	const pcmk_node_t *node)
	{
	// Check whether resource is currently active on node
	pcmk_node_t *running = pe_find_node_id(rsc->running_on, node->details->id);

	// The expected result is what we think the resource's current state is
	if (running == NULL) {
	pe__add_action_expected_result(probe, CRM_EX_NOT_RUNNING);

	} else if (rsc->role == pcmk_role_promoted) {
	pe__add_action_expected_result(probe, CRM_EX_PROMOTED);
	}
	}

	/*!
	* \internal
	* \brief Create any needed robes on a node for a list of resources
	*
	* \param[in,out] rscs List of resources to create probes for
	* \param[in,out] node Node to create probes on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_resource_list(GList rscs, pcmk_node_t node)
	{
	bool any_created = false;

	for (GList *iter = rscs; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->cmds->create_probe(rsc, node)) {
	any_created = true;
	}
	}
	return any_created;
	}

	/*!
	* \internal
	* \brief Order one resource's start after another's start-up probe
	*
	* \param[in,out] rsc1 Resource that might get start-up probe
	* \param[in] rsc2 Resource that might be started
	*/
	static void
	probe_then_start(pcmk_resource_t rsc1, pcmk_resource_t rsc2)
	{
	if ((rsc1->allocated_to != NULL)
	&& (g_hash_table_lookup(rsc1->known_on,
	rsc1->allocated_to->details->id) == NULL)) {

	pcmk__new_ordering(rsc1,
	pcmk__op_key(rsc1->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	rsc2, pcmk__op_key(rsc2->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_ordered, rsc1->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether a guest resource will stop
	*
	* \param[in] node Guest node to check
	*
	* \return true if guest resource will likely stop, otherwise false
	*/
	static bool
	guest_resource_will_stop(const pcmk_node_t *node)
	{
	const pcmk_resource_t *guest_rsc = node->details->remote_rsc->container;

	/* Ideally, we'd check whether the guest has a required stop, but that
	* information doesn't exist yet, so approximate it ...
	*/
	return node->details->remote_requires_reset
	\|\| node->details->unclean
	\|\| pcmk_is_set(guest_rsc->flags, pcmk_rsc_failed)
	\|\| (guest_rsc->next_role == pcmk_role_stopped)

	// Guest is moving
	\|\| ((guest_rsc->role > pcmk_role_stopped)
	&& (guest_rsc->allocated_to != NULL)
	&& (pe_find_node(guest_rsc->running_on,
	guest_rsc->allocated_to->details->uname) == NULL));
	}

	/*!
	* \internal
	* \brief Create a probe action for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return Newly created probe action
	*/
	static pcmk_action_t *
	probe_action(pcmk_resource_t rsc, pcmk_node_t node)
	{
	pcmk_action_t *probe = NULL;
	char *key = pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0);

	crm_debug("Scheduling probe of %s %s on %s",
	role2text(rsc->role), rsc->id, pe__node_name(node));

	probe = custom_action(rsc, key, PCMK_ACTION_MONITOR, node, FALSE,
	rsc->cluster);
	pe__clear_action_flags(probe, pcmk_action_optional);

	pcmk__order_vs_unfence(rsc, node, probe, pcmk__ar_ordered);
	add_expected_result(probe, rsc, node);
	return probe;
	}

	/*!
	* \internal
	* \brief Create probes for a resource on a node, if needed
	*
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_rsc_on_node(pcmk_resource_t rsc, pcmk_node_t node)
	{
	uint32_t flags = pcmk__ar_ordered;
	pcmk_action_t *probe = NULL;
	pcmk_node_t *allowed = NULL;
	pcmk_resource_t *top = uber_parent(rsc);
	const char *reason = NULL;

	CRM_ASSERT((rsc != NULL) && (node != NULL));

	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_probe_resources)) {
	reason = "start-up probes are disabled";
	goto no_probe;
	}

	if (pe__is_guest_or_remote_node(node)) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_none)) {
	reason = "Pacemaker Remote nodes cannot run stonith agents";
	goto no_probe;

	} else if (pe__is_guest_node(node)
	&& pe__resource_contains_guest_node(rsc->cluster, rsc)) {
	reason = "guest nodes cannot run resources containing guest nodes";
	goto no_probe;

	} else if (rsc->is_remote_node) {
	reason = "Pacemaker Remote nodes cannot host remote connections";
	goto no_probe;
	}
	}

	// If this is a collective resource, probes are created for its children
	if (rsc->children != NULL) {
	return pcmk__probe_resource_list(rsc->children, node);
	}

	if ((rsc->container != NULL) && !rsc->is_remote_node) {
	reason = "resource is inside a container";
	goto no_probe;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	reason = "resource is orphaned";
	goto no_probe;

	} else if (g_hash_table_lookup(rsc->known_on, node->details->id) != NULL) {
	reason = "resource state is already known";
	goto no_probe;
	}

	allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (rsc->exclusive_discover \|\| top->exclusive_discover) {
	// Exclusive discovery is enabled ...

	if (allowed == NULL) {
	// ... but this node is not allowed to run the resource
	reason = "resource has exclusive discovery but is not allowed "
	"on node";
	goto no_probe;

	} else if (allowed->rsc_discover_mode != pcmk_probe_exclusive) {
	// ... but no constraint marks this node for discovery of resource
	reason = "resource has exclusive discovery but is not enabled "
	"on node";
	goto no_probe;
	}
	}

	if (allowed == NULL) {
	allowed = node;
	}
	if (allowed->rsc_discover_mode == pcmk_probe_never) {
	reason = "node has discovery disabled";
	goto no_probe;
	}

	if (pe__is_guest_node(node)) {
	pcmk_resource_t *guest = node->details->remote_rsc->container;

	if (guest->role == pcmk_role_stopped) {
	// The guest is stopped, so we know no resource is active there
	reason = "node's guest is stopped";
	probe_then_start(guest, top);
	goto no_probe;

	} else if (guest_resource_will_stop(node)) {
	reason = "node's guest will stop";

	// Order resource start after guest stop (in case it's restarting)
	pcmk__new_ordering(guest,
	pcmk__op_key(guest->id, PCMK_ACTION_STOP, 0),
	NULL, top,
	pcmk__op_key(top->id, PCMK_ACTION_START, 0),
	NULL, pcmk__ar_ordered, rsc->cluster);
	goto no_probe;
	}
	}

	// We've eliminated all cases where a probe is not needed, so now it is
	probe = probe_action(rsc, node);

	/* Below, we will order the probe relative to start or reload. If this is a
	* clone instance, the start or reload is for the entire clone rather than
	* just the instance. Otherwise, the start or reload is for the resource
	* itself.
	*/
	if (!pe_rsc_is_clone(top)) {
	top = rsc;
	}

	/* Prevent a start if the resource can't be probed, but don't cause the
	* resource or entire clone to stop if already active.
	*/
	if (!pcmk_is_set(probe->flags, pcmk_action_runnable)
	&& (top->running_on == NULL)) {
	pe__set_order_flags(flags, pcmk__ar_unrunnable_first_blocks);
	}

	// Start or reload after probing the resource
	pcmk__new_ordering(rsc, NULL, probe,
	top, pcmk__op_key(top->id, PCMK_ACTION_START, 0), NULL,
	flags, rsc->cluster);
	pcmk__new_ordering(rsc, NULL, probe, top, reload_key(rsc), NULL,
	pcmk__ar_ordered, rsc->cluster);

	return true;

	no_probe:
	pcmk__rsc_trace(rsc,
	"Skipping probe for %s on %s because %s",
	rsc->id, node->details->id, reason);
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a probe should be ordered before another action
	*
	* \param[in] probe Probe action to check
	* \param[in] then Other action to check
	*
	* \return true if \p probe should be ordered before \p then, otherwise false
	*/
	static bool
	probe_needed_before_action(const pcmk_action_t *probe,
	const pcmk_action_t *then)
	{
	// Probes on a node are performed after unfencing it, not before
	if (pcmk__str_eq(then->task, PCMK_ACTION_STONITH, pcmk__str_none)
	&& pe__same_node(probe->node, then->node)) {
	const char *op = g_hash_table_lookup(then->meta, "stonith_action");

	if (pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	return false;
	}
	}

	// Probes should be done on a node before shutting it down
	if (pcmk__str_eq(then->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)
	&& (probe->node != NULL) && (then->node != NULL)
	&& !pe__same_node(probe->node, then->node)) {
	return false;
	}

	// Otherwise probes should always be done before any other action
	return true;
	}

	/*!
	* \internal
	* \brief Add implicit "probe then X" orderings for "stop then X" orderings
	*
	* If the state of a resource is not known yet, a probe will be scheduled,
	* expecting a "not running" result. If the probe fails, a stop will not be
	* scheduled until the next transition. Thus, if there are ordering constraints
	* like "stop this resource then do something else that's not for the same
	* resource", add implicit "probe this resource then do something" equivalents
	* so the relation is upheld until we know whether a stop is needed.
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	add_probe_orderings_for_stops(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->ordering_constraints; iter != NULL;
	iter = iter->next) {

	pcmk__action_relation_t *order = iter->data;
	uint32_t order_flags = pcmk__ar_ordered;
	GList *probes = NULL;
	GList *then_actions = NULL;
	pcmk_action_t *first = NULL;
	pcmk_action_t *then = NULL;

	// Skip disabled orderings
	if (order->flags == pcmk__ar_none) {
	continue;
	}

	// Skip non-resource orderings, and orderings for the same resource
	if ((order->rsc1 == NULL) \|\| (order->rsc1 == order->rsc2)) {
	continue;
	}

	// Skip invalid orderings (shouldn't be possible)
	first = order->action1;
	then = order->action2;
	if (((first == NULL) && (order->task1 == NULL))
	\|\| ((then == NULL) && (order->task2 == NULL))) {
	continue;
	}

	// Skip orderings for first actions other than stop
	if ((first != NULL) && !pcmk__str_eq(first->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((first == NULL)
	&& !pcmk__ends_with(order->task1,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}

	/* Do not imply a probe ordering for a resource inside of a stopping
	* container. Otherwise, it might introduce a transition loop, since a
	* probe could be scheduled after the container starts again.
	*/
	if ((order->rsc2 != NULL) && (order->rsc1->container == order->rsc2)) {

	if ((then != NULL) && pcmk__str_eq(then->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((then == NULL)
	&& pcmk__ends_with(order->task2,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}
	}

	// Preserve certain order options for future filtering
	if (pcmk_is_set(order->flags, pcmk__ar_if_first_unmigratable)) {
	pe__set_order_flags(order_flags, pcmk__ar_if_first_unmigratable);
	}
	if (pcmk_is_set(order->flags, pcmk__ar_if_on_same_node)) {
	pe__set_order_flags(order_flags, pcmk__ar_if_on_same_node);
	}

	// Preserve certain order types for future filtering
	if ((order->flags == pcmk__ar_if_required_on_same_node)
	\|\| (order->flags == pcmk__ar_if_on_same_node_or_target)) {
	order_flags = order->flags;
	}

	// List all scheduled probes for the first resource
	probes = pe__resource_actions(order->rsc1, NULL, PCMK_ACTION_MONITOR,
	FALSE);
	if (probes == NULL) { // There aren't any
	continue;
	}

	// List all relevant "then" actions
	if (then != NULL) {
	then_actions = g_list_prepend(NULL, then);

	} else if (order->rsc2 != NULL) {
	then_actions = find_actions(order->rsc2->actions, order->task2,
	NULL);
	if (then_actions == NULL) { // There aren't any
	g_list_free(probes);
	continue;
	}
	}

	crm_trace("Implying 'probe then' orderings for '%s then %s' "
	"(id=%d, type=%.6x)",
	((first == NULL)? order->task1 : first->uuid),
	((then == NULL)? order->task2 : then->uuid),
	order->id, order->flags);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	for (GList *then_iter = then_actions; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk_action_t then = (pcmk_action_t ) then_iter->data;

	if (probe_needed_before_action(probe, then)) {
	order_actions(probe, then, order_flags);
	}
	}
	}

	g_list_free(then_actions);
	g_list_free(probes);
	}
	}

	/*!
	* \internal
	* \brief Add necessary orderings between probe and starts of clone instances
	*
	* , in additon to the ordering with the parent resource added upon creating
	* the probe.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action wrapper in the ordering
	*/
	static void
	add_start_orderings_for_probe(pcmk_action_t *probe,
	pcmk__related_action_t *after)
	{
	uint32_t flags = pcmk__ar_ordered\|pcmk__ar_unrunnable_first_blocks;

	/* Although the ordering between the probe of the clone instance and the
	* start of its parent has been added in pcmk__probe_rsc_on_node(), we
	* avoided enforcing `pcmk__ar_unrunnable_first_blocks` order type for that
	* as long as any of the clone instances are running to prevent them from
	* being unexpectedly stopped.
	*
	* On the other hand, we still need to prevent any inactive instances from
	* starting unless the probe is runnable so that we don't risk starting too
	* many instances before we know the state on all nodes.
	*/
	if ((after->action->rsc->variant <= pcmk_rsc_variant_group)
	\|\| pcmk_is_set(probe->flags, pcmk_action_runnable)
	// The order type is already enforced for its parent.
	\|\| pcmk_is_set(after->type, pcmk__ar_unrunnable_first_blocks)
	\|\| (pe__const_top_resource(probe->rsc, false) != after->action->rsc)
	\|\| !pcmk__str_eq(after->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	return;
	}

	crm_trace("Adding probe start orderings for 'unrunnable %s@%s "
	"then instances of %s@%s'",
	probe->uuid, pe__node_name(probe->node),
	after->action->uuid, pe__node_name(after->action->node));

	for (GList *then_iter = after->action->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	if (then->action->rsc->running_on
	\|\| (pe__const_top_resource(then->action->rsc, false)
	!= after->action->rsc)
	\|\| !pcmk__str_eq(then->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	continue;
	}

	crm_trace("Adding probe start ordering for 'unrunnable %s@%s "
	"then %s@%s' (type=%#.6x)",
	probe->uuid, pe__node_name(probe->node),
	then->action->uuid, pe__node_name(then->action->node), flags);

	/* Prevent the instance from starting if the instance can't, but don't
	* cause any other intances to stop if already active.
	*/
	order_actions(probe, then->action, flags);
	}

	return;
	}

	/*!
	* \internal
	* \brief Order probes before restarts and re-promotes
	*
	* If a given ordering is a "probe then start" or "probe then promote" ordering,
	* add an implicit "probe then stop/demote" ordering in case the action is part
	* of a restart/re-promote, and do the same recursively for all actions ordered
	* after the "then" action.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action in the ordering
	*/
	static void
	add_restart_orderings_for_probe(pcmk_action_t probe, pcmk_action_t after)
	{
	GList *iter = NULL;
	bool interleave = false;
	pcmk_resource_t *compatible_rsc = NULL;

	// Validate that this is a resource probe followed by some action
	if ((after == NULL) \|\| (probe == NULL) \|\| (probe->rsc == NULL)
	\|\| (probe->rsc->variant != pcmk_rsc_variant_primitive)
	\|\| !pcmk__str_eq(probe->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	return;
	}

	// Avoid running into any possible loop
	if (pcmk_is_set(after->flags, pcmk_action_detect_loop)) {
	return;
	}
	pe__set_action_flags(after, pcmk_action_detect_loop);

	crm_trace("Adding probe restart orderings for '%s@%s then %s@%s'",
	probe->uuid, pe__node_name(probe->node),
	after->uuid, pe__node_name(after->node));

	/* Add restart orderings if "then" is for a different primitive.
	* Orderings for collective resources will be added later.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant == pcmk_rsc_variant_primitive)
	&& (probe->rsc != after->rsc)) {

	GList *then_actions = NULL;

	if (pcmk__str_eq(after->task, PCMK_ACTION_START, pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_STOP, FALSE);

	} else if (pcmk__str_eq(after->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_DEMOTE, FALSE);
	}

	for (iter = then_actions; iter != NULL; iter = iter->next) {
	pcmk_action_t then = (pcmk_action_t ) iter->data;

	// Skip pseudo-actions (for example, those implied by fencing)
	if (!pcmk_is_set(then->flags, pcmk_action_pseudo)) {
	order_actions(probe, then, pcmk__ar_ordered);
	}
	}
	g_list_free(then_actions);
	}

	/* Detect whether "then" is an interleaved clone action. For these, we want
	* to add orderings only for the relevant instance.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant > pcmk_rsc_variant_group)) {
	const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
	XML_RSC_ATTR_INTERLEAVE);

	interleave = crm_is_true(interleave_s);
	if (interleave) {
	compatible_rsc = pcmk__find_compatible_instance(probe->rsc,
	after->rsc,
	pcmk_role_unknown,
	false);
	}
	}

	/* Now recursively do the same for all actions ordered after "then". This
	* also handles collective resources since the collective action will be
	* ordered before its individual instances' actions.
	*/
	for (iter = after->actions_after; iter != NULL; iter = iter->next) {
	pcmk__related_action_t *after_wrapper = iter->data;

	/* pcmk__ar_first_implies_then is the reason why a required A.start
	* implies/enforces B.start to be required too, which is the cause of
	* B.restart/re-promote.
	*
	* Not sure about pcmk__ar_first_implies_same_node_then though. It's now
	* only used for unfencing case, which tends to introduce transition
	* loops...
	*/
	if (!pcmk_is_set(after_wrapper->type, pcmk__ar_first_implies_then)) {
	/* The order type between a group/clone and its child such as
	* B.start-> B_child.start is:
	* pcmk__ar_then_implies_first_graphed
	* \|pcmk__ar_unrunnable_first_blocks
	*
	* Proceed through the ordering chain and build dependencies with
	* its children.
	*/
	if ((after->rsc == NULL)
	\|\| (after->rsc->variant < pcmk_rsc_variant_group)
	\|\| (probe->rsc->parent == after->rsc)
	\|\| (after_wrapper->action->rsc == NULL)
	\|\| (after_wrapper->action->rsc->variant > pcmk_rsc_variant_group)
	\|\| (after->rsc != after_wrapper->action->rsc->parent)) {
	continue;
	}

	/* Proceed to the children of a group or a non-interleaved clone.
	* For an interleaved clone, proceed only to the relevant child.
	*/
	if ((after->rsc->variant > pcmk_rsc_variant_group) && interleave
	&& ((compatible_rsc == NULL)
	\|\| (compatible_rsc != after_wrapper->action->rsc))) {
	continue;
	}
	}

	crm_trace("Recursively adding probe restart orderings for "
	"'%s@%s then %s@%s' (type=%#.6x)",
	after->uuid, pe__node_name(after->node),
	after_wrapper->action->uuid,
	pe__node_name(after_wrapper->action->node),
	after_wrapper->type);

	add_restart_orderings_for_probe(probe, after_wrapper->action);
	}
	}

	/*!
	* \internal
	* \brief Clear the tracking flag on all scheduled actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	clear_actions_tracking_flag(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t *action = iter->data;

	pe__clear_action_flags(action, pcmk_action_detect_loop);
	}
	}

	/*!
	* \internal
	* \brief Add start and restart orderings for probes scheduled for a resource
	*
	* \param[in,out] data Resource whose probes should be ordered
	* \param[in] user_data Unused
	*/
	static void
	add_start_restart_orderings_for_rsc(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	GList *probes = NULL;

	// For collective resources, order each instance recursively
	if (rsc->variant != pcmk_rsc_variant_primitive) {
	g_list_foreach(rsc->children, add_start_restart_orderings_for_rsc,
	NULL);
	return;
	}

	// Find all probes for given resource
	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	// Add probe restart orderings for each probe found
	for (GList *iter = probes; iter != NULL; iter = iter->next) {
	pcmk_action_t probe = (pcmk_action_t ) iter->data;

	for (GList *then_iter = probe->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	add_start_orderings_for_probe(probe, then);
	add_restart_orderings_for_probe(probe, then->action);
	clear_actions_tracking_flag(rsc->cluster);
	}
	}

	g_list_free(probes);
	}

	/*!
	* \internal
	* \brief Add "A then probe B" orderings for "A then B" orderings
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This function is currently disabled (see next comment).
	*/
	static void
	order_then_probes(pcmk_scheduler_t *scheduler)
	{
	#if 0
	/* Given an ordering "A then B", we would prefer to wait for A to be started
	* before probing B.
	*
	* For example, if A is a filesystem which B can't even run without, it
	* would be helpful if the author of B's agent could assume that A is
	* running before B.monitor will be called.
	*
	* However, we can't _only_ probe after A is running, otherwise we wouldn't
	* detect the state of B if A could not be started. We can't even do an
	* opportunistic version of this, because B may be moving:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> B.start
	*
	* and if we add B.stop -> A.stop here, we get a loop:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> A.stop
	*
	* We could kill the "B.probe -> B.stop" dependency, but that could mean
	* stopping B "too" soon, because B.start must wait for the probe, and
	* we don't want to stop B if we can't start it.
	*
	* We could add the ordering only if A is an anonymous clone with
	* clone-max == node-max (since we'll never be moving it). However, we could
	* still be stopping one instance at the same time as starting another.
	*
	* The complexity of checking for allowed conditions combined with the ever
	* narrowing use case suggests that this code should remain disabled until
	* someone gets smarter.
	*/
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	pcmk_action_t *start = NULL;
	GList *actions = NULL;
	GList *probes = NULL;

	actions = pe__resource_actions(rsc, NULL, PCMK_ACTION_START, FALSE);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if (start == NULL) {
	- crm_err("No start action for %s", rsc->id);
	+ crm_debug("No start action for %s", rsc->id);
	continue;
	}

	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	for (actions = start->actions_before; actions != NULL;
	actions = actions->next) {

	pcmk__related_action_t *before = actions->data;

	pcmk_action_t *first = before->action;
	pcmk_resource_t *first_rsc = first->rsc;

	if (first->required_runnable_before) {
	for (GList *clone_actions = first->actions_before;
	clone_actions != NULL;
	clone_actions = clone_actions->next) {

	before = clone_actions->data;

	crm_trace("Testing '%s then %s' for %s",
	first->uuid, before->action->uuid, start->uuid);

	CRM_ASSERT(before->action->rsc != NULL);
	first_rsc = before->action->rsc;
	break;
	}

	} else if (!pcmk__str_eq(first->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if (first_rsc == NULL) {
	continue;

	} else if (pe__const_top_resource(first_rsc, false)
	== pe__const_top_resource(start->rsc, false)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if (!pe_rsc_is_clone(pe__const_top_resource(first_rsc,
	false))) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	- crm_err("Applying %s before %s %d", first->uuid, start->uuid,
	- pe__const_top_resource(first_rsc, false)->variant);
	+ crm_debug("Applying %s before %s %d", first->uuid, start->uuid,
	+ pe__const_top_resource(first_rsc, false)->variant);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	- crm_err("Ordering %s before %s", first->uuid, probe->uuid);
	+ crm_debug("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pcmk__ar_ordered);
	}
	}
	}
	#endif
	}

	void
	pcmk__order_probes(pcmk_scheduler_t *scheduler)
	{
	// Add orderings for "probe then X"
	g_list_foreach(scheduler->resources, add_start_restart_orderings_for_rsc,
	NULL);
	add_probe_orderings_for_stops(scheduler);

	order_then_probes(scheduler);
	}

	/*!
	* \internal
	* \brief Schedule any probes needed
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This may also schedule fencing of failed remote nodes.
	*/
	void
	pcmk__schedule_probes(pcmk_scheduler_t *scheduler)
	{
	// Schedule probes on each node in the cluster as needed
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	const char *probed = NULL;

	if (!node->details->online) { // Don't probe offline nodes
	if (pcmk__is_failed_remote_node(node)) {
	pe_fence_node(scheduler, node,
	"the connection is unrecoverable", FALSE);
	}
	continue;

	} else if (node->details->unclean) { // ... or nodes that need fencing
	continue;

	} else if (!node->details->rsc_discovery_enabled) {
	// The user requested that probes not be done on this node
	continue;
	}

	/* This is no longer needed for live clusters, since the probe_complete
	* node attribute will never be in the CIB. However this is still useful
	* for processing old saved CIBs (< 1.1.14), including the
	* reprobe-target_rc regression test.
	*/
	probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
	if (probed != NULL && crm_is_true(probed) == FALSE) {
	pcmk_action_t *probe_op = NULL;

	probe_op = custom_action(NULL,
	crm_strdup_printf("%s-%s", CRM_OP_REPROBE,
	node->details->uname),
	CRM_OP_REPROBE, node, FALSE, scheduler);
	add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT,
	XML_BOOLEAN_TRUE);
	continue;
	}

	// Probe each resource in the cluster on this node, as needed
	pcmk__probe_resource_list(scheduler->resources, node);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_promotable.c b/lib/pacemaker/pcmk_sched_promotable.c
	index 23133cc8b2..6b820f0e0d 100644
	--- a/lib/pacemaker/pcmk_sched_promotable.c
	+++ b/lib/pacemaker/pcmk_sched_promotable.c
	@@ -1,1312 +1,1312 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add implicit promotion ordering for a promotable instance
	*
	* \param[in,out] clone Clone resource
	* \param[in,out] child Instance of \p clone being ordered
	* \param[in,out] last Previous instance ordered (NULL if \p child is first)
	*/
	static void
	order_instance_promotion(pcmk_resource_t clone, pcmk_resource_t child,
	pcmk_resource_t *last)
	{
	// "Promote clone" -> promote instance -> "clone promoted"
	pcmk__order_resource_actions(clone, PCMK_ACTION_PROMOTE,
	child, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(child, PCMK_ACTION_PROMOTE,
	clone, PCMK_ACTION_PROMOTED,
	pcmk__ar_ordered);

	// If clone is ordered, order this instance relative to last
	if ((last != NULL) && pe__clone_is_ordered(clone)) {
	pcmk__order_resource_actions(last, PCMK_ACTION_PROMOTE,
	child, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Add implicit demotion ordering for a promotable instance
	*
	* \param[in,out] clone Clone resource
	* \param[in,out] child Instance of \p clone being ordered
	* \param[in] last Previous instance ordered (NULL if \p child is first)
	*/
	static void
	order_instance_demotion(pcmk_resource_t clone, pcmk_resource_t child,
	pcmk_resource_t *last)
	{
	// "Demote clone" -> demote instance -> "clone demoted"
	pcmk__order_resource_actions(clone, PCMK_ACTION_DEMOTE, child,
	PCMK_ACTION_DEMOTE,
	pcmk__ar_then_implies_first_graphed);
	pcmk__order_resource_actions(child, PCMK_ACTION_DEMOTE,
	clone, PCMK_ACTION_DEMOTED,
	pcmk__ar_first_implies_then_graphed);

	// If clone is ordered, order this instance relative to last
	if ((last != NULL) && pe__clone_is_ordered(clone)) {
	pcmk__order_resource_actions(child, PCMK_ACTION_DEMOTE, last,
	PCMK_ACTION_DEMOTE, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Check whether an instance will be promoted or demoted
	*
	* \param[in] rsc Instance to check
	* \param[out] demoting If \p rsc will be demoted, this will be set to true
	* \param[out] promoting If \p rsc will be promoted, this will be set to true
	*/
	static void
	check_for_role_change(const pcmk_resource_t rsc, bool demoting,
	bool *promoting)
	{
	const GList *iter = NULL;

	// If this is a cloned group, check group members recursively
	if (rsc->children != NULL) {
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	check_for_role_change((const pcmk_resource_t *) iter->data,
	demoting, promoting);
	}
	return;
	}

	for (iter = rsc->actions; iter != NULL; iter = iter->next) {
	const pcmk_action_t action = (const pcmk_action_t ) iter->data;

	if (promoting && demoting) {
	return;

	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	continue;

	} else if (pcmk__str_eq(PCMK_ACTION_DEMOTE, action->task,
	pcmk__str_none)) {
	*demoting = true;

	} else if (pcmk__str_eq(PCMK_ACTION_PROMOTE, action->task,
	pcmk__str_none)) {
	*promoting = true;
	}
	}
	}

	/*!
	* \internal
	* \brief Add promoted-role location constraint scores to an instance's priority
	*
	* Adjust a promotable clone instance's promotion priority by the scores of any
	* location constraints in a list that are both limited to the promoted role and
	* for the node where the instance will be placed.
	*
	* \param[in,out] child Promotable clone instance
	* \param[in] location_constraints List of location constraints to apply
	* \param[in] chosen Node where \p child will be placed
	*/
	static void
	apply_promoted_locations(pcmk_resource_t *child,
	const GList *location_constraints,
	const pcmk_node_t *chosen)
	{
	for (const GList *iter = location_constraints; iter; iter = iter->next) {
	const pcmk__location_t *location = iter->data;
	const pcmk_node_t *constraint_node = NULL;

	if (location->role_filter == pcmk_role_promoted) {
	constraint_node = pe_find_node_id(location->nodes,
	chosen->details->id);
	}
	if (constraint_node != NULL) {
	int new_priority = pcmk__add_scores(child->priority,
	constraint_node->weight);

	pcmk__rsc_trace(child,
	"Applying location %s to %s promotion priority on "
	"%s: %s + %s = %s",
	location->id, child->id,
	pe__node_name(constraint_node),
	pcmk_readable_score(child->priority),
	pcmk_readable_score(constraint_node->weight),
	pcmk_readable_score(new_priority));
	child->priority = new_priority;
	}
	}
	}

	/*!
	* \internal
	* \brief Get the node that an instance will be promoted on
	*
	* \param[in] rsc Promotable clone instance to check
	*
	* \return Node that \p rsc will be promoted on, or NULL if none
	*/
	static pcmk_node_t *
	node_to_be_promoted_on(const pcmk_resource_t *rsc)
	{
	pcmk_node_t *node = NULL;
	pcmk_node_t *local_node = NULL;
	const pcmk_resource_t *parent = NULL;

	// If this is a cloned group, bail if any group member can't be promoted
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	if (node_to_be_promoted_on(child) == NULL) {
	pcmk__rsc_trace(rsc,
	"%s can't be promoted because member %s can't",
	rsc->id, child->id);
	return NULL;
	}
	}

	node = rsc->fns->location(rsc, NULL, FALSE);
	if (node == NULL) {
	pcmk__rsc_trace(rsc, "%s can't be promoted because it won't be active",
	rsc->id);
	return NULL;

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	if (rsc->fns->state(rsc, TRUE) == pcmk_role_promoted) {
	crm_notice("Unmanaged instance %s will be left promoted on %s",
	rsc->id, pe__node_name(node));
	} else {
	pcmk__rsc_trace(rsc, "%s can't be promoted because it is unmanaged",
	rsc->id);
	return NULL;
	}

	} else if (rsc->priority < 0) {
	pcmk__rsc_trace(rsc,
	"%s can't be promoted because its promotion priority "
	"%d is negative",
	rsc->id, rsc->priority);
	return NULL;

	} else if (!pcmk__node_available(node, false, true)) {
	pcmk__rsc_trace(rsc,
	"%s can't be promoted because %s can't run resources",
	rsc->id, pe__node_name(node));
	return NULL;
	}

	parent = pe__const_top_resource(rsc, false);
	local_node = g_hash_table_lookup(parent->allowed_nodes, node->details->id);

	if (local_node == NULL) {
	/* It should not be possible for the scheduler to have assigned the
	* instance to a node where its parent is not allowed, but it's good to
	* have a fail-safe.
	*/
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	- crm_warn("%s can't be promoted because %s is not allowed on %s "
	- "(scheduler bug?)",
	- rsc->id, parent->id, pe__node_name(node));
	+ pcmk__sched_err("%s can't be promoted because %s is not allowed "
	+ "on %s (scheduler bug?)",
	+ rsc->id, parent->id, pe__node_name(node));
	} // else the instance is unmanaged and already promoted
	return NULL;

	} else if ((local_node->count >= pe__clone_promoted_node_max(parent))
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__rsc_trace(rsc,
	"%s can't be promoted because %s has "
	"maximum promoted instances already",
	rsc->id, pe__node_name(node));
	return NULL;
	}

	return local_node;
	}

	/*!
	* \internal
	* \brief Compare two promotable clone instances by promotion priority
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a has higher promotion priority,
	* a positive number if \p b has higher promotion priority,
	* or 0 if promotion priorities are equal
	*/
	static gint
	cmp_promotable_instance(gconstpointer a, gconstpointer b)
	{
	const pcmk_resource_t rsc1 = (const pcmk_resource_t ) a;
	const pcmk_resource_t rsc2 = (const pcmk_resource_t ) b;

	enum rsc_role_e role1 = pcmk_role_unknown;
	enum rsc_role_e role2 = pcmk_role_unknown;

	CRM_ASSERT((rsc1 != NULL) && (rsc2 != NULL));

	// Check sort index set by pcmk__set_instance_roles()
	if (rsc1->sort_index > rsc2->sort_index) {
	pcmk__rsc_trace(rsc1,
	"%s has higher promotion priority than %s "
	"(sort index %d > %d)",
	rsc1->id, rsc2->id, rsc1->sort_index, rsc2->sort_index);
	return -1;
	} else if (rsc1->sort_index < rsc2->sort_index) {
	pcmk__rsc_trace(rsc1,
	"%s has lower promotion priority than %s "
	"(sort index %d < %d)",
	rsc1->id, rsc2->id, rsc1->sort_index, rsc2->sort_index);
	return 1;
	}

	// If those are the same, prefer instance whose current role is higher
	role1 = rsc1->fns->state(rsc1, TRUE);
	role2 = rsc2->fns->state(rsc2, TRUE);
	if (role1 > role2) {
	pcmk__rsc_trace(rsc1,
	"%s has higher promotion priority than %s "
	"(higher current role)",
	rsc1->id, rsc2->id);
	return -1;
	} else if (role1 < role2) {
	pcmk__rsc_trace(rsc1,
	"%s has lower promotion priority than %s "
	"(lower current role)",
	rsc1->id, rsc2->id);
	return 1;
	}

	// Finally, do normal clone instance sorting
	return pcmk__cmp_instance(a, b);
	}

	/*!
	* \internal
	* \brief Add a promotable clone instance's sort index to its node's score
	*
	* Add a promotable clone instance's sort index (which sums its promotion
	* preferences and scores of relevant location constraints for the promoted
	* role) to the node score of the instance's assigned node.
	*
	* \param[in] data Promotable clone instance
	* \param[in,out] user_data Clone parent of \p data
	*/
	static void
	add_sort_index_to_node_score(gpointer data, gpointer user_data)
	{
	const pcmk_resource_t child = (const pcmk_resource_t ) data;
	pcmk_resource_t clone = (pcmk_resource_t ) user_data;

	pcmk_node_t *node = NULL;
	const pcmk_node_t *chosen = NULL;

	if (child->sort_index < 0) {
	pcmk__rsc_trace(clone, "Not adding sort index of %s: negative",
	child->id);
	return;
	}

	chosen = child->fns->location(child, NULL, FALSE);
	if (chosen == NULL) {
	pcmk__rsc_trace(clone, "Not adding sort index of %s: inactive",
	child->id);
	return;
	}

	node = g_hash_table_lookup(clone->allowed_nodes, chosen->details->id);
	CRM_ASSERT(node != NULL);

	node->weight = pcmk__add_scores(child->sort_index, node->weight);
	pcmk__rsc_trace(clone,
	"Added cumulative priority of %s (%s) to score on %s "
	"(now %s)",
	child->id, pcmk_readable_score(child->sort_index),
	pe__node_name(node), pcmk_readable_score(node->weight));
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent's node scores if for promoted role
	*
	* \param[in,out] data Colocation constraint to apply
	* \param[in,out] user_data Promotable clone that is constraint's dependent
	*/
	static void
	apply_coloc_to_dependent(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t *colocation = data;
	pcmk_resource_t *clone = user_data;
	pcmk_resource_t *primary = colocation->primary;
	uint32_t flags = pcmk__coloc_select_default;
	float factor = colocation->score / (float) INFINITY;

	if (colocation->dependent_role != pcmk_role_promoted) {
	return;
	}
	if (colocation->score < INFINITY) {
	flags = pcmk__coloc_select_active;
	}
	pcmk__rsc_trace(clone, "Applying colocation %s (promoted %s with %s) @%s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id,
	pcmk_readable_score(colocation->score));
	primary->cmds->add_colocated_node_scores(primary, clone, clone->id,
	&clone->allowed_nodes, colocation,
	factor, flags);
	}

	/*!
	* \internal
	* \brief Apply colocation to primary's node scores if for promoted role
	*
	* \param[in,out] data Colocation constraint to apply
	* \param[in,out] user_data Promotable clone that is constraint's primary
	*/
	static void
	apply_coloc_to_primary(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t *colocation = data;
	pcmk_resource_t *clone = user_data;
	pcmk_resource_t *dependent = colocation->dependent;
	const float factor = colocation->score / (float) INFINITY;
	const uint32_t flags = pcmk__coloc_select_active
	\|pcmk__coloc_select_nonnegative;

	if ((colocation->primary_role != pcmk_role_promoted)
	\|\| !pcmk__colocation_has_influence(colocation, NULL)) {
	return;
	}

	pcmk__rsc_trace(clone, "Applying colocation %s (%s with promoted %s) @%s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id,
	pcmk_readable_score(colocation->score));
	dependent->cmds->add_colocated_node_scores(dependent, clone, clone->id,
	&clone->allowed_nodes,
	colocation, factor, flags);
	}

	/*!
	* \internal
	* \brief Set clone instance's sort index to its node's score
	*
	* \param[in,out] data Promotable clone instance
	* \param[in] user_data Parent clone of \p data
	*/
	static void
	set_sort_index_to_node_score(gpointer data, gpointer user_data)
	{
	pcmk_resource_t child = (pcmk_resource_t ) data;
	const pcmk_resource_t clone = (const pcmk_resource_t ) user_data;

	pcmk_node_t *chosen = child->fns->location(child, NULL, FALSE);

	if (!pcmk_is_set(child->flags, pcmk_rsc_managed)
	&& (child->next_role == pcmk_role_promoted)) {
	child->sort_index = INFINITY;
	pcmk__rsc_trace(clone,
	"Final sort index for %s is INFINITY "
	"(unmanaged promoted)",
	child->id);

	} else if ((chosen == NULL) \|\| (child->sort_index < 0)) {
	pcmk__rsc_trace(clone,
	"Final sort index for %s is %d (ignoring node score)",
	child->id, child->sort_index);

	} else {
	const pcmk_node_t *node = g_hash_table_lookup(clone->allowed_nodes,
	chosen->details->id);

	CRM_ASSERT(node != NULL);
	child->sort_index = node->weight;
	pcmk__rsc_trace(clone,
	"Adding scores for %s: final sort index for %s is %d",
	clone->id, child->id, child->sort_index);
	}
	}

	/*!
	* \internal
	* \brief Sort a promotable clone's instances by descending promotion priority
	*
	* \param[in,out] clone Promotable clone to sort
	*/
	static void
	sort_promotable_instances(pcmk_resource_t *clone)
	{
	GList *colocations = NULL;

	if (pe__set_clone_flag(clone, pcmk__clone_promotion_constrained)
	== pcmk_rc_already) {
	return;
	}
	pe__set_resource_flags(clone, pcmk_rsc_updating_nodes);

	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	pcmk__rsc_trace(clone,
	"Adding scores for %s: initial sort index for %s is %d",
	clone->id, child->id, child->sort_index);
	}
	pe__show_node_scores(true, clone, "Before", clone->allowed_nodes,
	clone->cluster);

	g_list_foreach(clone->children, add_sort_index_to_node_score, clone);

	colocations = pcmk__this_with_colocations(clone);
	g_list_foreach(colocations, apply_coloc_to_dependent, clone);
	g_list_free(colocations);

	colocations = pcmk__with_this_colocations(clone);
	g_list_foreach(colocations, apply_coloc_to_primary, clone);
	g_list_free(colocations);

	// Ban resource from all nodes if it needs a ticket but doesn't have it
	pcmk__require_promotion_tickets(clone);

	pe__show_node_scores(true, clone, "After", clone->allowed_nodes,
	clone->cluster);

	// Reset sort indexes to final node scores
	g_list_foreach(clone->children, set_sort_index_to_node_score, clone);

	// Finally, sort instances in descending order of promotion priority
	clone->children = g_list_sort(clone->children, cmp_promotable_instance);
	pe__clear_resource_flags(clone, pcmk_rsc_updating_nodes);
	}

	/*!
	* \internal
	* \brief Find the active instance (if any) of an anonymous clone on a node
	*
	* \param[in] clone Anonymous clone to check
	* \param[in] id Instance ID (without instance number) to check
	* \param[in] node Node to check
	*
	* \return
	*/
	static pcmk_resource_t *
	find_active_anon_instance(const pcmk_resource_t clone, const char id,
	const pcmk_node_t *node)
	{
	for (GList *iter = clone->children; iter; iter = iter->next) {
	pcmk_resource_t *child = iter->data;
	pcmk_resource_t *active = NULL;

	// Use ->find_rsc() in case this is a cloned group
	active = clone->fns->find_rsc(child, id, node,
	pcmk_rsc_match_clone_only
	\|pcmk_rsc_match_current_node);
	if (active != NULL) {
	return active;
	}
	}
	return NULL;
	}

	/*
	* \brief Check whether an anonymous clone instance is known on a node
	*
	* \param[in] clone Anonymous clone to check
	* \param[in] id Instance ID (without instance number) to check
	* \param[in] node Node to check
	*
	* \return true if \p id instance of \p clone is known on \p node,
	* otherwise false
	*/
	static bool
	anonymous_known_on(const pcmk_resource_t clone, const char id,
	const pcmk_node_t *node)
	{
	for (GList *iter = clone->children; iter; iter = iter->next) {
	pcmk_resource_t *child = iter->data;

	/* Use ->find_rsc() because this might be a cloned group, and knowing
	* that other members of the group are known here implies nothing.
	*/
	child = clone->fns->find_rsc(child, id, NULL,
	pcmk_rsc_match_clone_only);
	CRM_LOG_ASSERT(child != NULL);
	if (child != NULL) {
	if (g_hash_table_lookup(child->known_on, node->details->id)) {
	return true;
	}
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node is allowed to run a resource
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is allowed to run \p rsc, otherwise false
	*/
	static bool
	is_allowed(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk_node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes,
	node->details->id);

	return (allowed != NULL) && (allowed->weight >= 0);
	}

	/*!
	* \brief Check whether a clone instance's promotion score should be considered
	*
	* \param[in] rsc Promotable clone instance to check
	* \param[in] node Node where score would be applied
	*
	* \return true if \p rsc's promotion score should be considered on \p node,
	* otherwise false
	*/
	static bool
	promotion_score_applies(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	char *id = clone_strip(rsc->id);
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
	pcmk_resource_t *active = NULL;
	const char *reason = "allowed";

	// Some checks apply only to anonymous clone instances
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {

	// If instance is active on the node, its score definitely applies
	active = find_active_anon_instance(parent, id, node);
	if (active == rsc) {
	reason = "active";
	goto check_allowed;
	}

	/* If no instance is active on this node, this instance's score will
	* count if it has been probed on this node.
	*/
	if ((active == NULL) && anonymous_known_on(parent, id, node)) {
	reason = "probed";
	goto check_allowed;
	}
	}

	/* If this clone's status is unknown on all nodes (e.g. cluster startup),
	* take all instances' scores into account, to make sure we use any
	* permanent promotion scores.
	*/
	if ((rsc->running_on == NULL) && (g_hash_table_size(rsc->known_on) == 0)) {
	reason = "none probed";
	goto check_allowed;
	}

	/* Otherwise, we've probed and/or started the resource somewhere, so
	* consider promotion scores on nodes where we know the status.
	*/
	if ((g_hash_table_lookup(rsc->known_on, node->details->id) != NULL)
	\|\| (pe_find_node_id(rsc->running_on, node->details->id) != NULL)) {
	reason = "known";
	} else {
	pcmk__rsc_trace(rsc,
	"Ignoring %s promotion score (for %s) on %s: "
	"not probed",
	rsc->id, id, pe__node_name(node));
	free(id);
	return false;
	}

	check_allowed:
	if (is_allowed(rsc, node)) {
	pcmk__rsc_trace(rsc, "Counting %s promotion score (for %s) on %s: %s",
	rsc->id, id, pe__node_name(node), reason);
	free(id);
	return true;
	}

	pcmk__rsc_trace(rsc,
	"Ignoring %s promotion score (for %s) on %s: not allowed",
	rsc->id, id, pe__node_name(node));
	free(id);
	return false;
	}

	/*!
	* \internal
	* \brief Get the value of a promotion score node attribute
	*
	* \param[in] rsc Promotable clone instance to get promotion score for
	* \param[in] node Node to get promotion score for
	* \param[in] name Resource name to use in promotion score attribute name
	*
	* \return Value of promotion score node attribute for \p rsc on \p node
	*/
	static const char *
	promotion_attr_value(const pcmk_resource_t rsc, const pcmk_node_t node,
	const char *name)
	{
	char *attr_name = NULL;
	const char *attr_value = NULL;
	enum pcmk__rsc_node node_type = pcmk__rsc_node_assigned;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	// Not assigned yet
	node_type = pcmk__rsc_node_current;
	}
	attr_name = pcmk_promotion_score_name(name);
	attr_value = pe__node_attribute_calculated(node, attr_name, rsc, node_type,
	false);
	free(attr_name);
	return attr_value;
	}

	/*!
	* \internal
	* \brief Get the promotion score for a clone instance on a node
	*
	* \param[in] rsc Promotable clone instance to get score for
	* \param[in] node Node to get score for
	* \param[out] is_default If non-NULL, will be set true if no score available
	*
	* \return Promotion score for \p rsc on \p node (or 0 if none)
	*/
	static int
	promotion_score(const pcmk_resource_t rsc, const pcmk_node_t node,
	bool *is_default)
	{
	char *name = NULL;
	const char *attr_value = NULL;

	if (is_default != NULL) {
	*is_default = true;
	}

	CRM_CHECK((rsc != NULL) && (node != NULL), return 0);

	/* If this is an instance of a cloned group, the promotion score is the sum
	* of all members' promotion scores.
	*/
	if (rsc->children != NULL) {
	int score = 0;

	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;
	bool child_default = false;
	int child_score = promotion_score(child, node, &child_default);

	if (!child_default && (is_default != NULL)) {
	*is_default = false;
	}
	score += child_score;
	}
	return score;
	}

	if (!promotion_score_applies(rsc, node)) {
	return 0;
	}

	/* For the promotion score attribute name, use the name the resource is
	* known as in resource history, since that's what crm_attribute --promotion
	* would have used.
	*/
	name = (rsc->clone_name == NULL)? rsc->id : rsc->clone_name;

	attr_value = promotion_attr_value(rsc, node, name);
	if (attr_value != NULL) {
	pcmk__rsc_trace(rsc, "Promotion score for %s on %s = %s",
	name, pe__node_name(node),
	pcmk__s(attr_value, "(unset)"));
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	/* If we don't have any resource history yet, we won't have clone_name.
	* In that case, for anonymous clones, try the resource name without
	* any instance number.
	*/
	name = clone_strip(rsc->id);
	if (strcmp(rsc->id, name) != 0) {
	attr_value = promotion_attr_value(rsc, node, name);
	pcmk__rsc_trace(rsc, "Promotion score for %s on %s (for %s) = %s",
	name, pe__node_name(node), rsc->id,
	pcmk__s(attr_value, "(unset)"));
	}
	free(name);
	}

	if (attr_value == NULL) {
	return 0;
	}

	if (is_default != NULL) {
	*is_default = false;
	}
	return char2score(attr_value);
	}

	/*!
	* \internal
	* \brief Include promotion scores in instances' node scores and priorities
	*
	* \param[in,out] rsc Promotable clone resource to update
	*/
	void
	pcmk__add_promotion_scores(pcmk_resource_t *rsc)
	{
	if (pe__set_clone_flag(rsc,
	pcmk__clone_promotion_added) == pcmk_rc_already) {
	return;
	}

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) iter->data;

	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	int score, new_score;

	g_hash_table_iter_init(&iter, child_rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (!pcmk__node_available(node, false, false)) {
	/* This node will never be promoted, so don't apply the
	* promotion score, as that may lead to clone shuffling.
	*/
	continue;
	}

	score = promotion_score(child_rsc, node, NULL);
	if (score > 0) {
	new_score = pcmk__add_scores(node->weight, score);
	if (new_score != node->weight) { // Could remain INFINITY
	node->weight = new_score;
	pcmk__rsc_trace(rsc,
	"Added %s promotion priority (%s) to score "
	"on %s (now %s)",
	child_rsc->id, pcmk_readable_score(score),
	pe__node_name(node),
	pcmk_readable_score(new_score));
	}
	}

	if (score > child_rsc->priority) {
	pcmk__rsc_trace(rsc,
	"Updating %s priority to promotion score "
	"(%d->%d)",
	child_rsc->id, child_rsc->priority, score);
	child_rsc->priority = score;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief If a resource's current role is started, change it to unpromoted
	*
	* \param[in,out] data Resource to update
	* \param[in] user_data Ignored
	*/
	static void
	set_current_role_unpromoted(void data, void user_data)
	{
	pcmk_resource_t rsc = (pcmk_resource_t ) data;

	if (rsc->role == pcmk_role_started) {
	// Promotable clones should use unpromoted role instead of started
	rsc->role = pcmk_role_unpromoted;
	}
	g_list_foreach(rsc->children, set_current_role_unpromoted, NULL);
	}

	/*!
	* \internal
	* \brief Set a resource's next role to unpromoted (or stopped if unassigned)
	*
	* \param[in,out] data Resource to update
	* \param[in] user_data Ignored
	*/
	static void
	set_next_role_unpromoted(void data, void user_data)
	{
	pcmk_resource_t rsc = (pcmk_resource_t ) data;
	GList *assigned = NULL;

	rsc->fns->location(rsc, &assigned, FALSE);
	if (assigned == NULL) {
	pe__set_next_role(rsc, pcmk_role_stopped, "stopped instance");
	} else {
	pe__set_next_role(rsc, pcmk_role_unpromoted, "unpromoted instance");
	g_list_free(assigned);
	}
	g_list_foreach(rsc->children, set_next_role_unpromoted, NULL);
	}

	/*!
	* \internal
	* \brief Set a resource's next role to promoted if not already set
	*
	* \param[in,out] data Resource to update
	* \param[in] user_data Ignored
	*/
	static void
	set_next_role_promoted(void *data, gpointer user_data)
	{
	pcmk_resource_t rsc = (pcmk_resource_t ) data;

	if (rsc->next_role == pcmk_role_unknown) {
	pe__set_next_role(rsc, pcmk_role_promoted, "promoted instance");
	}
	g_list_foreach(rsc->children, set_next_role_promoted, NULL);
	}

	/*!
	* \internal
	* \brief Show instance's promotion score on node where it will be active
	*
	* \param[in,out] instance Promotable clone instance to show
	*/
	static void
	show_promotion_score(pcmk_resource_t *instance)
	{
	pcmk_node_t *chosen = instance->fns->location(instance, NULL, FALSE);

	if (pcmk_is_set(instance->cluster->flags, pcmk_sched_output_scores)
	&& !pcmk__is_daemon && (instance->cluster->priv != NULL)) {

	pcmk__output_t *out = instance->cluster->priv;

	out->message(out, "promotion-score", instance, chosen,
	pcmk_readable_score(instance->sort_index));
	} else {
	pcmk__rsc_debug(pe__const_top_resource(instance, false),
	"%s promotion score on %s: sort=%s priority=%s",
	instance->id,
	((chosen == NULL)? "none" : pe__node_name(chosen)),
	pcmk_readable_score(instance->sort_index),
	pcmk_readable_score(instance->priority));
	}
	}

	/*!
	* \internal
	* \brief Set a clone instance's promotion priority
	*
	* \param[in,out] data Promotable clone instance to update
	* \param[in] user_data Instance's parent clone
	*/
	static void
	set_instance_priority(gpointer data, gpointer user_data)
	{
	pcmk_resource_t instance = (pcmk_resource_t ) data;
	const pcmk_resource_t clone = (const pcmk_resource_t ) user_data;
	const pcmk_node_t *chosen = NULL;
	enum rsc_role_e next_role = pcmk_role_unknown;
	GList *list = NULL;

	pcmk__rsc_trace(clone, "Assigning priority for %s: %s", instance->id,
	role2text(instance->next_role));

	if (instance->fns->state(instance, TRUE) == pcmk_role_started) {
	set_current_role_unpromoted(instance, NULL);
	}

	// Only an instance that will be active can be promoted
	chosen = instance->fns->location(instance, &list, FALSE);
	if (pcmk__list_of_multiple(list)) {
	pcmk__config_err("Cannot promote non-colocated child %s",
	instance->id);
	}
	g_list_free(list);
	if (chosen == NULL) {
	return;
	}

	next_role = instance->fns->state(instance, FALSE);
	switch (next_role) {
	case pcmk_role_started:
	case pcmk_role_unknown:
	// Set instance priority to its promotion score (or -1 if none)
	{
	bool is_default = false;

	instance->priority = promotion_score(instance, chosen,
	&is_default);
	if (is_default) {
	/*
	* Default to -1 if no value is set. This allows
	* instances eligible for promotion to be specified
	* based solely on rsc_location constraints, but
	* prevents any instance from being promoted if neither
	* a constraint nor a promotion score is present
	*/
	instance->priority = -1;
	}
	}
	break;

	case pcmk_role_unpromoted:
	case pcmk_role_stopped:
	// Instance can't be promoted
	instance->priority = -INFINITY;
	break;

	case pcmk_role_promoted:
	// Nothing needed (re-creating actions after scheduling fencing)
	break;

	default:
	CRM_CHECK(FALSE, crm_err("Unknown resource role %d for %s",
	next_role, instance->id));
	}

	// Add relevant location constraint scores for promoted role
	apply_promoted_locations(instance, instance->rsc_location, chosen);
	apply_promoted_locations(instance, clone->rsc_location, chosen);

	// Consider instance's role-based colocations with other resources
	list = pcmk__this_with_colocations(instance);
	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) iter->data;

	instance->cmds->apply_coloc_score(instance, cons->primary, cons, true);
	}
	g_list_free(list);

	instance->sort_index = instance->priority;
	if (next_role == pcmk_role_promoted) {
	instance->sort_index = INFINITY;
	}
	pcmk__rsc_trace(clone, "Assigning %s priority = %d",
	instance->id, instance->priority);
	}

	/*!
	* \internal
	* \brief Set a promotable clone instance's role
	*
	* \param[in,out] data Promotable clone instance to update
	* \param[in,out] user_data Pointer to count of instances chosen for promotion
	*/
	static void
	set_instance_role(gpointer data, gpointer user_data)
	{
	pcmk_resource_t instance = (pcmk_resource_t ) data;
	int count = (int ) user_data;

	const pcmk_resource_t *clone = pe__const_top_resource(instance, false);
	pcmk_node_t *chosen = NULL;

	show_promotion_score(instance);

	if (instance->sort_index < 0) {
	pcmk__rsc_trace(clone, "Not supposed to promote instance %s",
	instance->id);

	} else if ((*count < pe__clone_promoted_max(instance))
	\|\| !pcmk_is_set(clone->flags, pcmk_rsc_managed)) {
	chosen = node_to_be_promoted_on(instance);
	}

	if (chosen == NULL) {
	set_next_role_unpromoted(instance, NULL);
	return;
	}

	if ((instance->role < pcmk_role_promoted)
	&& !pcmk_is_set(instance->cluster->flags, pcmk_sched_quorate)
	&& (instance->cluster->no_quorum_policy == pcmk_no_quorum_freeze)) {
	crm_notice("Clone instance %s cannot be promoted without quorum",
	instance->id);
	set_next_role_unpromoted(instance, NULL);
	return;
	}

	chosen->count++;
	pcmk__rsc_info(clone, "Choosing %s (%s) on %s for promotion",
	instance->id, role2text(instance->role),
	pe__node_name(chosen));
	set_next_role_promoted(instance, NULL);
	(*count)++;
	}

	/*!
	* \internal
	* \brief Set roles for all instances of a promotable clone
	*
	* \param[in,out] rsc Promotable clone resource to update
	*/
	void
	pcmk__set_instance_roles(pcmk_resource_t *rsc)
	{
	int promoted = 0;
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	// Repurpose count to track the number of promoted instances assigned
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	node->count = 0;
	}

	// Set instances' promotion priorities and sort by highest priority first
	g_list_foreach(rsc->children, set_instance_priority, rsc);
	sort_promotable_instances(rsc);

	// Choose the first N eligible instances to be promoted
	g_list_foreach(rsc->children, set_instance_role, &promoted);
	pcmk__rsc_info(rsc, "%s: Promoted %d instances of a possible %d",
	rsc->id, promoted, pe__clone_promoted_max(rsc));
	}

	/*!
	*
	* \internal
	* \brief Create actions for promotable clone instances
	*
	* \param[in,out] clone Promotable clone to create actions for
	* \param[out] any_promoting Will be set true if any instance is promoting
	* \param[out] any_demoting Will be set true if any instance is demoting
	*/
	static void
	create_promotable_instance_actions(pcmk_resource_t *clone,
	bool any_promoting, bool any_demoting)
	{
	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->cmds->create_actions(instance);
	check_for_role_change(instance, any_demoting, any_promoting);
	}
	}

	/*!
	* \internal
	* \brief Reset each promotable instance's resource priority
	*
	* Reset the priority of each instance of a promotable clone to the clone's
	* priority (after promotion actions are scheduled, when instance priorities
	* were repurposed as promotion scores).
	*
	* \param[in,out] clone Promotable clone to reset
	*/
	static void
	reset_instance_priorities(pcmk_resource_t *clone)
	{
	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->priority = clone->priority;
	}
	}

	/*!
	* \internal
	* \brief Create actions specific to promotable clones
	*
	* \param[in,out] clone Promotable clone to create actions for
	*/
	void
	pcmk__create_promotable_actions(pcmk_resource_t *clone)
	{
	bool any_promoting = false;
	bool any_demoting = false;

	// Create actions for each clone instance individually
	create_promotable_instance_actions(clone, &any_promoting, &any_demoting);

	// Create pseudo-actions for clone as a whole
	pe__create_promotable_pseudo_ops(clone, any_promoting, any_demoting);

	// Undo our temporary repurposing of resource priority for instances
	reset_instance_priorities(clone);
	}

	/*!
	* \internal
	* \brief Create internal orderings for a promotable clone's instances
	*
	* \param[in,out] clone Promotable clone instance to order
	*/
	void
	pcmk__order_promotable_instances(pcmk_resource_t *clone)
	{
	pcmk_resource_t *previous = NULL; // Needed for ordered clones

	pcmk__promotable_restart_ordering(clone);

	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	// Demote before promote
	pcmk__order_resource_actions(instance, PCMK_ACTION_DEMOTE,
	instance, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	order_instance_promotion(clone, instance, previous);
	order_instance_demotion(clone, instance, previous);
	previous = instance;
	}
	}

	/*!
	* \internal
	* \brief Update dependent's allowed nodes for colocation with promotable
	*
	* \param[in,out] dependent Dependent resource to update
	* \param[in] primary Primary resource
	* \param[in] primary_node Node where an instance of the primary will be
	* \param[in] colocation Colocation constraint to apply
	*/
	static void
	update_dependent_allowed_nodes(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk_node_t *primary_node,
	const pcmk__colocation_t *colocation)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	const char *primary_value = NULL;
	const char *attr = colocation->node_attribute;

	if (colocation->score >= INFINITY) {
	return; // Colocation is mandatory, so allowed node scores don't matter
	}

	primary_value = pcmk__colocation_node_attr(primary_node, attr, primary);

	pcmk__rsc_trace(colocation->primary,
	"Applying %s (%s with %s on %s by %s @%d) to %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, pe__node_name(primary_node), attr,
	colocation->score, dependent->id);

	g_hash_table_iter_init(&iter, dependent->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	const char *dependent_value = pcmk__colocation_node_attr(node, attr,
	dependent);

	if (pcmk__str_eq(primary_value, dependent_value, pcmk__str_casei)) {
	node->weight = pcmk__add_scores(node->weight, colocation->score);
	pcmk__rsc_trace(colocation->primary,
	"Added %s score (%s) to %s (now %s)",
	colocation->id,
	pcmk_readable_score(colocation->score),
	pe__node_name(node),
	pcmk_readable_score(node->weight));
	}
	}
	}

	/*!
	* \brief Update dependent for a colocation with a promotable clone
	*
	* \param[in] primary Primary resource in the colocation
	* \param[in,out] dependent Dependent resource in the colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	void
	pcmk__update_dependent_with_promotable(const pcmk_resource_t *primary,
	pcmk_resource_t *dependent,
	const pcmk__colocation_t *colocation)
	{
	GList *affected_nodes = NULL;

	/* Build a list of all nodes where an instance of the primary will be, and
	* (for optional colocations) update the dependent's allowed node scores for
	* each one.
	*/
	for (GList *iter = primary->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;
	pcmk_node_t *node = instance->fns->location(instance, NULL, FALSE);

	if (node == NULL) {
	continue;
	}
	if (instance->fns->state(instance, FALSE) == colocation->primary_role) {
	update_dependent_allowed_nodes(dependent, primary, node,
	colocation);
	affected_nodes = g_list_prepend(affected_nodes, node);
	}
	}

	/* For mandatory colocations, add the primary's node score to the
	* dependent's node score for each affected node, and ban the dependent
	* from all other nodes.
	*
	* However, skip this for promoted-with-promoted colocations, otherwise
	* inactive dependent instances can't start (in the unpromoted role).
	*/
	if ((colocation->score >= INFINITY)
	&& ((colocation->dependent_role != pcmk_role_promoted)
	\|\| (colocation->primary_role != pcmk_role_promoted))) {

	pcmk__rsc_trace(colocation->primary,
	"Applying %s (mandatory %s with %s) to %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, dependent->id);
	pcmk__colocation_intersect_nodes(dependent, primary, colocation,
	affected_nodes, true);
	}
	g_list_free(affected_nodes);
	}

	/*!
	* \internal
	* \brief Update dependent priority for colocation with promotable
	*
	* \param[in] primary Primary resource in the colocation
	* \param[in,out] dependent Dependent resource in the colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	void
	pcmk__update_promotable_dependent_priority(const pcmk_resource_t *primary,
	pcmk_resource_t *dependent,
	const pcmk__colocation_t *colocation)
	{
	pcmk_resource_t *primary_instance = NULL;

	// Look for a primary instance where dependent will be
	primary_instance = pcmk__find_compatible_instance(dependent, primary,
	colocation->primary_role,
	false);

	if (primary_instance != NULL) {
	// Add primary instance's priority to dependent's
	int new_priority = pcmk__add_scores(dependent->priority,
	colocation->score);

	pcmk__rsc_trace(colocation->primary,
	"Applying %s (%s with %s) to %s priority "
	"(%s + %s = %s)",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, dependent->id,
	pcmk_readable_score(dependent->priority),
	pcmk_readable_score(colocation->score),
	pcmk_readable_score(new_priority));
	dependent->priority = new_priority;

	} else if (colocation->score >= INFINITY) {
	// Mandatory colocation, but primary won't be here
	pcmk__rsc_trace(colocation->primary,
	"Applying %s (%s with %s) to %s: can't be promoted",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, dependent->id);
	dependent->priority = -INFINITY;
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c
	index 0bed2553d9..7465baf6d8 100644
	--- a/lib/pacemaker/pcmk_sched_resource.c
	+++ b/lib/pacemaker/pcmk_sched_resource.c
	@@ -1,773 +1,773 @@
	/*
	* Copyright 2014-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <string.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Resource assignment methods by resource variant
	static pcmk_assignment_methods_t assignment_methods[] = {
	{
	pcmk__primitive_assign,
	pcmk__primitive_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__primitive_internal_constraints,
	pcmk__primitive_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_primitive_colocations,
	pcmk__primitive_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__apply_location,
	pcmk__primitive_action_flags,
	pcmk__update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__primitive_add_graph_meta,
	pcmk__primitive_add_utilization,
	pcmk__primitive_shutdown_lock,
	},
	{
	pcmk__group_assign,
	pcmk__group_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__group_internal_constraints,
	pcmk__group_apply_coloc_score,
	pcmk__group_colocated_resources,
	pcmk__with_group_colocations,
	pcmk__group_with_colocations,
	pcmk__group_add_colocated_node_scores,
	pcmk__group_apply_location,
	pcmk__group_action_flags,
	pcmk__group_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__group_add_utilization,
	pcmk__group_shutdown_lock,
	},
	{
	pcmk__clone_assign,
	pcmk__clone_create_actions,
	pcmk__clone_create_probe,
	pcmk__clone_internal_constraints,
	pcmk__clone_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_clone_colocations,
	pcmk__clone_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__clone_apply_location,
	pcmk__clone_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__clone_add_actions_to_graph,
	pcmk__clone_add_graph_meta,
	pcmk__clone_add_utilization,
	pcmk__clone_shutdown_lock,
	},
	{
	pcmk__bundle_assign,
	pcmk__bundle_create_actions,
	pcmk__bundle_create_probe,
	pcmk__bundle_internal_constraints,
	pcmk__bundle_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_bundle_colocations,
	pcmk__bundle_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__bundle_apply_location,
	pcmk__bundle_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_bundle_actions,
	pcmk__bundle_add_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__bundle_add_utilization,
	pcmk__bundle_shutdown_lock,
	}
	};

	/*!
	* \internal
	* \brief Check whether a resource's agent standard, provider, or type changed
	*
	* \param[in,out] rsc Resource to check
	* \param[in,out] node Node needing unfencing if agent changed
	* \param[in] rsc_entry XML with previously known agent information
	* \param[in] active_on_node Whether \p rsc is active on \p node
	*
	* \return true if agent for \p rsc changed, otherwise false
	*/
	bool
	pcmk__rsc_agent_changed(pcmk_resource_t rsc, pcmk_node_t node,
	const xmlNode *rsc_entry, bool active_on_node)
	{
	bool changed = false;
	const char *attr_list[] = {
	XML_ATTR_TYPE,
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER
	};

	for (int i = 0; i < PCMK__NELEM(attr_list); i++) {
	const char *value = crm_element_value(rsc->xml, attr_list[i]);
	const char *old_value = crm_element_value(rsc_entry, attr_list[i]);

	if (!pcmk__str_eq(value, old_value, pcmk__str_none)) {
	changed = true;
	trigger_unfencing(rsc, node, "Device definition changed", NULL,
	rsc->cluster);
	if (active_on_node) {
	crm_notice("Forcing restart of %s on %s "
	"because %s changed from '%s' to '%s'",
	rsc->id, pe__node_name(node), attr_list[i],
	pcmk__s(old_value, ""), pcmk__s(value, ""));
	}
	}
	}
	if (changed && active_on_node) {
	// Make sure the resource is restarted
	custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE,
	rsc->cluster);
	pe__set_resource_flags(rsc, pcmk_rsc_start_pending);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Add resource (and any matching children) to list if it matches ID
	*
	* \param[in] result List to add resource to
	* \param[in] rsc Resource to check
	* \param[in] id ID to match
	*
	* \return (Possibly new) head of list
	*/
	static GList *
	add_rsc_if_matching(GList result, pcmk_resource_t rsc, const char *id)
	{
	if ((strcmp(rsc->id, id) == 0)
	\|\| ((rsc->clone_name != NULL) && (strcmp(rsc->clone_name, id) == 0))) {
	result = g_list_prepend(result, rsc);
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	result = add_rsc_if_matching(result, child, id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Find all resources matching a given ID by either ID or clone name
	*
	* \param[in] id Resource ID to check
	* \param[in] scheduler Scheduler data
	*
	* \return List of all resources that match \p id
	* \note The caller is responsible for freeing the return value with
	* g_list_free().
	*/
	GList *
	pcmk__rscs_matching_id(const char id, const pcmk_scheduler_t scheduler)
	{
	GList *result = NULL;

	CRM_CHECK((id != NULL) && (scheduler != NULL), return NULL);
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	result = add_rsc_if_matching(result, (pcmk_resource_t *) iter->data,
	id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for a resource
	*
	* \param[in,out] data Resource to set assignment methods for
	* \param[in] user_data Ignored
	*/
	static void
	set_assignment_methods_for_rsc(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;

	rsc->cmds = &assignment_methods[rsc->variant];
	g_list_foreach(rsc->children, set_assignment_methods_for_rsc, NULL);
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for all resources
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__set_assignment_methods(pcmk_scheduler_t *scheduler)
	{
	g_list_foreach(scheduler->resources, set_assignment_methods_for_rsc, NULL);
	}

	/*!
	* \internal
	* \brief Wrapper for colocated_resources() method for readability
	*
	* \param[in] rsc Resource to add to colocated list
	* \param[in] orig_rsc Resource originally requested
	* \param[in,out] list Pointer to list to add to
	*
	* \return (Possibly new) head of list
	*/
	static inline void
	add_colocated_resources(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	list = rsc->cmds->colocated_resources(rsc, orig_rsc, list);
	}

	// Shared implementation of pcmk_assignment_methods_t:colocated_resources()
	GList *
	pcmk__colocated_resources(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList *colocated_rscs)
	{
	const GList *iter = NULL;
	GList *colocations = NULL;

	if (orig_rsc == NULL) {
	orig_rsc = rsc;
	}

	if ((rsc == NULL) \|\| (g_list_find(colocated_rscs, rsc) != NULL)) {
	return colocated_rscs;
	}

	pcmk__rsc_trace(orig_rsc, "%s is in colocation chain with %s",
	rsc->id, orig_rsc->id);
	colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);

	// Follow colocations where this resource is the dependent resource
	colocations = pcmk__this_with_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pcmk_resource_t *primary = constraint->primary;

	if (primary == orig_rsc) {
	continue; // Break colocation loop
	}

	if ((constraint->score == INFINITY) &&
	(pcmk__colocation_affects(rsc, primary, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(primary, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	// Follow colocations where this resource is the primary resource
	colocations = pcmk__with_this_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pcmk_resource_t *dependent = constraint->dependent;

	if (dependent == orig_rsc) {
	continue; // Break colocation loop
	}

	if (pe_rsc_is_clone(rsc) && !pe_rsc_is_clone(dependent)) {
	continue; // We can't be sure whether dependent will be colocated
	}

	if ((constraint->score == INFINITY) &&
	(pcmk__colocation_affects(dependent, rsc, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(dependent, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	return colocated_rscs;
	}

	// No-op function for variants that don't need to implement add_graph_meta()
	void
	pcmk__noop_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml)
	{
	}

	/*!
	* \internal
	* \brief Output a summary of scheduled actions for a resource
	*
	* \param[in,out] rsc Resource to output actions for
	*/
	void
	pcmk__output_resource_actions(pcmk_resource_t *rsc)
	{
	pcmk_node_t *next = NULL;
	pcmk_node_t *current = NULL;
	pcmk__output_t *out = NULL;

	CRM_ASSERT(rsc != NULL);

	out = rsc->cluster->priv;
	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	child->cmds->output_actions(child);
	}
	return;
	}

	next = rsc->allocated_to;
	if (rsc->running_on) {
	current = pe__current_node(rsc);
	if (rsc->role == pcmk_role_stopped) {
	/* This can occur when resources are being recovered because
	* the current role can change in pcmk__primitive_create_actions()
	*/
	rsc->role = pcmk_role_started;
	}
	}

	if ((current == NULL) && pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	/* Don't log stopped orphans */
	return;
	}

	out->message(out, "rsc-action", rsc, current, next);
	}

	/*!
	* \internal
	* \brief Add a resource to a node's list of assigned resources
	*
	* \param[in,out] node Node to add resource to
	* \param[in] rsc Resource to add
	*/
	static inline void
	add_assigned_resource(pcmk_node_t node, pcmk_resource_t rsc)
	{
	node->details->allocated_rsc = g_list_prepend(node->details->allocated_rsc,
	rsc);
	}

	/*!
	* \internal
	* \brief Assign a specified resource (of any variant) to a node
	*
	* Assign a specified resource and its children (if any) to a specified node, if
	* the node can run the resource (or unconditionally, if \p force is true). Mark
	* the resources as no longer provisional.
	*
	* If a resource can't be assigned (or \p node is \c NULL), unassign any
	* previous assignment. If \p stop_if_fail is \c true, set next role to stopped
	* and update any existing actions scheduled for the resource.
	*
	* \param[in,out] rsc Resource to assign
	* \param[in,out] node Node to assign \p rsc to
	* \param[in] force If true, assign to \p node even if unavailable
	* \param[in] stop_if_fail If \c true and either \p rsc can't be assigned
	* or \p chosen is \c NULL, set next role to
	* stopped and update existing actions (if \p rsc
	* is not a primitive, this applies to its
	* primitive descendants instead)
	*
	* \return \c true if the assignment of \p rsc changed, or \c false otherwise
	*
	* \note Assigning a resource to the NULL node using this function is different
	* from calling pcmk__unassign_resource(), in that it may also update any
	* actions created for the resource.
	* \note The \c pcmk_assignment_methods_t:assign() method is preferred, unless
	* a resource should be assigned to the \c NULL node or every resource in
	* a tree should be assigned to the same node.
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	bool
	pcmk__assign_resource(pcmk_resource_t rsc, pcmk_node_t node, bool force,
	bool stop_if_fail)
	{
	bool changed = false;

	CRM_ASSERT(rsc != NULL);

	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t *child_rsc = iter->data;

	changed \|= pcmk__assign_resource(child_rsc, node, force,
	stop_if_fail);
	}
	return changed;
	}

	// Assigning a primitive

	if (!force && (node != NULL)
	&& ((node->weight < 0)
	// Allow graph to assume that guest node connections will come up
	\|\| (!pcmk__node_available(node, true, false)
	&& !pe__is_guest_node(node)))) {

	pcmk__rsc_debug(rsc,
	"All nodes for resource %s are unavailable, unclean or "
	"shutting down (%s can%s run resources, with score %s)",
	rsc->id, pe__node_name(node),
	(pcmk__node_available(node, true, false)? "" : "not"),
	pcmk_readable_score(node->weight));

	if (stop_if_fail) {
	pe__set_next_role(rsc, pcmk_role_stopped, "node availability");
	}
	node = NULL;
	}

	if (rsc->allocated_to != NULL) {
	changed = !pe__same_node(rsc->allocated_to, node);
	} else {
	changed = (node != NULL);
	}
	pcmk__unassign_resource(rsc);
	pe__clear_resource_flags(rsc, pcmk_rsc_unassigned);

	if (node == NULL) {
	char *rc_stopped = NULL;

	pcmk__rsc_debug(rsc, "Could not assign %s to a node", rsc->id);

	if (!stop_if_fail) {
	return changed;
	}
	pe__set_next_role(rsc, pcmk_role_stopped, "unable to assign");

	for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t op = (pcmk_action_t ) iter->data;

	pcmk__rsc_debug(rsc, "Updating %s for %s assignment failure",
	op->uuid, rsc->id);

	if (pcmk__str_eq(op->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	pe__clear_action_flags(op, pcmk_action_optional);

	} else if (pcmk__str_eq(op->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	pe__clear_action_flags(op, pcmk_action_runnable);

	} else {
	// Cancel recurring actions, unless for stopped state
	const char *interval_ms_s = NULL;
	const char *target_rc_s = NULL;

	interval_ms_s = g_hash_table_lookup(op->meta,
	XML_LRM_ATTR_INTERVAL_MS);
	target_rc_s = g_hash_table_lookup(op->meta,
	XML_ATTR_TE_TARGET_RC);
	if (rc_stopped == NULL) {
	rc_stopped = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
	}

	if (!pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)
	&& !pcmk__str_eq(rc_stopped, target_rc_s, pcmk__str_none)) {

	pe__clear_action_flags(op, pcmk_action_runnable);
	}
	}
	}
	free(rc_stopped);
	return changed;
	}

	pcmk__rsc_debug(rsc, "Assigning %s to %s", rsc->id, pe__node_name(node));
	rsc->allocated_to = pe__copy_node(node);

	add_assigned_resource(node, rsc);
	node->details->num_resources++;
	node->count++;
	pcmk__consume_node_capacity(node->details->utilization, rsc);

	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_show_utilization)) {
	pcmk__output_t *out = rsc->cluster->priv;

	out->message(out, "resource-util", rsc, node, __func__);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Remove any node assignment from a specified resource and its children
	*
	* If a specified resource has been assigned to a node, remove that assignment
	* and mark the resource as provisional again.
	*
	* \param[in,out] rsc Resource to unassign
	*
	* \note This function is called recursively on \p rsc and its children.
	*/
	void
	pcmk__unassign_resource(pcmk_resource_t *rsc)
	{
	pcmk_node_t *old = rsc->allocated_to;

	if (old == NULL) {
	crm_info("Unassigning %s", rsc->id);
	} else {
	crm_info("Unassigning %s from %s", rsc->id, pe__node_name(old));
	}

	pe__set_resource_flags(rsc, pcmk_rsc_unassigned);

	if (rsc->children == NULL) {
	if (old == NULL) {
	return;
	}
	rsc->allocated_to = NULL;

	/* We're going to free the pcmk_node_t, but its details member is shared
	* and will remain, so update that appropriately first.
	*/
	old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc,
	rsc);
	old->details->num_resources--;
	pcmk__release_node_capacity(old->details->utilization, rsc);
	free(old);
	return;
	}

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk__unassign_resource((pcmk_resource_t *) iter->data);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has reached its migration threshold on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node Node to check
	* \param[out] failed If threshold has been reached, this will be set to
	* resource that failed (possibly a parent of \p rsc)
	*
	* \return true if the migration threshold has been reached, false otherwise
	*/
	bool
	pcmk__threshold_reached(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_resource_t **failed)
	{
	int fail_count, remaining_tries;
	pcmk_resource_t *rsc_to_ban = rsc;

	// Migration threshold of 0 means never force away
	if (rsc->migration_threshold == 0) {
	return false;
	}

	// If we're ignoring failures, also ignore the migration threshold
	if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	return false;
	}

	// If there are no failures, there's no need to force away
	fail_count = pe_get_failcount(node, rsc, NULL,
	pcmk__fc_effective\|pcmk__fc_fillers, NULL);
	if (fail_count <= 0) {
	return false;
	}

	// If failed resource is anonymous clone instance, we'll force clone away
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	rsc_to_ban = uber_parent(rsc);
	}

	// How many more times recovery will be tried on this node
	remaining_tries = rsc->migration_threshold - fail_count;

	if (remaining_tries <= 0) {
	- crm_warn("%s cannot run on %s due to reaching migration threshold "
	- "(clean up resource to allow again)"
	- CRM_XS " failures=%d migration-threshold=%d",
	- rsc_to_ban->id, pe__node_name(node), fail_count,
	- rsc->migration_threshold);
	+ pcmk__sched_warn("%s cannot run on %s due to reaching migration "
	+ "threshold (clean up resource to allow again)"
	+ CRM_XS " failures=%d migration-threshold=%d",
	+ rsc_to_ban->id, pe__node_name(node), fail_count,
	+ rsc->migration_threshold);
	if (failed != NULL) {
	*failed = rsc_to_ban;
	}
	return true;
	}

	crm_info("%s can fail %d more time%s on "
	"%s before reaching migration threshold (%d)",
	rsc_to_ban->id, remaining_tries, pcmk__plural_s(remaining_tries),
	pe__node_name(node), rsc->migration_threshold);
	return false;
	}

	/*!
	* \internal
	* \brief Get a node's score
	*
	* \param[in] node Node with ID to check
	* \param[in] nodes List of nodes to look for \p node score in
	*
	* \return Node's score, or -INFINITY if not found
	*/
	static int
	get_node_score(const pcmk_node_t node, GHashTable nodes)
	{
	pcmk_node_t *found_node = NULL;

	if ((node != NULL) && (nodes != NULL)) {
	found_node = g_hash_table_lookup(nodes, node->details->id);
	}
	return (found_node == NULL)? -INFINITY : found_node->weight;
	}

	/*!
	* \internal
	* \brief Compare two resources according to which should be assigned first
	*
	* \param[in] a First resource to compare
	* \param[in] b Second resource to compare
	* \param[in] data Sorted list of all nodes in cluster
	*
	* \return -1 if \p a should be assigned before \b, 0 if they are equal,
	* or +1 if \p a should be assigned after \b
	*/
	static gint
	cmp_resources(gconstpointer a, gconstpointer b, gpointer data)
	{
	/* GLib insists that this function require gconstpointer arguments, but we
	* make a small, temporary change to each argument (setting the
	* pe_rsc_merging flag) during comparison
	*/
	pcmk_resource_t resource1 = (pcmk_resource_t ) a;
	pcmk_resource_t resource2 = (pcmk_resource_t ) b;
	const GList *nodes = data;

	int rc = 0;
	int r1_score = -INFINITY;
	int r2_score = -INFINITY;
	pcmk_node_t *r1_node = NULL;
	pcmk_node_t *r2_node = NULL;
	GHashTable *r1_nodes = NULL;
	GHashTable *r2_nodes = NULL;
	const char *reason = NULL;

	// Resources with highest priority should be assigned first
	reason = "priority";
	r1_score = resource1->priority;
	r2_score = resource2->priority;
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// We need nodes to make any other useful comparisons
	reason = "no node list";
	if (nodes == NULL) {
	goto done;
	}

	// Calculate and log node scores
	resource1->cmds->add_colocated_node_scores(resource1, NULL, resource1->id,
	&r1_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	resource2->cmds->add_colocated_node_scores(resource2, NULL, resource2->id,
	&r2_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	pe__show_node_scores(true, NULL, resource1->id, r1_nodes,
	resource1->cluster);
	pe__show_node_scores(true, NULL, resource2->id, r2_nodes,
	resource2->cluster);

	// The resource with highest score on its current node goes first
	reason = "current location";
	if (resource1->running_on != NULL) {
	r1_node = pe__current_node(resource1);
	}
	if (resource2->running_on != NULL) {
	r2_node = pe__current_node(resource2);
	}
	r1_score = get_node_score(r1_node, r1_nodes);
	r2_score = get_node_score(r2_node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// Otherwise a higher score on any node will do
	reason = "score";
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	const pcmk_node_t node = (const pcmk_node_t ) iter->data;

	r1_score = get_node_score(node, r1_nodes);
	r2_score = get_node_score(node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}
	}

	done:
	crm_trace("%s (%d)%s%s %c %s (%d)%s%s: %s",
	resource1->id, r1_score,
	((r1_node == NULL)? "" : " on "),
	((r1_node == NULL)? "" : r1_node->details->id),
	((rc < 0)? '>' : ((rc > 0)? '<' : '=')),
	resource2->id, r2_score,
	((r2_node == NULL)? "" : " on "),
	((r2_node == NULL)? "" : r2_node->details->id),
	reason);
	if (r1_nodes != NULL) {
	g_hash_table_destroy(r1_nodes);
	}
	if (r2_nodes != NULL) {
	g_hash_table_destroy(r2_nodes);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Sort resources in the order they should be assigned to nodes
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__sort_resources(pcmk_scheduler_t *scheduler)
	{
	GList *nodes = g_list_copy(scheduler->nodes);

	nodes = pcmk__sort_nodes(nodes, NULL);
	scheduler->resources = g_list_sort_with_data(scheduler->resources,
	cmp_resources, nodes);
	g_list_free(nodes);
	}
	diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
	index 42cd1dcf6b..f1a32aabfe 100644
	--- a/lib/pengine/complex.c
	+++ b/lib/pengine/complex.c
	@@ -1,1199 +1,1199 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include "pe_status_private.h"

	void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);

	static pcmk_node_t active_node(const pcmk_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);

	pcmk_rsc_methods_t resource_class_functions[] = {
	{
	native_unpack,
	native_find_rsc,
	native_parameter,
	native_print,
	native_active,
	native_resource_state,
	native_location,
	native_free,
	pe__count_common,
	pe__native_is_filtered,
	active_node,
	pe__primitive_max_per_node,
	},
	{
	group_unpack,
	native_find_rsc,
	native_parameter,
	group_print,
	group_active,
	group_resource_state,
	native_location,
	group_free,
	pe__count_common,
	pe__group_is_filtered,
	active_node,
	pe__group_max_per_node,
	},
	{
	clone_unpack,
	native_find_rsc,
	native_parameter,
	clone_print,
	clone_active,
	clone_resource_state,
	native_location,
	clone_free,
	pe__count_common,
	pe__clone_is_filtered,
	active_node,
	pe__clone_max_per_node,
	},
	{
	pe__unpack_bundle,
	native_find_rsc,
	native_parameter,
	pe__print_bundle,
	pe__bundle_active,
	pe__bundle_resource_state,
	native_location,
	pe__free_bundle,
	pe__count_bundle,
	pe__bundle_is_filtered,
	pe__bundle_active_node,
	pe__bundle_max_per_node,
	}
	};

	static enum pe_obj_types
	get_resource_type(const char *name)
	{
	if (pcmk__str_eq(name, XML_CIB_TAG_RESOURCE, pcmk__str_casei)) {
	return pcmk_rsc_variant_primitive;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_GROUP, pcmk__str_casei)) {
	return pcmk_rsc_variant_group;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_INCARNATION, pcmk__str_casei)) {
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, PCMK_XE_PROMOTABLE_LEGACY, pcmk__str_casei)) {
	// @COMPAT deprecated since 2.0.0
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_CONTAINER, pcmk__str_casei)) {
	return pcmk_rsc_variant_bundle;
	}

	return pcmk_rsc_variant_unknown;
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	add_hash_param(user_data, key, value);
	}

	static void
	expand_parents_fixed_nvpairs(pcmk_resource_t *rsc,
	pe_rule_eval_data_t *rule_data,
	GHashTable meta_hash, pcmk_scheduler_t scheduler)
	{
	GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
	pcmk_resource_t *p = rsc->parent;

	if (p == NULL) {
	return ;
	}

	/* Search all parent resources, get the fixed value of "meta_attributes" set only in the original xml, and stack it in the hash table. */
	/* The fixed value of the lower parent resource takes precedence and is not overwritten. */
	while(p != NULL) {
	/* A hash table for comparison is generated, including the id-ref. */
	pe__unpack_dataset_nvpairs(p->xml, XML_TAG_META_SETS, rule_data,
	parent_orig_meta, NULL, FALSE, scheduler);
	p = p->parent;
	}

	/* If there is a fixed value of "meta_attributes" of the parent resource, it will be processed. */
	if (parent_orig_meta != NULL) {
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, parent_orig_meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	/* Parameters set in the original xml of the parent resource will also try to overwrite the child resource. */
	/* Attributes that already exist in the child lease are not updated. */
	dup_attr(key, value, meta_hash);
	}
	}

	if (parent_orig_meta != NULL) {
	g_hash_table_destroy(parent_orig_meta);
	}

	return ;

	}
	void
	get_meta_attributes(GHashTable * meta_hash, pcmk_resource_t * rsc,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE)
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->xml); a != NULL; a = a->next) {
	const char prop_name = (const char ) a->name;
	const char *prop_value = pcmk__xml_attr_value(a);

	add_hash_param(meta_hash, prop_name, prop_value);
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, &rule_data,
	meta_hash, NULL, FALSE, scheduler);

	/* Set the "meta_attributes" explicitly set in the parent resource to the hash table of the child resource. */
	/* If it is already explicitly set as a child, it will not be overwritten. */
	if (rsc->parent != NULL) {
	expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, scheduler);
	}

	/* check the defaults */
	pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_META_SETS,
	&rule_data, meta_hash, NULL, FALSE, scheduler);

	/* If there is "meta_attributes" that the parent resource has not explicitly set, set a value that is not set from rsc_default either. */
	/* The values already set up to this point will not be overwritten. */
	if (rsc->parent) {
	g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash);
	}
	}

	void
	get_rsc_attributes(GHashTable meta_hash, const pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, &rule_data,
	meta_hash, NULL, FALSE, scheduler);

	/* set anything else based on the parent */
	if (rsc->parent != NULL) {
	get_rsc_attributes(meta_hash, rsc->parent, node, scheduler);

	} else {
	/* and finally check the defaults */
	pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_ATTR_SETS,
	&rule_data, meta_hash, NULL, FALSE,
	scheduler);
	}
	}

	static char *
	template_op_key(xmlNode * op)
	{
	const char *name = crm_element_value(op, "name");
	const char *role = crm_element_value(op, "role");
	char *key = NULL;

	if ((role == NULL)
	\|\| pcmk__strcase_any_of(role, PCMK__ROLE_STARTED, PCMK__ROLE_UNPROMOTED,
	PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) {
	role = PCMK__ROLE_UNKNOWN;
	}

	key = crm_strdup_printf("%s-%s", name, role);
	return key;
	}

	static gboolean
	unpack_template(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *cib_resources = NULL;
	xmlNode *template = NULL;
	xmlNode *new_xml = NULL;
	xmlNode *child_xml = NULL;
	xmlNode *rsc_ops = NULL;
	xmlNode *template_ops = NULL;
	const char *template_ref = NULL;
	const char *clone = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pcmk__config_err("No resource object for template unpacking");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pcmk__config_err("The resource object '%s' should not reference itself",
	id);
	return FALSE;
	}

	cib_resources = get_xpath_object("//" XML_CIB_TAG_RESOURCES,
	scheduler->input, LOG_TRACE);
	if (cib_resources == NULL) {
	pcmk__config_err("No resources configured");
	return FALSE;
	}

	template = pcmk__xe_match(cib_resources, XML_CIB_TAG_RSC_TEMPLATE,
	XML_ATTR_ID, template_ref);
	if (template == NULL) {
	pcmk__config_err("No template named '%s'", template_ref);
	return FALSE;
	}

	new_xml = copy_xml(template);
	xmlNodeSetName(new_xml, xml_obj->name);
	crm_xml_add(new_xml, XML_ATTR_ID, id);

	clone = crm_element_value(xml_obj, XML_RSC_ATTR_INCARNATION);
	if(clone) {
	crm_xml_add(new_xml, XML_RSC_ATTR_INCARNATION, clone);
	}

	template_ops = find_xml_node(new_xml, "operations", FALSE);

	for (child_xml = pcmk__xe_first_child(xml_obj); child_xml != NULL;
	child_xml = pcmk__xe_next(child_xml)) {
	xmlNode *new_child = NULL;

	new_child = add_node_copy(new_xml, child_xml);

	if (pcmk__str_eq((const char *)new_child->name, "operations", pcmk__str_none)) {
	rsc_ops = new_child;
	}
	}

	if (template_ops && rsc_ops) {
	xmlNode *op = NULL;
	GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);

	for (op = pcmk__xe_first_child(rsc_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	g_hash_table_insert(rsc_ops_hash, key, op);
	}

	for (op = pcmk__xe_first_child(template_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
	add_node_copy(rsc_ops, op);
	}

	free(key);
	}

	if (rsc_ops_hash) {
	g_hash_table_destroy(rsc_ops_hash);
	}

	free_xml(template_ops);
	}

	/free_xml(expanded_xml); */
	*expanded_xml = new_xml;

	#if 0 /* Disable multi-level templates for now */
	if (!unpack_template(new_xml, expanded_xml, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return FALSE;
	}
	#endif

	return TRUE;
	}

	static gboolean
	add_template_rsc(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pcmk__config_err("No resource object for processing resource list "
	"of template");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pcmk__config_err("The resource object '%s' should not reference itself",
	id);
	return FALSE;
	}

	if (add_tag_ref(scheduler->template_rsc_sets, template_ref, id) == FALSE) {
	return FALSE;
	}

	return TRUE;
	}

	static bool
	detect_promotable(pcmk_resource_t *rsc)
	{
	const char *promotable = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_PROMOTABLE);

	if (crm_is_true(promotable)) {
	return TRUE;
	}

	// @COMPAT deprecated since 2.0.0
	if (pcmk__xe_is(rsc->xml, PCMK_XE_PROMOTABLE_LEGACY)) {
	/* @TODO in some future version, pe_warn_once() here,
	* then drop support in even later version
	*/
	g_hash_table_insert(rsc->meta, strdup(XML_RSC_ATTR_PROMOTABLE),
	strdup(XML_BOOLEAN_TRUE));
	return TRUE;
	}
	return FALSE;
	}

	static void
	free_params_table(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \brief Get a table of resource parameters
	*
	* \param[in,out] rsc Resource to query
	* \param[in] node Node for evaluating rules (NULL for defaults)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Hash table containing resource parameter names and values
	* (or NULL if \p rsc or \p scheduler is NULL)
	* \note The returned table will be destroyed when the resource is freed, so
	* callers should not destroy it.
	*/
	GHashTable *
	pe_rsc_params(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params_on_node = NULL;

	/* A NULL node is used to request the resource's default parameters
	* (not evaluated for node), but we always want something non-NULL
	* as a hash table key.
	*/
	const char *node_name = "";

	// Sanity check
	if ((rsc == NULL) \|\| (scheduler == NULL)) {
	return NULL;
	}
	if ((node != NULL) && (node->details->uname != NULL)) {
	node_name = node->details->uname;
	}

	// Find the parameter table for given node
	if (rsc->parameter_cache == NULL) {
	rsc->parameter_cache = pcmk__strikey_table(free, free_params_table);
	} else {
	params_on_node = g_hash_table_lookup(rsc->parameter_cache, node_name);
	}

	// If none exists yet, create one with parameters evaluated for node
	if (params_on_node == NULL) {
	params_on_node = pcmk__strkey_table(free, free);
	get_rsc_attributes(params_on_node, rsc, node, scheduler);
	g_hash_table_insert(rsc->parameter_cache, strdup(node_name),
	params_on_node);
	}
	return params_on_node;
	}

	/*!
	* \internal
	* \brief Unpack a resource's "requires" meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	* \param[in] value Value of "requires" meta-attribute
	* \param[in] is_default Whether \p value was selected by default
	*/
	static void
	unpack_requires(pcmk_resource_t rsc, const char value, bool is_default)
	{
	if (pcmk__str_eq(value, PCMK__VALUE_NOTHING, pcmk__str_casei)) {

	} else if (pcmk__str_eq(value, PCMK__VALUE_QUORUM, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pcmk_rsc_needs_quorum);

	} else if (pcmk__str_eq(value, PCMK__VALUE_FENCING, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pcmk_rsc_needs_fencing);
	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("%s requires fencing but fencing is disabled",
	rsc->id);
	}

	} else if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	pcmk__config_warn("Resetting \"" XML_RSC_ATTR_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"devices cannot require unfencing", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else if (!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Resetting \"" XML_RSC_ATTR_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"is disabled", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else {
	pe__set_resource_flags(rsc, pcmk_rsc_needs_fencing
	\|pcmk_rsc_needs_unfencing);
	}

	} else {
	const char *orig_value = value;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	value = PCMK__VALUE_QUORUM;

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& xml_contains_remote_node(rsc->xml)) {
	value = PCMK__VALUE_QUORUM;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_enable_unfencing)) {
	value = PCMK__VALUE_UNFENCING;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	value = PCMK__VALUE_FENCING;

	} else if (rsc->cluster->no_quorum_policy == pcmk_no_quorum_ignore) {
	value = PCMK__VALUE_NOTHING;

	} else {
	value = PCMK__VALUE_QUORUM;
	}

	if (orig_value != NULL) {
	pcmk__config_err("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
	"to '%s' because '%s' is not valid",
	rsc->id, value, orig_value);
	}
	unpack_requires(rsc, value, true);
	return;
	}

	pcmk__rsc_trace(rsc, "\tRequired to start: %s%s", value,
	(is_default? " (default)" : ""));
	}

	#ifndef PCMK__COMPAT_2_0
	static void
	warn_about_deprecated_classes(pcmk_resource_t *rsc)
	{
	const char *std = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_none)) {
	pe_warn_once(pcmk__wo_upstart,
	"Support for Upstart resources (such as %s) is deprecated "
	"and will be removed in a future release of Pacemaker",
	rsc->id);

	} else if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_none)) {
	pe_warn_once(pcmk__wo_nagios,
	"Support for Nagios resources (such as %s) is deprecated "
	"and will be removed in a future release of Pacemaker",
	rsc->id);
	}
	}
	#endif

	/*!
	* \internal
	* \brief Unpack configuration XML for a given resource
	*
	* Unpack the XML object containing a resource's configuration into a new
	* \c pcmk_resource_t object.
	*
	* \param[in] xml_obj XML node containing the resource's configuration
	* \param[out] rsc Where to store the unpacked resource information
	* \param[in] parent Resource's parent, if any
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	* \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and
	* the caller is responsible for freeing it using its variant-specific
	* free() method. Otherwise, \p *rsc is guaranteed to be NULL.
	*/
	int
	pe__unpack_resource(xmlNode xml_obj, pcmk_resource_t *rsc,
	pcmk_resource_t parent, pcmk_scheduler_t scheduler)
	{
	xmlNode *expanded_xml = NULL;
	xmlNode *ops = NULL;
	const char *value = NULL;
	const char *id = NULL;
	bool guest_node = false;
	bool remote_node = false;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = NULL,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	CRM_CHECK(rsc != NULL, return EINVAL);
	CRM_CHECK((xml_obj != NULL) && (scheduler != NULL),
	*rsc = NULL;
	return EINVAL);

	rule_data.now = scheduler->now;

	crm_log_xml_trace(xml_obj, "[raw XML]");

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> configuration without " XML_ATTR_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	if (unpack_template(xml_obj, &expanded_xml, scheduler) == FALSE) {
	return pcmk_rc_unpack_error;
	}

	*rsc = calloc(1, sizeof(pcmk_resource_t));
	if (*rsc == NULL) {
	- crm_crit("Unable to allocate memory for resource '%s'", id);
	+ pcmk__sched_err("Unable to allocate memory for resource '%s'", id);
	return ENOMEM;
	}
	(*rsc)->cluster = scheduler;

	if (expanded_xml) {
	crm_log_xml_trace(expanded_xml, "[expanded XML]");
	(*rsc)->xml = expanded_xml;
	(*rsc)->orig_xml = xml_obj;

	} else {
	(*rsc)->xml = xml_obj;
	(*rsc)->orig_xml = NULL;
	}

	/* Do not use xml_obj from here on, use (rsc)->xml in case templates are involved /

	(*rsc)->parent = parent;

	ops = find_xml_node((*rsc)->xml, "operations", FALSE);
	(*rsc)->ops_xml = expand_idref(ops, scheduler->input);

	(rsc)->variant = get_resource_type((const char ) (*rsc)->xml->name);
	if ((*rsc)->variant == pcmk_rsc_variant_unknown) {
	pcmk__config_err("Ignoring resource '%s' of unknown type '%s'",
	id, (*rsc)->xml->name);
	common_free(*rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	#ifndef PCMK__COMPAT_2_0
	warn_about_deprecated_classes(*rsc);
	#endif

	(*rsc)->meta = pcmk__strkey_table(free, free);
	(*rsc)->allowed_nodes = pcmk__strkey_table(NULL, free);
	(*rsc)->known_on = pcmk__strkey_table(NULL, free);

	value = crm_element_value((*rsc)->xml, XML_RSC_ATTR_INCARNATION);
	if (value) {
	(*rsc)->id = crm_strdup_printf("%s:%s", id, value);
	add_hash_param((*rsc)->meta, XML_RSC_ATTR_INCARNATION, value);

	} else {
	(*rsc)->id = strdup(id);
	}

	(rsc)->fns = &resource_class_functions[(rsc)->variant];

	get_meta_attributes((rsc)->meta, rsc, NULL, scheduler);
	(rsc)->parameters = pe_rsc_params(rsc, NULL, scheduler); // \deprecated

	(*rsc)->flags = 0;
	pe__set_resource_flags(*rsc, pcmk_rsc_runnable\|pcmk_rsc_unassigned);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_managed);
	}

	(*rsc)->rsc_cons = NULL;
	(*rsc)->rsc_tickets = NULL;
	(*rsc)->actions = NULL;
	(*rsc)->role = pcmk_role_stopped;
	(*rsc)->next_role = pcmk_role_unknown;

	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	(*rsc)->stickiness = 0;
	(*rsc)->migration_threshold = INFINITY;
	(*rsc)->failure_timeout = 0;

	value = g_hash_table_lookup((*rsc)->meta, XML_CIB_ATTR_PRIORITY);
	(*rsc)->priority = char2score(value);

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CRITICAL);
	if ((value == NULL) \|\| crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_critical);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_NOTIFY);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_notify);
	}

	if (xml_contains_remote_node((*rsc)->xml)) {
	(*rsc)->is_remote_node = TRUE;
	if (g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CONTAINER)) {
	guest_node = true;
	} else {
	remote_node = true;
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_OP_ATTR_ALLOW_MIGRATE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_migratable);
	} else if ((value == NULL) && remote_node) {
	/* By default, we want remote nodes to be able
	* to float around the cluster without having to stop all the
	* resources within the remote-node before moving. Allowing
	* migration support enables this feature. If this ever causes
	* problems, migration support can be explicitly turned off with
	* allow-migrate=false.
	*/
	pe__set_resource_flags(*rsc, pcmk_rsc_migratable);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MANAGED);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_managed);
	} else {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MAINTENANCE);
	if (crm_is_true(value)) {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	pe__set_resource_flags(*rsc, pcmk_rsc_maintenance);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	pe__set_resource_flags(*rsc, pcmk_rsc_maintenance);
	}

	if (pe_rsc_is_clone(pe__const_top_resource(*rsc, false))) {
	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_UNIQUE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_unique);
	}
	if (detect_promotable(*rsc)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_promotable);
	}
	} else {
	pe__set_resource_flags(*rsc, pcmk_rsc_unique);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_RESTART);
	if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	(*rsc)->restart_type = pe_restart_restart;
	pcmk__rsc_trace(*rsc, "%s dependency restart handling: restart",
	(*rsc)->id);
	pe_warn_once(pcmk__wo_restart_type,
	"Support for restart-type is deprecated and will be removed in a future release");

	} else {
	(*rsc)->restart_type = pe_restart_ignore;
	pcmk__rsc_trace(*rsc, "%s dependency restart handling: ignore",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MULTIPLE);
	if (pcmk__str_eq(value, "stop_only", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_stop;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: stop only",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_block;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: block",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "stop_unexpected", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_unexpected;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: "
	"stop unexpected instances",
	(*rsc)->id);

	} else { // "stop_start"
	if (!pcmk__str_eq(value, "stop_start",
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_warn("%s is not a valid value for "
	XML_RSC_ATTR_MULTIPLE
	", using default of \"stop_start\"", value);
	}
	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: stop/start",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_STICKINESS);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->stickiness = char2score(value);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MIGRATION_THRESHOLD);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->migration_threshold = char2score(value);
	if ((*rsc)->migration_threshold < 0) {
	/* @TODO We use 1 here to preserve previous behavior, but this
	* should probably use the default (INFINITY) or 0 (to disable)
	* instead.
	*/
	pe_warn_once(pcmk__wo_neg_threshold,
	PCMK_META_MIGRATION_THRESHOLD
	" must be non-negative, using 1 instead");
	(*rsc)->migration_threshold = 1;
	}
	}

	if (pcmk__str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS),
	PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pe__set_working_set_flags(scheduler, pcmk_sched_have_fencing);
	pe__set_resource_flags(*rsc, pcmk_rsc_fence_device);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_REQUIRES);
	unpack_requires(*rsc, value, false);

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_FAILURE_TIMEOUT);
	if (value != NULL) {
	// Stored as seconds
	(*rsc)->failure_timeout = (int) (crm_parse_interval_spec(value) / 1000);
	}

	if (remote_node) {
	GHashTable params = pe_rsc_params(rsc, NULL, scheduler);

	/* Grabbing the value now means that any rules based on node attributes
	* will evaluate to false, so such rules should not be used with
	* reconnect_interval.
	*
	* @TODO Evaluate per node before using
	*/
	value = g_hash_table_lookup(params, XML_REMOTE_ATTR_RECONNECT_INTERVAL);
	if (value) {
	/* reconnect delay works by setting failure_timeout and preventing the
	* connection from starting until the failure is cleared. */
	(*rsc)->remote_reconnect_ms = crm_parse_interval_spec(value);
	/* we want to override any default failure_timeout in use when remote
	* reconnect_interval is in use. */
	(rsc)->failure_timeout = (rsc)->remote_reconnect_ms / 1000;
	}
	}

	get_target_role(rsc, &((rsc)->next_role));
	pcmk__rsc_trace(rsc, "%s desired next state: %s", (rsc)->id,
	(rsc)->next_role != pcmk_role_unknown? role2text((rsc)->next_role) : "default");

	if ((rsc)->fns->unpack(rsc, scheduler) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	// This tag must stay exactly the same because it is tested elsewhere
	resource_location(*rsc, NULL, 0, "symmetric_default", scheduler);
	} else if (guest_node) {
	/* remote resources tied to a container resource must always be allowed
	* to opt-in to the cluster. Whether the connection resource is actually
	* allowed to be placed on a node is dependent on the container resource */
	resource_location(*rsc, NULL, 0, "remote_connection_default",
	scheduler);
	}

	pcmk__rsc_trace(rsc, "%s action notification: %s", (rsc)->id,
	pcmk_is_set((*rsc)->flags, pcmk_rsc_notify)? "required" : "not required");

	(*rsc)->utilization = pcmk__strkey_table(free, free);

	pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, &rule_data,
	(*rsc)->utilization, NULL, FALSE, scheduler);

	if (expanded_xml) {
	if (add_template_rsc(xml_obj, scheduler) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}
	}
	return pcmk_rc_ok;
	}

	gboolean
	is_parent(pcmk_resource_t child, pcmk_resource_t rsc)
	{
	pcmk_resource_t *parent = child;

	if (parent == NULL \|\| rsc == NULL) {
	return FALSE;
	}
	while (parent->parent != NULL) {
	if (parent->parent == rsc) {
	return TRUE;
	}
	parent = parent->parent;
	}
	return FALSE;
	}

	pcmk_resource_t *
	uber_parent(pcmk_resource_t *rsc)
	{
	pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while ((parent->parent != NULL)
	&& (parent->parent->variant != pcmk_rsc_variant_bundle)) {
	parent = parent->parent;
	}
	return parent;
	}

	/*!
	* \internal
	* \brief Get the topmost parent of a resource as a const pointer
	*
	* \param[in] rsc Resource to check
	* \param[in] include_bundle If true, go all the way to bundle
	*
	* \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent,
	* the bundle if \p rsc is bundled and \p include_bundle is true,
	* otherwise the topmost parent of \p rsc up to a clone
	*/
	const pcmk_resource_t *
	pe__const_top_resource(const pcmk_resource_t *rsc, bool include_bundle)
	{
	const pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while (parent->parent != NULL) {
	if (!include_bundle
	&& (parent->parent->variant == pcmk_rsc_variant_bundle)) {
	break;
	}
	parent = parent->parent;
	}
	return parent;
	}

	void
	common_free(pcmk_resource_t * rsc)
	{
	if (rsc == NULL) {
	return;
	}

	pcmk__rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant);

	g_list_free(rsc->rsc_cons);
	g_list_free(rsc->rsc_cons_lhs);
	g_list_free(rsc->rsc_tickets);
	g_list_free(rsc->dangling_migrations);

	if (rsc->parameter_cache != NULL) {
	g_hash_table_destroy(rsc->parameter_cache);
	}
	if (rsc->meta != NULL) {
	g_hash_table_destroy(rsc->meta);
	}
	if (rsc->utilization != NULL) {
	g_hash_table_destroy(rsc->utilization);
	}

	if ((rsc->parent == NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {

	free_xml(rsc->xml);
	rsc->xml = NULL;
	free_xml(rsc->orig_xml);
	rsc->orig_xml = NULL;

	/* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */
	} else if (rsc->orig_xml) {
	free_xml(rsc->xml);
	rsc->xml = NULL;
	}
	if (rsc->running_on) {
	g_list_free(rsc->running_on);
	rsc->running_on = NULL;
	}
	if (rsc->known_on) {
	g_hash_table_destroy(rsc->known_on);
	rsc->known_on = NULL;
	}
	if (rsc->actions) {
	g_list_free(rsc->actions);
	rsc->actions = NULL;
	}
	if (rsc->allowed_nodes) {
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = NULL;
	}
	g_list_free(rsc->fillers);
	g_list_free(rsc->rsc_location);
	pcmk__rsc_trace(rsc, "Resource freed");
	free(rsc->id);
	free(rsc->clone_name);
	free(rsc->allocated_to);
	free(rsc->variant_opaque);
	free(rsc->pending_task);
	free(rsc);
	}

	/*!
	* \internal
	* \brief Count a node and update most preferred to it as appropriate
	*
	* \param[in] rsc An active resource
	* \param[in] node A node that \p rsc is active on
	* \param[in,out] active This will be set to \p node if \p node is more
	* preferred than the current value
	* \param[in,out] count_all If not NULL, this will be incremented
	* \param[in,out] count_clean If not NULL, this will be incremented if \p node
	* is online and clean
	*
	* \return true if the count should continue, or false if sufficiently known
	*/
	bool
	pe__count_active_node(const pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_node_t *active, unsigned int count_all,
	unsigned int *count_clean)
	{
	bool keep_looking = false;
	bool is_happy = false;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL),
	return false);

	is_happy = node->details->online && !node->details->unclean;

	if (count_all != NULL) {
	++*count_all;
	}
	if ((count_clean != NULL) && is_happy) {
	++*count_clean;
	}
	if ((count_all != NULL) \|\| (count_clean != NULL)) {
	keep_looking = true; // We're counting, so go through entire list
	}

	if (rsc->partial_migration_source != NULL) {
	if (node->details == rsc->partial_migration_source->details) {
	*active = node; // This is the migration source
	} else {
	keep_looking = true;
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	if (is_happy && ((active == NULL) \|\| !(active)->details->online
	\|\| (*active)->details->unclean)) {
	*active = node; // This is the first clean node
	} else {
	keep_looking = true;
	}
	}
	if (*active == NULL) {
	*active = node; // This is the first node checked
	}
	return keep_looking;
	}

	// Shared implementation of pcmk_rsc_methods_t:active_node()
	static pcmk_node_t *
	active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	if (!pe__count_active_node(rsc, (pcmk_node_t *) iter->data, &active,
	count_all, count_clean)) {
	break; // Don't waste time iterating if we don't have to
	}
	}
	return active;
	}

	/*!
	* \brief
	* \internal Find and count active nodes according to "requires"
	*
	* \param[in] rsc Resource to check
	* \param[out] count If not NULL, will be set to count of active nodes
	*
	* \return An active node (or NULL if resource is not active anywhere)
	*
	* \note This is a convenience wrapper for active_node() where the count of all
	* active nodes or only clean active nodes is desired according to the
	* "requires" meta-attribute.
	*/
	pcmk_node_t *
	pe__find_active_requires(const pcmk_resource_t rsc, unsigned int count)
	{
	if (rsc == NULL) {
	if (count != NULL) {
	*count = 0;
	}
	return NULL;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	return rsc->fns->active_node(rsc, count, NULL);

	} else {
	return rsc->fns->active_node(rsc, NULL, count);
	}
	}

	void
	pe__count_common(pcmk_resource_t *rsc)
	{
	if (rsc->children != NULL) {
	for (GList *item = rsc->children; item != NULL; item = item->next) {
	((pcmk_resource_t *) item->data)->fns->count(item->data);
	}

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)
	\|\| (rsc->role > pcmk_role_stopped)) {
	rsc->cluster->ninstances++;
	if (pe__resource_is_disabled(rsc)) {
	rsc->cluster->disabled_resources++;
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	rsc->cluster->blocked_resources++;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource's next role
	*
	* \param[in,out] rsc Resource to be updated
	* \param[in] role Resource's new next role
	* \param[in] why Human-friendly reason why role is changing (for logs)
	*/
	void
	pe__set_next_role(pcmk_resource_t rsc, enum rsc_role_e role, const char why)
	{
	CRM_ASSERT((rsc != NULL) && (why != NULL));
	if (rsc->next_role != role) {
	pcmk__rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
	rsc->id, role2text(rsc->next_role), role2text(role),
	why);
	rsc->next_role = role;
	}
	}
	diff --git a/lib/pengine/pe_digest.c b/lib/pengine/pe_digest.c
	index ed097b6368..0c5c865c36 100644
	--- a/lib/pengine/pe_digest.c
	+++ b/lib/pengine/pe_digest.c
	@@ -1,607 +1,608 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/pengine/internal.h>
	#include "pe_status_private.h"

	extern bool pcmk__is_daemon;

	/*!
	* \internal
	* \brief Free an operation digest cache entry
	*
	* \param[in,out] ptr Pointer to cache entry to free
	*
	* \note The argument is a gpointer so this can be used as a hash table
	* free function.
	*/
	void
	pe__free_digests(gpointer ptr)
	{
	pcmk__op_digest_t *data = ptr;

	if (data != NULL) {
	free_xml(data->params_all);
	free_xml(data->params_secure);
	free_xml(data->params_restart);

	free(data->digest_all_calc);
	free(data->digest_restart_calc);
	free(data->digest_secure_calc);

	free(data);
	}
	}

	// Return true if XML attribute name is not substring of a given string
	static bool
	attr_not_in_string(xmlAttrPtr a, void *user_data)
	{
	bool filter = false;
	char name = crm_strdup_printf(" %s ", (const char ) a->name);

	if (strstr((const char *) user_data, name) == NULL) {
	crm_trace("Filtering %s (not found in '%s')",
	(const char ) a->name, (const char ) user_data);
	filter = true;
	}
	free(name);
	return filter;
	}

	// Return true if XML attribute name is substring of a given string
	static bool
	attr_in_string(xmlAttrPtr a, void *user_data)
	{
	bool filter = false;
	char name = crm_strdup_printf(" %s ", (const char ) a->name);

	if (strstr((const char *) user_data, name) != NULL) {
	crm_trace("Filtering %s (found in '%s')",
	(const char ) a->name, (const char ) user_data);
	filter = true;
	}
	free(name);
	return filter;
	}

	/*!
	* \internal
	* \brief Add digest of all parameters to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in,out] rsc Resource that action was for
	* \param[in] node Node action was performed on
	* \param[in] params Resource parameters evaluated for node
	* \param[in] task Name of action performed
	* \param[in,out] interval_ms Action's interval (will be reset if in overrides)
	* \param[in] xml_op Unused
	* \param[in] op_version CRM feature set to use for digest calculation
	* \param[in] overrides Key/value table to override resource parameters
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	calculate_main_digest(pcmk__op_digest_t data, pcmk_resource_t rsc,
	const pcmk_node_t node, GHashTable params,
	const char task, guint interval_ms,
	const xmlNode xml_op, const char op_version,
	GHashTable overrides, pcmk_scheduler_t scheduler)
	{
	xmlNode *action_config = NULL;

	data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);

	/* REMOTE_CONTAINER_HACK: Allow Pacemaker Remote nodes to run containers
	* that themselves are Pacemaker Remote nodes
	*/
	(void) pe__add_bundle_remote_name(rsc, scheduler, data->params_all,
	XML_RSC_ATTR_REMOTE_RA_ADDR);

	if (overrides != NULL) {
	// If interval was overridden, reset it
	const char *interval_s = g_hash_table_lookup(overrides, CRM_META "_"
	XML_LRM_ATTR_INTERVAL);

	if (interval_s != NULL) {
	long long value_ll;

	if ((pcmk__scan_ll(interval_s, &value_ll, 0LL) == pcmk_rc_ok)
	&& (value_ll >= 0) && (value_ll <= G_MAXUINT)) {
	*interval_ms = (guint) value_ll;
	}
	}

	// Add overrides to list of all parameters
	g_hash_table_foreach(overrides, hash2field, data->params_all);
	}

	// Add provided instance parameters
	g_hash_table_foreach(params, hash2field, data->params_all);

	// Find action configuration XML in CIB
	action_config = pcmk__find_action_config(rsc, task, *interval_ms, true);

	/* Add action-specific resource instance attributes to the digest list.
	*
	* If this is a one-time action with action-specific instance attributes,
	* enforce a restart instead of reload-agent in case the main digest doesn't
	* match, even if the restart digest does. This ensures any changes of the
	* action-specific parameters get applied for this specific action, and
	* digests calculated for the resulting history will be correct. Default the
	* result to RSC_DIGEST_RESTART for the case where the main digest doesn't
	* match.
	*/
	params = pcmk__unpack_action_rsc_params(action_config, node->details->attrs,
	scheduler);
	if ((*interval_ms == 0) && (g_hash_table_size(params) > 0)) {
	data->rc = pcmk__digest_restart;
	}
	g_hash_table_foreach(params, hash2field, data->params_all);
	g_hash_table_destroy(params);

	// Add action meta-attributes
	params = pcmk__unpack_action_meta(rsc, node, task, *interval_ms,
	action_config);
	g_hash_table_foreach(params, hash2metafield, data->params_all);
	g_hash_table_destroy(params);

	pcmk__filter_op_for_digest(data->params_all);

	data->digest_all_calc = calculate_operation_digest(data->params_all,
	op_version);
	}

	// Return true if XML attribute name is a Pacemaker-defined fencing parameter
	static bool
	is_fence_param(xmlAttrPtr attr, void *user_data)
	{
	return pcmk_stonith_param((const char *) attr->name);
	}

	/*!
	* \internal
	* \brief Add secure digest to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in] rsc Resource that action was for
	* \param[in] params Resource parameters evaluated for node
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] op_version CRM feature set to use for digest calculation
	* \param[in] overrides Key/value hash table to override resource parameters
	*/
	static void
	calculate_secure_digest(pcmk__op_digest_t data, const pcmk_resource_t rsc,
	GHashTable params, const xmlNode xml_op,
	const char op_version, GHashTable overrides)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *secure_list = NULL;
	bool old_version = (compare_version(op_version, "3.16.0") < 0);

	if (xml_op == NULL) {
	secure_list = " passwd password user ";
	} else {
	secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
	}

	if (old_version) {
	data->params_secure = create_xml_node(NULL, XML_TAG_PARAMS);
	if (overrides != NULL) {
	g_hash_table_foreach(overrides, hash2field, data->params_secure);
	}

	g_hash_table_foreach(params, hash2field, data->params_secure);

	} else {
	// Start with a copy of all parameters
	data->params_secure = copy_xml(data->params_all);
	}

	if (secure_list != NULL) {
	pcmk__xe_remove_matching_attrs(data->params_secure, attr_in_string,
	(void *) secure_list);
	}
	if (old_version
	&& pcmk_is_set(pcmk_get_ra_caps(class),
	pcmk_ra_cap_fence_params)) {
	/* For stonith resources, Pacemaker adds special parameters,
	* but these are not listed in fence agent meta-data, so with older
	* versions of DC, the controller will not hash them. That means we have
	* to filter them out before calculating our hash for comparison.
	*/
	pcmk__xe_remove_matching_attrs(data->params_secure, is_fence_param,
	NULL);
	}
	pcmk__filter_op_for_digest(data->params_secure);

	/* CRM_meta_timeout should be part of a digest for recurring operations.
	* However, with older versions of DC, the controller does not add timeout
	* to secure digests, because it only includes parameters declared by the
	* resource agent.
	* Remove any timeout that made it this far, to match.
	*/
	if (old_version) {
	xml_remove_prop(data->params_secure, CRM_META "_" XML_ATTR_TIMEOUT);
	}

	data->digest_secure_calc = calculate_operation_digest(data->params_secure,
	op_version);
	}

	/*!
	* \internal
	* \brief Add restart digest to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] op_version CRM feature set to use for digest calculation
	*
	* \note This function doesn't need to handle overrides because it starts with
	* data->params_all, which already has overrides applied.
	*/
	static void
	calculate_restart_digest(pcmk__op_digest_t data, const xmlNode xml_op,
	const char *op_version)
	{
	const char *value = NULL;

	// We must have XML of resource operation history
	if (xml_op == NULL) {
	return;
	}

	// And the history must have a restart digest to compare against
	if (crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) == NULL) {
	return;
	}

	// Start with a copy of all parameters
	data->params_restart = copy_xml(data->params_all);

	// Then filter out reloadable parameters, if any
	value = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);
	if (value != NULL) {
	pcmk__xe_remove_matching_attrs(data->params_restart, attr_not_in_string,
	(void *) value);
	}

	value = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	data->digest_restart_calc = calculate_operation_digest(data->params_restart,
	value);
	}

	/*!
	* \internal
	* \brief Create a new digest cache entry with calculated digests
	*
	* \param[in,out] rsc Resource that action was for
	* \param[in] task Name of action performed
	* \param[in,out] interval_ms Action's interval (will be reset if in overrides)
	* \param[in] node Node action was performed on
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] overrides Key/value table to override resource parameters
	* \param[in] calc_secure Whether to calculate secure digest
	* \param[in,out] scheduler Scheduler data
	*
	* \return Pointer to new digest cache entry (or NULL on memory error)
	* \note It is the caller's responsibility to free the result using
	* pe__free_digests().
	*/
	pcmk__op_digest_t *
	pe__calculate_digests(pcmk_resource_t rsc, const char task,
	guint interval_ms, const pcmk_node_t node,
	const xmlNode xml_op, GHashTable overrides,
	bool calc_secure, pcmk_scheduler_t *scheduler)
	{
	pcmk__op_digest_t *data = calloc(1, sizeof(pcmk__op_digest_t));
	const char *op_version = NULL;
	GHashTable *params = NULL;

	if (data == NULL) {
	+ pcmk__sched_err("Could not allocate memory for operation digest");
	return NULL;
	}

	data->rc = pcmk__digest_match;

	if (xml_op != NULL) {
	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	}

	if (op_version == NULL && scheduler != NULL && scheduler->input != NULL) {
	op_version = crm_element_value(scheduler->input, XML_ATTR_CRM_VERSION);
	}

	if (op_version == NULL) {
	op_version = CRM_FEATURE_SET;
	}

	params = pe_rsc_params(rsc, node, scheduler);
	calculate_main_digest(data, rsc, node, params, task, interval_ms, xml_op,
	op_version, overrides, scheduler);
	if (calc_secure) {
	calculate_secure_digest(data, rsc, params, xml_op, op_version,
	overrides);
	}
	calculate_restart_digest(data, xml_op, op_version);
	return data;
	}

	/*!
	* \internal
	* \brief Calculate action digests and store in node's digest cache
	*
	* \param[in,out] rsc Resource that action was for
	* \param[in] task Name of action performed
	* \param[in] interval_ms Action's interval
	* \param[in,out] node Node action was performed on
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] calc_secure Whether to calculate secure digest
	* \param[in,out] scheduler Scheduler data
	*
	* \return Pointer to node's digest cache entry
	*/
	static pcmk__op_digest_t *
	rsc_action_digest(pcmk_resource_t rsc, const char task, guint interval_ms,
	pcmk_node_t node, const xmlNode xml_op,
	bool calc_secure, pcmk_scheduler_t *scheduler)
	{
	pcmk__op_digest_t *data = NULL;
	char *key = pcmk__op_key(rsc->id, task, interval_ms);

	data = g_hash_table_lookup(node->details->digest_cache, key);
	if (data == NULL) {
	data = pe__calculate_digests(rsc, task, &interval_ms, node, xml_op,
	NULL, calc_secure, scheduler);
	CRM_ASSERT(data != NULL);
	g_hash_table_insert(node->details->digest_cache, strdup(key), data);
	}
	free(key);
	return data;
	}

	/*!
	* \internal
	* \brief Calculate operation digests and compare against an XML history entry
	*
	* \param[in,out] rsc Resource to check
	* \param[in] xml_op Resource history XML
	* \param[in,out] node Node to use for digest calculation
	* \param[in,out] scheduler Scheduler data
	*
	* \return Pointer to node's digest cache entry, with comparison result set
	*/
	pcmk__op_digest_t *
	rsc_action_digest_cmp(pcmk_resource_t rsc, const xmlNode xml_op,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pcmk__op_digest_t *data = NULL;
	guint interval_ms = 0;

	const char *op_version;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *digest_all;
	const char *digest_restart;

	CRM_ASSERT(node != NULL);

	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
	digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	data = rsc_action_digest(rsc, task, interval_ms, node, xml_op,
	pcmk_is_set(scheduler->flags,
	pcmk_sched_sanitized),
	scheduler);

	if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) {
	pcmk__rsc_info(rsc,
	"Parameters to %ums-interval %s action for %s on %s "
	"changed: hash was %s vs. now %s (restart:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_restart, "missing"),
	data->digest_restart_calc, op_version,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	data->rc = pcmk__digest_restart;

	} else if (digest_all == NULL) {
	/* it is unknown what the previous op digest was */
	data->rc = pcmk__digest_unknown;

	} else if (strcmp(digest_all, data->digest_all_calc) != 0) {
	/* Given a non-recurring operation with extra parameters configured,
	* in case that the main digest doesn't match, even if the restart
	* digest matches, enforce a restart rather than a reload-agent anyway.
	* So that it ensures any changes of the extra parameters get applied
	* for this specific operation, and the digests calculated for the
	* resulting lrm_rsc_op will be correct.
	* Preserve the implied rc pcmk__digest_restart for the case that the
	* main digest doesn't match.
	*/
	if ((interval_ms == 0) && (data->rc == pcmk__digest_restart)) {
	pcmk__rsc_info(rsc,
	"Parameters containing extra ones to %ums-interval"
	" %s action for %s on %s "
	"changed: hash was %s vs. now %s (restart:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_all, "missing"),
	data->digest_all_calc, op_version,
	crm_element_value(xml_op,
	XML_ATTR_TRANSITION_MAGIC));

	} else {
	pcmk__rsc_info(rsc,
	"Parameters to %ums-interval %s action for %s on %s "
	"changed: hash was %s vs. now %s (%s:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_all, "missing"),
	data->digest_all_calc,
	(interval_ms > 0)? "reschedule" : "reload",
	op_version,
	crm_element_value(xml_op,
	XML_ATTR_TRANSITION_MAGIC));
	data->rc = pcmk__digest_mismatch;
	}

	} else {
	data->rc = pcmk__digest_match;
	}
	return data;
	}

	/*!
	* \internal
	* \brief Create an unfencing summary for use in special node attribute
	*
	* Create a string combining a fence device's resource ID, agent type, and
	* parameter digest (whether for all parameters or just non-private parameters).
	* This can be stored in a special node attribute, allowing us to detect changes
	* in either the agent type or parameters, to know whether unfencing must be
	* redone or can be safely skipped when the device's history is cleaned.
	*
	* \param[in] rsc_id Fence device resource ID
	* \param[in] agent_type Fence device agent
	* \param[in] param_digest Fence device parameter digest
	*
	* \return Newly allocated string with unfencing digest
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	create_unfencing_summary(const char rsc_id, const char agent_type,
	const char *param_digest)
	{
	return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest);
	}

	/*!
	* \internal
	* \brief Check whether a node can skip unfencing
	*
	* Check whether a fence device's current definition matches a node's
	* stored summary of when it was last unfenced by the device.
	*
	* \param[in] rsc_id Fence device's resource ID
	* \param[in] agent Fence device's agent type
	* \param[in] digest_calc Fence device's current parameter digest
	* \param[in] node_summary Value of node's special unfencing node attribute
	* (a comma-separated list of unfencing summaries for
	* all devices that have unfenced this node)
	*
	* \return TRUE if digest matches, FALSE otherwise
	*/
	static bool
	unfencing_digest_matches(const char rsc_id, const char agent,
	const char digest_calc, const char node_summary)
	{
	bool matches = FALSE;

	if (rsc_id && agent && digest_calc && node_summary) {
	char *search_secure = create_unfencing_summary(rsc_id, agent,
	digest_calc);

	/* The digest was calculated including the device ID and agent,
	* so there is no risk of collision using strstr().
	*/
	matches = (strstr(node_summary, search_secure) != NULL);
	crm_trace("Calculated unfencing digest '%s' %sfound in '%s'",
	search_secure, matches? "" : "not ", node_summary);
	free(search_secure);
	}
	return matches;
	}

	/* Magic string to use as action name for digest cache entries used for
	* unfencing checks. This is not a real action name (i.e. "on"), so
	* pcmk__check_action_config() won't confuse these entries with real actions.
	*/
	#define STONITH_DIGEST_TASK "stonith-on"

	/*!
	* \internal
	* \brief Calculate fence device digests and digest comparison result
	*
	* \param[in,out] rsc Fence device resource
	* \param[in] agent Fence device's agent type
	* \param[in,out] node Node with digest cache to use
	* \param[in,out] scheduler Scheduler data
	*
	* \return Node's digest cache entry
	*/
	pcmk__op_digest_t *
	pe__compare_fencing_digest(pcmk_resource_t rsc, const char agent,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	const char *node_summary = NULL;

	// Calculate device's current parameter digests
	pcmk__op_digest_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, 0U,
	node, NULL, TRUE, scheduler);

	// Check whether node has special unfencing summary node attribute
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL);
	if (node_summary == NULL) {
	data->rc = pcmk__digest_unknown;
	return data;
	}

	// Check whether full parameter digest matches
	if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc,
	node_summary)) {
	data->rc = pcmk__digest_match;
	return data;
	}

	// Check whether secure parameter digest matches
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE);
	if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc,
	node_summary)) {
	data->rc = pcmk__digest_match;
	if (!pcmk__is_daemon && scheduler->priv != NULL) {
	pcmk__output_t *out = scheduler->priv;
	out->info(out, "Only 'private' parameters to %s "
	"for unfencing %s changed", rsc->id,
	pe__node_name(node));
	}
	return data;
	}

	// Parameters don't match
	data->rc = pcmk__digest_mismatch;
	if (pcmk_is_set(scheduler->flags, pcmk_sched_sanitized)
	&& (data->digest_secure_calc != NULL)) {

	if (scheduler->priv != NULL) {
	pcmk__output_t *out = scheduler->priv;
	char *digest = create_unfencing_summary(rsc->id, agent,
	data->digest_secure_calc);

	out->info(out, "Parameters to %s for unfencing "
	"%s changed, try '%s'", rsc->id,
	pe__node_name(node), digest);
	free(digest);
	} else if (!pcmk__is_daemon) {
	char *digest = create_unfencing_summary(rsc->id, agent,
	data->digest_secure_calc);

	printf("Parameters to %s for unfencing %s changed, try '%s'\n",
	rsc->id, pe__node_name(node), digest);
	free(digest);
	}
	}
	return data;
	}
	diff --git a/lib/pengine/pe_notif.c b/lib/pengine/pe_notif.c
	index 111592342e..f3725e8ebc 100644
	--- a/lib/pengine/pe_notif.c
	+++ b/lib/pengine/pe_notif.c
	@@ -1,1008 +1,1008 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>

	#include <crm/pengine/internal.h>
	#include <pacemaker-internal.h>

	#include "pe_status_private.h"

	typedef struct notify_entry_s {
	const pcmk_resource_t *rsc;
	const pcmk_node_t *node;
	} notify_entry_t;

	/*!
	* \internal
	* \brief Compare two notification entries
	*
	* Compare two notification entries, where the one with the alphabetically first
	* resource name (or if equal, node name) sorts as first, with NULL sorting as
	* less than non-NULL.
	*
	* \param[in] a First notification entry to compare
	* \param[in] b Second notification entry to compare
	*
	* \return -1 if \p a sorts before \p b, 0 if they are equal, otherwise 1
	*/
	static gint
	compare_notify_entries(gconstpointer a, gconstpointer b)
	{
	int tmp;
	const notify_entry_t *entry_a = a;
	const notify_entry_t *entry_b = b;

	// NULL a or b is not actually possible
	if ((entry_a == NULL) && (entry_b == NULL)) {
	return 0;
	}
	if (entry_a == NULL) {
	return 1;
	}
	if (entry_b == NULL) {
	return -1;
	}

	// NULL resources sort first
	if ((entry_a->rsc == NULL) && (entry_b->rsc == NULL)) {
	return 0;
	}
	if (entry_a->rsc == NULL) {
	return 1;
	}
	if (entry_b->rsc == NULL) {
	return -1;
	}

	// Compare resource names
	tmp = strcmp(entry_a->rsc->id, entry_b->rsc->id);
	if (tmp != 0) {
	return tmp;
	}

	// Otherwise NULL nodes sort first
	if ((entry_a->node == NULL) && (entry_b->node == NULL)) {
	return 0;
	}
	if (entry_a->node == NULL) {
	return 1;
	}
	if (entry_b->node == NULL) {
	return -1;
	}

	// Finally, compare node names
	return strcmp(entry_a->node->details->id, entry_b->node->details->id);
	}

	/*!
	* \internal
	* \brief Duplicate a notification entry
	*
	* \param[in] entry Entry to duplicate
	*
	* \return Newly allocated duplicate of \p entry
	* \note It is the caller's responsibility to free the return value.
	*/
	static notify_entry_t *
	dup_notify_entry(const notify_entry_t *entry)
	{
	notify_entry_t *dup = calloc(1, sizeof(notify_entry_t));

	CRM_ASSERT(dup != NULL);
	dup->rsc = entry->rsc;
	dup->node = entry->node;
	return dup;
	}

	/*!
	* \internal
	* \brief Given a list of nodes, create strings with node names
	*
	* \param[in] list List of nodes (as pcmk_node_t *)
	* \param[out] all_node_names If not NULL, will be set to space-separated list
	* of the names of all nodes in \p list
	* \param[out] host_node_names Same as \p all_node_names, except active
	* guest nodes will list the name of their host
	*
	* \note The caller is responsible for freeing the output argument values using
	* \p g_string_free().
	*/
	static void
	get_node_names(const GList list, GString *all_node_names,
	GString **host_node_names)
	{
	if (all_node_names != NULL) {
	*all_node_names = NULL;
	}
	if (host_node_names != NULL) {
	*host_node_names = NULL;
	}

	for (const GList *iter = list; iter != NULL; iter = iter->next) {
	const pcmk_node_t node = (const pcmk_node_t ) iter->data;

	if (node->details->uname == NULL) {
	continue;
	}

	// Always add to list of all node names
	if (all_node_names != NULL) {
	pcmk__add_word(all_node_names, 1024, node->details->uname);
	}

	// Add to host node name list if appropriate
	if (host_node_names != NULL) {
	if (pe__is_guest_node(node)
	&& (node->details->remote_rsc->container->running_on != NULL)) {
	node = pe__current_node(node->details->remote_rsc->container);
	if (node->details->uname == NULL) {
	continue;
	}
	}
	pcmk__add_word(host_node_names, 1024, node->details->uname);
	}
	}

	if ((all_node_names != NULL) && (*all_node_names == NULL)) {
	*all_node_names = g_string_new(" ");
	}
	if ((host_node_names != NULL) && (*host_node_names == NULL)) {
	*host_node_names = g_string_new(" ");
	}
	}

	/*!
	* \internal
	* \brief Create strings of instance and node names from notification entries
	*
	* \param[in,out] list List of notification entries (will be sorted here)
	* \param[out] rsc_names If not NULL, will be set to space-separated list
	* of clone instances from \p list
	* \param[out] node_names If not NULL, will be set to space-separated list
	* of node names from \p list
	*
	* \return (Possibly new) head of sorted \p list
	* \note The caller is responsible for freeing the output argument values using
	* \p g_list_free_full() and \p g_string_free().
	*/
	static GList *
	notify_entries_to_strings(GList list, GString *rsc_names,
	GString **node_names)
	{
	const char *last_rsc_id = NULL;

	// Initialize output lists to NULL
	if (rsc_names != NULL) {
	*rsc_names = NULL;
	}
	if (node_names != NULL) {
	*node_names = NULL;
	}

	// Sort input list for user-friendliness (and ease of filtering duplicates)
	list = g_list_sort(list, compare_notify_entries);

	for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	notify_entry_t entry = (notify_entry_t ) gIter->data;

	// Entry must have a resource (with ID)
	CRM_LOG_ASSERT((entry != NULL) && (entry->rsc != NULL)
	&& (entry->rsc->id != NULL));
	if ((entry == NULL) \|\| (entry->rsc == NULL)
	\|\| (entry->rsc->id == NULL)) {
	continue;
	}

	// Entry must have a node unless listing inactive resources
	CRM_LOG_ASSERT((node_names == NULL) \|\| (entry->node != NULL));
	if ((node_names != NULL) && (entry->node == NULL)) {
	continue;
	}

	// Don't add duplicates of a particular clone instance
	if (pcmk__str_eq(entry->rsc->id, last_rsc_id, pcmk__str_none)) {
	continue;
	}
	last_rsc_id = entry->rsc->id;

	if (rsc_names != NULL) {
	pcmk__add_word(rsc_names, 1024, entry->rsc->id);
	}
	if ((node_names != NULL) && (entry->node->details->uname != NULL)) {
	pcmk__add_word(node_names, 1024, entry->node->details->uname);
	}
	}

	// If there are no entries, return "empty" lists
	if ((rsc_names != NULL) && (*rsc_names == NULL)) {
	*rsc_names = g_string_new(" ");
	}
	if ((node_names != NULL) && (*node_names == NULL)) {
	*node_names = g_string_new(" ");
	}

	return list;
	}

	/*!
	* \internal
	* \brief Copy a meta-attribute into a notify action
	*
	* \param[in] key Name of meta-attribute to copy
	* \param[in] value Value of meta-attribute to copy
	* \param[in,out] user_data Notify action to copy into
	*/
	static void
	copy_meta_to_notify(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk_action_t notify = (pcmk_action_t ) user_data;

	/* Any existing meta-attributes (for example, the action timeout) are for
	* the notify action itself, so don't override those.
	*/
	if (g_hash_table_lookup(notify->meta, (const char *) key) != NULL) {
	return;
	}

	g_hash_table_insert(notify->meta, strdup((const char *) key),
	strdup((const char *) value));
	}

	static void
	add_notify_data_to_action_meta(const notify_data_t *n_data,
	pcmk_action_t *action)
	{
	for (const GSList *item = n_data->keys; item; item = item->next) {
	const pcmk_nvpair_t nvpair = (const pcmk_nvpair_t ) item->data;

	add_hash_param(action->meta, nvpair->name, nvpair->value);
	}
	}

	/*!
	* \internal
	* \brief Create a new notify pseudo-action for a clone resource
	*
	* \param[in,out] rsc Clone resource that notification is for
	* \param[in] action Action to use in notify action key
	* \param[in] notif_action PCMK_ACTION_NOTIFY or PCMK_ACTION_NOTIFIED
	* \param[in] notif_type "pre", "post", "confirmed-pre", "confirmed-post"
	*
	* \return Newly created notify pseudo-action
	*/
	static pcmk_action_t *
	new_notify_pseudo_action(pcmk_resource_t rsc, const pcmk_action_t action,
	const char notif_action, const char notif_type)
	{
	pcmk_action_t *notify = NULL;

	notify = custom_action(rsc,
	pcmk__notify_key(rsc->id, notif_type, action->task),
	notif_action, NULL,
	pcmk_is_set(action->flags, pcmk_action_optional),
	rsc->cluster);
	pe__set_action_flags(notify, pcmk_action_pseudo);
	add_hash_param(notify->meta, "notify_key_type", notif_type);
	add_hash_param(notify->meta, "notify_key_operation", action->task);
	return notify;
	}

	/*!
	* \internal
	* \brief Create a new notify action for a clone instance
	*
	* \param[in,out] rsc Clone instance that notification is for
	* \param[in] node Node that notification is for
	* \param[in,out] op Action that notification is for
	* \param[in,out] notify_done Parent pseudo-action for notifications complete
	* \param[in] n_data Notification values to add to action meta-data
	*
	* \return Newly created notify action
	*/
	static pcmk_action_t *
	new_notify_action(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_action_t op, pcmk_action_t notify_done,
	const notify_data_t *n_data)
	{
	char *key = NULL;
	pcmk_action_t *notify_action = NULL;
	const char *value = NULL;
	const char *task = NULL;
	const char *skip_reason = NULL;

	CRM_CHECK((rsc != NULL) && (node != NULL), return NULL);

	// Ensure we have all the info we need
	if (op == NULL) {
	skip_reason = "no action";
	} else if (notify_done == NULL) {
	skip_reason = "no parent notification";
	} else if (!node->details->online) {
	skip_reason = "node offline";
	} else if (!pcmk_is_set(op->flags, pcmk_action_runnable)) {
	skip_reason = "original action not runnable";
	}
	if (skip_reason != NULL) {
	pcmk__rsc_trace(rsc, "Skipping notify action for %s on %s: %s",
	rsc->id, pe__node_name(node), skip_reason);
	return NULL;
	}

	value = g_hash_table_lookup(op->meta, "notify_type"); // "pre" or "post"
	task = g_hash_table_lookup(op->meta, "notify_operation"); // original action

	pcmk__rsc_trace(rsc, "Creating notify action for %s on %s (%s-%s)",
	rsc->id, pe__node_name(node), value, task);

	// Create the notify action
	key = pcmk__notify_key(rsc->id, value, task);
	notify_action = custom_action(rsc, key, op->task, node,
	pcmk_is_set(op->flags, pcmk_action_optional),
	rsc->cluster);

	// Add meta-data to notify action
	g_hash_table_foreach(op->meta, copy_meta_to_notify, notify_action);
	add_notify_data_to_action_meta(n_data, notify_action);

	// Order notify after original action and before parent notification
	order_actions(op, notify_action, pcmk__ar_ordered);
	order_actions(notify_action, notify_done, pcmk__ar_ordered);
	return notify_action;
	}

	/*!
	* \internal
	* \brief Create a new "post-" notify action for a clone instance
	*
	* \param[in,out] rsc Clone instance that notification is for
	* \param[in] node Node that notification is for
	* \param[in,out] n_data Notification values to add to action meta-data
	*/
	static void
	new_post_notify_action(pcmk_resource_t rsc, const pcmk_node_t node,
	notify_data_t *n_data)
	{
	pcmk_action_t *notify = NULL;

	CRM_ASSERT(n_data != NULL);

	// Create the "post-" notify action for specified instance
	notify = new_notify_action(rsc, node, n_data->post, n_data->post_done,
	n_data);
	if (notify != NULL) {
	notify->priority = INFINITY;
	}

	// Order recurring monitors after all "post-" notifications complete
	if (n_data->post_done == NULL) {
	return;
	}
	for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t mon = (pcmk_action_t ) iter->data;
	const char *interval_ms_s = NULL;

	interval_ms_s = g_hash_table_lookup(mon->meta,
	XML_LRM_ATTR_INTERVAL_MS);
	if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)
	\|\| pcmk__str_eq(mon->task, PCMK_ACTION_CANCEL, pcmk__str_none)) {
	continue; // Not a recurring monitor
	}
	order_actions(n_data->post_done, mon, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Create and order notification pseudo-actions for a clone action
	*
	* In addition to the actual notify actions needed for each clone instance,
	* clone notifications also require pseudo-actions to provide ordering points
	* in the notification process. This creates the notification data, along with
	* appropriate pseudo-actions and their orderings.
	*
	* For example, the ordering sequence for starting a clone is:
	*
	* "pre-" notify pseudo-action for clone
	* -> "pre-" notify actions for each clone instance
	* -> "pre-" notifications complete pseudo-action for clone
	* -> start actions for each clone instance
	* -> "started" pseudo-action for clone
	* -> "post-" notify pseudo-action for clone
	* -> "post-" notify actions for each clone instance
	* -> "post-" notifications complete pseudo-action for clone
	*
	* \param[in,out] rsc Clone that notifications are for
	* \param[in] task Name of action that notifications are for
	* \param[in,out] action If not NULL, create a "pre-" pseudo-action ordered
	* before a "pre-" complete pseudo-action, ordered
	* before this action
	* \param[in,out] complete If not NULL, create a "post-" pseudo-action ordered
	* after this action, and a "post-" complete
	* pseudo-action ordered after that
	*
	* \return Newly created notification data
	*/
	notify_data_t *
	pe__action_notif_pseudo_ops(pcmk_resource_t rsc, const char task,
	pcmk_action_t action, pcmk_action_t complete)
	{
	notify_data_t *n_data = NULL;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_notify)) {
	return NULL;
	}

	n_data = calloc(1, sizeof(notify_data_t));
	CRM_ASSERT(n_data != NULL);

	n_data->action = task;

	if (action != NULL) { // Need "pre-" pseudo-actions

	// Create "pre-" notify pseudo-action for clone
	n_data->pre = new_notify_pseudo_action(rsc, action, PCMK_ACTION_NOTIFY,
	"pre");
	pe__set_action_flags(n_data->pre, pcmk_action_runnable);
	add_hash_param(n_data->pre->meta, "notify_type", "pre");
	add_hash_param(n_data->pre->meta, "notify_operation", n_data->action);

	// Create "pre-" notifications complete pseudo-action for clone
	n_data->pre_done = new_notify_pseudo_action(rsc, action,
	PCMK_ACTION_NOTIFIED,
	"confirmed-pre");
	pe__set_action_flags(n_data->pre_done, pcmk_action_runnable);
	add_hash_param(n_data->pre_done->meta, "notify_type", "pre");
	add_hash_param(n_data->pre_done->meta,
	"notify_operation", n_data->action);

	// Order "pre-" -> "pre-" complete -> original action
	order_actions(n_data->pre, n_data->pre_done, pcmk__ar_ordered);
	order_actions(n_data->pre_done, action, pcmk__ar_ordered);
	}

	if (complete != NULL) { // Need "post-" pseudo-actions

	// Create "post-" notify pseudo-action for clone
	n_data->post = new_notify_pseudo_action(rsc, complete,
	PCMK_ACTION_NOTIFY, "post");
	n_data->post->priority = INFINITY;
	if (pcmk_is_set(complete->flags, pcmk_action_runnable)) {
	pe__set_action_flags(n_data->post, pcmk_action_runnable);
	} else {
	pe__clear_action_flags(n_data->post, pcmk_action_runnable);
	}
	add_hash_param(n_data->post->meta, "notify_type", "post");
	add_hash_param(n_data->post->meta, "notify_operation", n_data->action);

	// Create "post-" notifications complete pseudo-action for clone
	n_data->post_done = new_notify_pseudo_action(rsc, complete,
	PCMK_ACTION_NOTIFIED,
	"confirmed-post");
	n_data->post_done->priority = INFINITY;
	if (pcmk_is_set(complete->flags, pcmk_action_runnable)) {
	pe__set_action_flags(n_data->post_done, pcmk_action_runnable);
	} else {
	pe__clear_action_flags(n_data->post_done, pcmk_action_runnable);
	}
	add_hash_param(n_data->post_done->meta, "notify_type", "post");
	add_hash_param(n_data->post_done->meta,
	"notify_operation", n_data->action);

	// Order original action complete -> "post-" -> "post-" complete
	order_actions(complete, n_data->post, pcmk__ar_first_implies_then);
	order_actions(n_data->post, n_data->post_done,
	pcmk__ar_first_implies_then);
	}

	// If we created both, order "pre-" complete -> "post-"
	if ((action != NULL) && (complete != NULL)) {
	order_actions(n_data->pre_done, n_data->post, pcmk__ar_ordered);
	}
	return n_data;
	}

	/*!
	* \internal
	* \brief Create a new notification entry
	*
	* \param[in] rsc Resource for notification
	* \param[in] node Node for notification
	*
	* \return Newly allocated notification entry
	* \note The caller is responsible for freeing the return value.
	*/
	static notify_entry_t *
	new_notify_entry(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	notify_entry_t *entry = calloc(1, sizeof(notify_entry_t));

	CRM_ASSERT(entry != NULL);
	entry->rsc = rsc;
	entry->node = node;
	return entry;
	}

	/*!
	* \internal
	* \brief Add notification data for resource state and optionally actions
	*
	* \param[in] rsc Clone or clone instance being notified
	* \param[in] activity Whether to add notification entries for actions
	* \param[in,out] n_data Notification data for clone
	*/
	static void
	collect_resource_data(const pcmk_resource_t *rsc, bool activity,
	notify_data_t *n_data)
	{
	const GList *iter = NULL;
	notify_entry_t *entry = NULL;
	const pcmk_node_t *node = NULL;

	if (n_data == NULL) {
	return;
	}

	if (n_data->allowed_nodes == NULL) {
	n_data->allowed_nodes = rsc->allowed_nodes;
	}

	// If this is a clone, call recursively for each instance
	if (rsc->children != NULL) {
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;

	collect_resource_data(child, activity, n_data);
	}
	return;
	}

	// This is a notification for a single clone instance

	if (rsc->running_on != NULL) {
	node = rsc->running_on->data; // First is sufficient
	}
	entry = new_notify_entry(rsc, node);

	// Add notification indicating the resource state
	switch (rsc->role) {
	case pcmk_role_stopped:
	n_data->inactive = g_list_prepend(n_data->inactive, entry);
	break;

	case pcmk_role_started:
	n_data->active = g_list_prepend(n_data->active, entry);
	break;

	case pcmk_role_unpromoted:
	n_data->unpromoted = g_list_prepend(n_data->unpromoted, entry);
	n_data->active = g_list_prepend(n_data->active,
	dup_notify_entry(entry));
	break;

	case pcmk_role_promoted:
	n_data->promoted = g_list_prepend(n_data->promoted, entry);
	n_data->active = g_list_prepend(n_data->active,
	dup_notify_entry(entry));
	break;

	default:
	- crm_err("Resource %s role on %s (%s) is not supported for "
	- "notifications (bug?)",
	- rsc->id, pe__node_name(node), role2text(rsc->role));
	+ pcmk__sched_err("Resource %s role on %s (%s) is not supported for "
	+ "notifications (bug?)",
	+ rsc->id, pe__node_name(node), role2text(rsc->role));
	free(entry);
	break;
	}

	if (!activity) {
	return;
	}

	// Add notification entries for each of the resource's actions
	for (iter = rsc->actions; iter != NULL; iter = iter->next) {
	const pcmk_action_t op = (const pcmk_action_t ) iter->data;

	if (!pcmk_is_set(op->flags, pcmk_action_optional)
	&& (op->node != NULL)) {
	enum action_tasks task = text2task(op->task);

	if ((task == pcmk_action_stop) && op->node->details->unclean) {
	// Create anyway (additional noise if node can't be fenced)
	} else if (!pcmk_is_set(op->flags, pcmk_action_runnable)) {
	continue;
	}

	entry = new_notify_entry(rsc, op->node);

	switch (task) {
	case pcmk_action_start:
	n_data->start = g_list_prepend(n_data->start, entry);
	break;
	case pcmk_action_stop:
	n_data->stop = g_list_prepend(n_data->stop, entry);
	break;
	case pcmk_action_promote:
	n_data->promote = g_list_prepend(n_data->promote, entry);
	break;
	case pcmk_action_demote:
	n_data->demote = g_list_prepend(n_data->demote, entry);
	break;
	default:
	free(entry);
	break;
	}
	}
	}
	}

	// For (char *) value
	#define add_notify_env(n_data, key, value) do { \
	n_data->keys = pcmk_prepend_nvpair(n_data->keys, key, value); \
	} while (0)

	// For (GString *) value
	#define add_notify_env_gs(n_data, key, value) do { \
	n_data->keys = pcmk_prepend_nvpair(n_data->keys, key, \
	(const char *) value->str); \
	} while (0)

	// For (GString *) value
	#define add_notify_env_free_gs(n_data, key, value) do { \
	n_data->keys = pcmk_prepend_nvpair(n_data->keys, key, \
	(const char *) value->str); \
	g_string_free(value, TRUE); value = NULL; \
	} while (0)

	/*!
	* \internal
	* \brief Create notification name/value pairs from structured data
	*
	* \param[in] rsc Resource that notification is for
	* \param[in,out] n_data Notification data
	*/
	static void
	add_notif_keys(const pcmk_resource_t rsc, notify_data_t n_data)
	{
	bool required = false; // Whether to make notify actions required
	GString *rsc_list = NULL;
	GString *node_list = NULL;
	GString *metal_list = NULL;
	const char *source = NULL;
	GList *nodes = NULL;

	n_data->stop = notify_entries_to_strings(n_data->stop,
	&rsc_list, &node_list);
	if ((strcmp(" ", (const char *) rsc_list->str) != 0)
	&& pcmk__str_eq(n_data->action, PCMK_ACTION_STOP, pcmk__str_none)) {
	required = true;
	}
	add_notify_env_free_gs(n_data, "notify_stop_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_stop_uname", node_list);

	if ((n_data->start != NULL)
	&& pcmk__str_eq(n_data->action, PCMK_ACTION_START, pcmk__str_none)) {
	required = true;
	}
	n_data->start = notify_entries_to_strings(n_data->start,
	&rsc_list, &node_list);
	add_notify_env_free_gs(n_data, "notify_start_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_start_uname", node_list);

	if ((n_data->demote != NULL)
	&& pcmk__str_eq(n_data->action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
	required = true;
	}
	n_data->demote = notify_entries_to_strings(n_data->demote,
	&rsc_list, &node_list);
	add_notify_env_free_gs(n_data, "notify_demote_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_demote_uname", node_list);

	if ((n_data->promote != NULL)
	&& pcmk__str_eq(n_data->action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	required = true;
	}
	n_data->promote = notify_entries_to_strings(n_data->promote,
	&rsc_list, &node_list);
	add_notify_env_free_gs(n_data, "notify_promote_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_promote_uname", node_list);

	n_data->active = notify_entries_to_strings(n_data->active,
	&rsc_list, &node_list);
	add_notify_env_free_gs(n_data, "notify_active_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_active_uname", node_list);

	n_data->unpromoted = notify_entries_to_strings(n_data->unpromoted,
	&rsc_list, &node_list);
	add_notify_env_gs(n_data, "notify_unpromoted_resource", rsc_list);
	add_notify_env_gs(n_data, "notify_unpromoted_uname", node_list);

	// Deprecated: kept for backward compatibility with older resource agents
	add_notify_env_free_gs(n_data, "notify_slave_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_slave_uname", node_list);

	n_data->promoted = notify_entries_to_strings(n_data->promoted,
	&rsc_list, &node_list);
	add_notify_env_gs(n_data, "notify_promoted_resource", rsc_list);
	add_notify_env_gs(n_data, "notify_promoted_uname", node_list);

	// Deprecated: kept for backward compatibility with older resource agents
	add_notify_env_free_gs(n_data, "notify_master_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_master_uname", node_list);

	n_data->inactive = notify_entries_to_strings(n_data->inactive,
	&rsc_list, NULL);
	add_notify_env_free_gs(n_data, "notify_inactive_resource", rsc_list);

	nodes = g_hash_table_get_values(n_data->allowed_nodes);
	if (!pcmk__is_daemon) {
	/* For display purposes, sort the node list, for consistent
	* regression test output (while avoiding the performance hit
	* for the live cluster).
	*/
	nodes = g_list_sort(nodes, pe__cmp_node_name);
	}
	get_node_names(nodes, &node_list, NULL);
	add_notify_env_free_gs(n_data, "notify_available_uname", node_list);
	g_list_free(nodes);

	source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET);
	if (pcmk__str_eq("host", source, pcmk__str_none)) {
	get_node_names(rsc->cluster->nodes, &node_list, &metal_list);
	add_notify_env_free_gs(n_data, "notify_all_hosts", metal_list);
	} else {
	get_node_names(rsc->cluster->nodes, &node_list, NULL);
	}
	add_notify_env_free_gs(n_data, "notify_all_uname", node_list);

	if (required && (n_data->pre != NULL)) {
	pe__clear_action_flags(n_data->pre, pcmk_action_optional);
	pe__clear_action_flags(n_data->pre_done, pcmk_action_optional);
	}

	if (required && (n_data->post != NULL)) {
	pe__clear_action_flags(n_data->post, pcmk_action_optional);
	pe__clear_action_flags(n_data->post_done, pcmk_action_optional);
	}
	}

	/*
	* \internal
	* \brief Find any remote connection start relevant to an action
	*
	* \param[in] action Action to check
	*
	* \return If action is behind a remote connection, connection's start
	*/
	static pcmk_action_t *
	find_remote_start(pcmk_action_t *action)
	{
	if ((action != NULL) && (action->node != NULL)) {
	pcmk_resource_t *remote_rsc = action->node->details->remote_rsc;

	if (remote_rsc != NULL) {
	return find_first_action(remote_rsc->actions, NULL,
	PCMK_ACTION_START,
	NULL);
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Create notify actions, and add notify data to original actions
	*
	* \param[in,out] rsc Clone or clone instance that notification is for
	* \param[in,out] n_data Clone notification data for some action
	*/
	static void
	create_notify_actions(pcmk_resource_t rsc, notify_data_t n_data)
	{
	GList *iter = NULL;
	pcmk_action_t *stop = NULL;
	pcmk_action_t *start = NULL;
	enum action_tasks task = text2task(n_data->action);

	// If this is a clone, call recursively for each instance
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, (GFunc) create_notify_actions, n_data);
	return;
	}

	// Add notification meta-attributes to original actions
	for (iter = rsc->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t op = (pcmk_action_t ) iter->data;

	if (!pcmk_is_set(op->flags, pcmk_action_optional)
	&& (op->node != NULL)) {
	switch (text2task(op->task)) {
	case pcmk_action_start:
	case pcmk_action_stop:
	case pcmk_action_promote:
	case pcmk_action_demote:
	add_notify_data_to_action_meta(n_data, op);
	break;
	default:
	break;
	}
	}
	}

	// Skip notify action itself if original action was not needed
	switch (task) {
	case pcmk_action_start:
	if (n_data->start == NULL) {
	pcmk__rsc_trace(rsc, "No notify action needed for %s %s",
	rsc->id, n_data->action);
	return;
	}
	break;

	case pcmk_action_promote:
	if (n_data->promote == NULL) {
	pcmk__rsc_trace(rsc, "No notify action needed for %s %s",
	rsc->id, n_data->action);
	return;
	}
	break;

	case pcmk_action_demote:
	if (n_data->demote == NULL) {
	pcmk__rsc_trace(rsc, "No notify action needed for %s %s",
	rsc->id, n_data->action);
	return;
	}
	break;

	default:
	// We cannot do same for stop because it might be implied by fencing
	break;
	}

	pcmk__rsc_trace(rsc, "Creating notify actions for %s %s",
	rsc->id, n_data->action);

	// Create notify actions for stop or demote
	if ((rsc->role != pcmk_role_stopped)
	&& ((task == pcmk_action_stop) \|\| (task == pcmk_action_demote))) {

	stop = find_first_action(rsc->actions, NULL, PCMK_ACTION_STOP, NULL);

	for (iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t current_node = (pcmk_node_t ) iter->data;

	/* If a stop is a pseudo-action implied by fencing, don't try to
	* notify the node getting fenced.
	*/
	if ((stop != NULL)
	&& pcmk_is_set(stop->flags, pcmk_action_pseudo)
	&& (current_node->details->unclean
	\|\| current_node->details->remote_requires_reset)) {
	continue;
	}

	new_notify_action(rsc, current_node, n_data->pre,
	n_data->pre_done, n_data);

	if ((task == pcmk_action_demote) \|\| (stop == NULL)
	\|\| pcmk_is_set(stop->flags, pcmk_action_optional)) {
	new_post_notify_action(rsc, current_node, n_data);
	}
	}
	}

	// Create notify actions for start or promote
	if ((rsc->next_role != pcmk_role_stopped)
	&& ((task == pcmk_action_start) \|\| (task == pcmk_action_promote))) {

	start = find_first_action(rsc->actions, NULL, PCMK_ACTION_START, NULL);
	if (start != NULL) {
	pcmk_action_t *remote_start = find_remote_start(start);

	if ((remote_start != NULL)
	&& !pcmk_is_set(remote_start->flags, pcmk_action_runnable)) {
	/* Start and promote actions for a clone instance behind
	* a Pacemaker Remote connection happen after the
	* connection starts. If the connection start is blocked, do
	* not schedule notifications for these actions.
	*/
	return;
	}
	}
	if (rsc->allocated_to == NULL) {
	pcmk__sched_err("Next role '%s' but %s is not allocated",
	role2text(rsc->next_role), rsc->id);
	return;
	}
	if ((task != pcmk_action_start) \|\| (start == NULL)
	\|\| pcmk_is_set(start->flags, pcmk_action_optional)) {

	new_notify_action(rsc, rsc->allocated_to, n_data->pre,
	n_data->pre_done, n_data);
	}
	new_post_notify_action(rsc, rsc->allocated_to, n_data);
	}
	}

	/*!
	* \internal
	* \brief Create notification data and actions for one clone action
	*
	* \param[in,out] rsc Clone resource that notification is for
	* \param[in,out] n_data Clone notification data for some action
	*/
	void
	pe__create_action_notifications(pcmk_resource_t rsc, notify_data_t n_data)
	{
	if ((rsc == NULL) \|\| (n_data == NULL)) {
	return;
	}
	collect_resource_data(rsc, true, n_data);
	add_notif_keys(rsc, n_data);
	create_notify_actions(rsc, n_data);
	}

	/*!
	* \internal
	* \brief Free notification data for one action
	*
	* \param[in,out] n_data Notification data to free
	*/
	void
	pe__free_action_notification_data(notify_data_t *n_data)
	{
	if (n_data == NULL) {
	return;
	}
	g_list_free_full(n_data->stop, free);
	g_list_free_full(n_data->start, free);
	g_list_free_full(n_data->demote, free);
	g_list_free_full(n_data->promote, free);
	g_list_free_full(n_data->promoted, free);
	g_list_free_full(n_data->unpromoted, free);
	g_list_free_full(n_data->active, free);
	g_list_free_full(n_data->inactive, free);
	pcmk_free_nvpairs(n_data->keys);
	free(n_data);
	}

	/*!
	* \internal
	* \brief Order clone "notifications complete" pseudo-action after fencing
	*
	* If a stop action is implied by fencing, the usual notification pseudo-actions
	* will not be sufficient to order things properly, or even create all needed
	* notifications if the clone is also stopping on another node, and another
	* clone is ordered after it. This function creates new notification
	* pseudo-actions relative to the fencing to ensure everything works properly.
	*
	* \param[in] stop Stop action implied by fencing
	* \param[in,out] rsc Clone resource that notification is for
	* \param[in,out] stonith_op Fencing action that implies \p stop
	*/
	void
	pe__order_notifs_after_fencing(const pcmk_action_t stop, pcmk_resource_t rsc,
	pcmk_action_t *stonith_op)
	{
	notify_data_t *n_data;

	crm_info("Ordering notifications for implied %s after fencing", stop->uuid);
	n_data = pe__action_notif_pseudo_ops(rsc, PCMK_ACTION_STOP, NULL,
	stonith_op);

	if (n_data != NULL) {
	collect_resource_data(rsc, false, n_data);
	add_notify_env(n_data, "notify_stop_resource", rsc->id);
	add_notify_env(n_data, "notify_stop_uname", stop->node->details->uname);
	create_notify_actions(uber_parent(rsc), n_data);
	pe__free_action_notification_data(n_data);
	}
	}
	diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c
	index 754db9f57e..4e16020a07 100644
	--- a/lib/pengine/rules.c
	+++ b/lib/pengine/rules.c
	@@ -1,1337 +1,1338 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <glib.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/rules_internal.h>
	#include <crm/pengine/internal.h>

	#include <sys/types.h>
	#include <regex.h>
	#include <ctype.h>

	CRM_TRACE_INIT_DATA(pe_rules);

	/*!
	* \brief Evaluate any rules contained by given XML element
	*
	* \param[in,out] xml XML element to check for rules
	* \param[in] node_hash Node attributes to use to evaluate expressions
	* \param[in] now Time to use when evaluating expressions
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if no rules, or any of rules present is in effect, else FALSE
	*/
	gboolean
	pe_evaluate_rules(xmlNode ruleset, GHashTable node_hash, crm_time_t *now,
	crm_time_t *next_change)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe_eval_rules(ruleset, &rule_data, next_change);
	}

	gboolean
	pe_test_rule(xmlNode rule, GHashTable node_hash, enum rsc_role_e role,
	crm_time_t now, crm_time_t next_change,
	pe_match_data_t *match_data)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = role,
	.now = now,
	.match_data = match_data,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe_eval_expr(rule, &rule_data, next_change);
	}

	/*!
	* \brief Evaluate one rule subelement (pass/fail)
	*
	* A rule element may contain another rule, a node attribute expression, or a
	* date expression. Given any one of those, evaluate it and return whether it
	* passed.
	*
	* \param[in,out] expr Rule subelement XML
	* \param[in] node_hash Node attributes to use when evaluating expression
	* \param[in] role Resource role to use when evaluating expression
	* \param[in] now Time to use when evaluating expression
	* \param[out] next_change If not NULL, set to when evaluation will change
	* \param[in] match_data If not NULL, resource back-references and params
	*
	* \return TRUE if expression is in effect under given conditions, else FALSE
	*/
	gboolean
	pe_test_expression(xmlNode expr, GHashTable node_hash, enum rsc_role_e role,
	crm_time_t now, crm_time_t next_change,
	pe_match_data_t *match_data)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = role,
	.now = now,
	.match_data = match_data,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe_eval_subexpr(expr, &rule_data, next_change);
	}

	enum expression_type
	find_expression_type(xmlNode * expr)
	{
	const char *attr = NULL;

	attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE);

	if (pcmk__xe_is(expr, PCMK_XE_DATE_EXPRESSION)) {
	return pcmk__subexpr_datetime;

	} else if (pcmk__xe_is(expr, PCMK_XE_RSC_EXPRESSION)) {
	return pcmk__subexpr_resource;

	} else if (pcmk__xe_is(expr, PCMK_XE_OP_EXPRESSION)) {
	return pcmk__subexpr_operation;

	} else if (pcmk__xe_is(expr, XML_TAG_RULE)) {
	return pcmk__subexpr_rule;

	} else if (!pcmk__xe_is(expr, XML_TAG_EXPRESSION)) {
	return pcmk__subexpr_unknown;

	} else if (pcmk__str_any_of(attr, CRM_ATTR_UNAME, CRM_ATTR_KIND, CRM_ATTR_ID, NULL)) {
	return pcmk__subexpr_location;

	} else if (pcmk__str_eq(attr, CRM_ATTR_ROLE, pcmk__str_none)) {
	return pcmk__subexpr_role;
	}

	return pcmk__subexpr_attribute;
	}

	/* As per the nethack rules:
	*
	* moon period = 29.53058 days ~= 30, year = 365.2422 days
	* days moon phase advances on first day of year compared to preceding year
	* = 365.2422 - 12*29.53058 ~= 11
	* years in Metonic cycle (time until same phases fall on the same days of
	* the month) = 18.6 ~= 19
	* moon phase on first day of year (epact) ~= (11*(year%19) + 29) % 30
	* (29 as initial condition)
	* current phase in days = first day phase + days elapsed in year
	* 6 moons ~= 177 days
	* 177 ~= 8 reported phases * 22
	* + 11/22 for rounding
	*
	* 0-7, with 0: new, 4: full
	*/

	static int
	phase_of_the_moon(const crm_time_t *now)
	{
	uint32_t epact, diy, goldn;
	uint32_t y;

	crm_time_get_ordinal(now, &y, &diy);

	goldn = (y % 19) + 1;
	epact = (11 * goldn + 18) % 30;
	if ((epact == 25 && goldn > 11) \|\| epact == 24)
	epact++;

	return ((((((diy + epact) * 6) + 11) % 177) / 22) & 7);
	}

	static int
	check_one(const xmlNode cron_spec, const char xml_field, uint32_t time_field)
	{
	int rc = pcmk_rc_undetermined;
	const char *value = crm_element_value(cron_spec, xml_field);
	long long low, high;

	if (value == NULL) {
	/* Return pe_date_result_undetermined if the field is missing. */
	goto bail;
	}

	if (pcmk__parse_ll_range(value, &low, &high) != pcmk_rc_ok) {
	goto bail;
	} else if (low == high) {
	/* A single number was given, not a range. */
	if (time_field < low) {
	rc = pcmk_rc_before_range;
	} else if (time_field > high) {
	rc = pcmk_rc_after_range;
	} else {
	rc = pcmk_rc_within_range;
	}
	} else if (low != -1 && high != -1) {
	/* This is a range with both bounds. */
	if (time_field < low) {
	rc = pcmk_rc_before_range;
	} else if (time_field > high) {
	rc = pcmk_rc_after_range;
	} else {
	rc = pcmk_rc_within_range;
	}
	} else if (low == -1) {
	/* This is a range with no starting value. */
	rc = time_field <= high ? pcmk_rc_within_range : pcmk_rc_after_range;
	} else if (high == -1) {
	/* This is a range with no ending value. */
	rc = time_field >= low ? pcmk_rc_within_range : pcmk_rc_before_range;
	}

	bail:
	if (rc == pcmk_rc_within_range) {
	crm_debug("Condition '%s' in %s: passed", value, xml_field);
	} else {
	crm_debug("Condition '%s' in %s: failed", value, xml_field);
	}

	return rc;
	}

	static gboolean
	check_passes(int rc) {
	/* _within_range is obvious. _undetermined is a pass because
	* this is the return value if a field is not given. In this
	* case, we just want to ignore it and check other fields to
	* see if they place some restriction on what can pass.
	*/
	return rc == pcmk_rc_within_range \|\| rc == pcmk_rc_undetermined;
	}

	#define CHECK_ONE(spec, name, var) do { \
	int subpart_rc = check_one(spec, name, var); \
	if (check_passes(subpart_rc) == FALSE) { \
	return subpart_rc; \
	} \
	} while (0)

	int
	pe_cron_range_satisfied(const crm_time_t now, const xmlNode cron_spec)
	{
	uint32_t h, m, s, y, d, w;

	CRM_CHECK(now != NULL, return pcmk_rc_op_unsatisfied);

	crm_time_get_gregorian(now, &y, &m, &d);
	CHECK_ONE(cron_spec, "years", y);
	CHECK_ONE(cron_spec, "months", m);
	CHECK_ONE(cron_spec, "monthdays", d);

	crm_time_get_timeofday(now, &h, &m, &s);
	CHECK_ONE(cron_spec, "hours", h);
	CHECK_ONE(cron_spec, "minutes", m);
	CHECK_ONE(cron_spec, "seconds", s);

	crm_time_get_ordinal(now, &y, &d);
	CHECK_ONE(cron_spec, "yeardays", d);

	crm_time_get_isoweek(now, &y, &w, &d);
	CHECK_ONE(cron_spec, "weekyears", y);
	CHECK_ONE(cron_spec, "weeks", w);
	CHECK_ONE(cron_spec, "weekdays", d);

	CHECK_ONE(cron_spec, "moon", phase_of_the_moon(now));
	if (crm_element_value(cron_spec, "moon") != NULL) {
	pcmk__config_warn("Support for 'moon' in date_spec elements "
	"(such as %s) is deprecated and will be removed "
	"in a future release of Pacemaker", ID(cron_spec));
	}

	/* If we get here, either no fields were specified (which is success), or all
	* the fields that were specified had their conditions met (which is also a
	* success). Thus, the result is success.
	*/
	return pcmk_rc_ok;
	}

	static void
	update_field(crm_time_t t, const xmlNode xml, const char *attr,
	void (time_fn)(crm_time_t , int))
	{
	long long value;

	if ((pcmk__scan_ll(crm_element_value(xml, attr), &value, 0LL) == pcmk_rc_ok)
	&& (value != 0LL) && (value >= INT_MIN) && (value <= INT_MAX)) {
	time_fn(t, (int) value);
	}
	}

	static crm_time_t *
	parse_xml_duration(const crm_time_t start, const xmlNode duration_spec)
	{
	crm_time_t *end = pcmk_copy_time(start);

	update_field(end, duration_spec, "years", crm_time_add_years);
	update_field(end, duration_spec, "months", crm_time_add_months);
	update_field(end, duration_spec, "weeks", crm_time_add_weeks);
	update_field(end, duration_spec, "days", crm_time_add_days);
	update_field(end, duration_spec, "hours", crm_time_add_hours);
	update_field(end, duration_spec, "minutes", crm_time_add_minutes);
	update_field(end, duration_spec, "seconds", crm_time_add_seconds);

	return end;
	}

	// Set next_change to t if t is earlier
	static void
	crm_time_set_if_earlier(crm_time_t next_change, crm_time_t t)
	{
	if ((next_change != NULL) && (t != NULL)) {
	if (!crm_time_is_defined(next_change)
	\|\| (crm_time_compare(t, next_change) < 0)) {
	crm_time_set(next_change, t);
	}
	}
	}

	// Information about a block of nvpair elements
	typedef struct sorted_set_s {
	int score; // This block's score for sorting
	const char *name; // This block's ID
	const char *special_name; // ID that should sort first
	xmlNode *attr_set; // This block
	gboolean overwrite; // Whether existing values will be overwritten
	} sorted_set_t;

	static gint
	sort_pairs(gconstpointer a, gconstpointer b)
	{
	const sorted_set_t *pair_a = a;
	const sorted_set_t *pair_b = b;

	if (a == NULL && b == NULL) {
	return 0;
	} else if (a == NULL) {
	return 1;
	} else if (b == NULL) {
	return -1;
	}

	if (pcmk__str_eq(pair_a->name, pair_a->special_name, pcmk__str_casei)) {
	return -1;

	} else if (pcmk__str_eq(pair_b->name, pair_a->special_name, pcmk__str_casei)) {
	return 1;
	}

	/* If we're overwriting values, we want lowest score first, so the highest
	* score is processed last; if we're not overwriting values, we want highest
	* score first, so nothing else overwrites it.
	*/
	if (pair_a->score < pair_b->score) {
	return pair_a->overwrite? -1 : 1;
	} else if (pair_a->score > pair_b->score) {
	return pair_a->overwrite? 1 : -1;
	}
	return 0;
	}

	static void
	populate_hash(xmlNode * nvpair_list, GHashTable * hash, gboolean overwrite, xmlNode * top)
	{
	const char *name = NULL;
	const char *value = NULL;
	const char *old_value = NULL;
	xmlNode *list = nvpair_list;
	xmlNode *an_attr = NULL;

	if (pcmk__xe_is(list->children, XML_TAG_ATTRS)) {
	list = list->children;
	}

	for (an_attr = pcmk__xe_first_child(list); an_attr != NULL;
	an_attr = pcmk__xe_next(an_attr)) {

	if (pcmk__str_eq((const char *)an_attr->name, XML_CIB_TAG_NVPAIR, pcmk__str_none)) {
	xmlNode *ref_nvpair = expand_idref(an_attr, top);

	name = crm_element_value(an_attr, XML_NVPAIR_ATTR_NAME);
	if (name == NULL) {
	name = crm_element_value(ref_nvpair, XML_NVPAIR_ATTR_NAME);
	}

	value = crm_element_value(an_attr, XML_NVPAIR_ATTR_VALUE);
	if (value == NULL) {
	value = crm_element_value(ref_nvpair, XML_NVPAIR_ATTR_VALUE);
	}

	if (name == NULL \|\| value == NULL) {
	continue;
	}

	old_value = g_hash_table_lookup(hash, name);

	if (pcmk__str_eq(value, "#default", pcmk__str_casei)) {
	if (old_value) {
	crm_trace("Letting %s default (removing explicit value \"%s\")",
	name, value);
	g_hash_table_remove(hash, name);
	}
	continue;

	} else if (old_value == NULL) {
	crm_trace("Setting %s=\"%s\"", name, value);
	g_hash_table_insert(hash, strdup(name), strdup(value));

	} else if (overwrite) {
	crm_trace("Setting %s=\"%s\" (overwriting old value \"%s\")",
	name, value, old_value);
	g_hash_table_replace(hash, strdup(name), strdup(value));
	}
	}
	}
	}

	typedef struct unpack_data_s {
	gboolean overwrite;
	void *hash;
	crm_time_t *next_change;
	const pe_rule_eval_data_t *rule_data;
	xmlNode *top;
	} unpack_data_t;

	static void
	unpack_attr_set(gpointer data, gpointer user_data)
	{
	sorted_set_t *pair = data;
	unpack_data_t *unpack_data = user_data;

	if (!pe_eval_rules(pair->attr_set, unpack_data->rule_data,
	unpack_data->next_change)) {
	return;
	}

	crm_trace("Adding attributes from %s (score %d) %s overwrite",
	pair->name, pair->score,
	(unpack_data->overwrite? "with" : "without"));
	populate_hash(pair->attr_set, unpack_data->hash, unpack_data->overwrite, unpack_data->top);
	}

	/*!
	* \internal
	* \brief Create a sorted list of nvpair blocks
	*
	* \param[in,out] top XML document root (used to expand id-ref's)
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name If not NULL, only get blocks of this element
	* \param[in] always_first If not NULL, sort block with this ID as first
	*
	* \return List of sorted_set_t entries for nvpair blocks
	*/
	static GList *
	make_pairs(xmlNode top, const xmlNode xml_obj, const char *set_name,
	const char *always_first, gboolean overwrite)
	{
	GList *unsorted = NULL;

	if (xml_obj == NULL) {
	return NULL;
	}
	for (xmlNode *attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (pcmk__str_eq(set_name, (const char *) attr_set->name,
	pcmk__str_null_matches)) {
	const char *score = NULL;
	sorted_set_t *pair = NULL;
	xmlNode *expanded_attr_set = expand_idref(attr_set, top);

	if (expanded_attr_set == NULL) {
	// Schema (if not "none") prevents this
	continue;
	}

	pair = calloc(1, sizeof(sorted_set_t));
	pair->name = ID(expanded_attr_set);
	pair->special_name = always_first;
	pair->attr_set = expanded_attr_set;
	pair->overwrite = overwrite;

	score = crm_element_value(expanded_attr_set, XML_RULE_ATTR_SCORE);
	pair->score = char2score(score);

	unsorted = g_list_prepend(unsorted, pair);
	}
	}
	return g_list_sort(unsorted, sort_pairs);
	}

	/*!
	* \brief Extract nvpair blocks contained by an XML element into a hash table
	*
	* \param[in,out] top XML document root (used to expand id-ref's)
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name If not NULL, only use blocks of this element
	* \param[in] rule_data Matching parameters to use when unpacking
	* \param[out] hash Where to store extracted name/value pairs
	* \param[in] always_first If not NULL, process block with this ID first
	* \param[in] overwrite Whether to replace existing values with same name
	* \param[out] next_change If not NULL, set to when evaluation will change
	*/
	void
	pe_eval_nvpairs(xmlNode top, const xmlNode xml_obj, const char *set_name,
	const pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	crm_time_t *next_change)
	{
	GList *pairs = make_pairs(top, xml_obj, set_name, always_first, overwrite);

	if (pairs) {
	unpack_data_t data = {
	.hash = hash,
	.overwrite = overwrite,
	.next_change = next_change,
	.top = top,
	.rule_data = rule_data
	};

	g_list_foreach(pairs, unpack_attr_set, &data);
	g_list_free_full(pairs, free);
	}
	}

	/*!
	* \brief Extract nvpair blocks contained by an XML element into a hash table
	*
	* \param[in,out] top XML document root (used to expand id-ref's)
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name Element name to identify nvpair blocks
	* \param[in] node_hash Node attributes to use when evaluating rules
	* \param[out] hash Where to store extracted name/value pairs
	* \param[in] always_first If not NULL, process block with this ID first
	* \param[in] overwrite Whether to replace existing values with same name
	* \param[in] now Time to use when evaluating rules
	* \param[out] next_change If not NULL, set to when evaluation will change
	*/
	void
	pe_unpack_nvpairs(xmlNode top, const xmlNode xml_obj, const char *set_name,
	GHashTable node_hash, GHashTable hash,
	const char *always_first, gboolean overwrite,
	crm_time_t now, crm_time_t next_change)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe_eval_nvpairs(top, xml_obj, set_name, &rule_data, hash,
	always_first, overwrite, next_change);
	}

	/*!
	* \brief Expand any regular expression submatches (%0-%9) in a string
	*
	* \param[in] string String possibly containing submatch variables
	* \param[in] match_data If not NULL, regular expression matches
	*
	* \return Newly allocated string identical to \p string with submatches
	* expanded, or NULL if there were no matches
	*/
	char *
	pe_expand_re_matches(const char string, const pe_re_match_data_t match_data)
	{
	size_t len = 0;
	int i;
	const char p, last_match_index;
	char p_dst, result = NULL;

	if (pcmk__str_empty(string) \|\| !match_data) {
	return NULL;
	}

	p = last_match_index = string;

	while (*p) {
	if (p == '%' && (p + 1) && isdigit(*(p + 1))) {
	i = *(p + 1) - '0';
	if (match_data->nregs >= i && match_data->pmatch[i].rm_so != -1 &&
	match_data->pmatch[i].rm_eo > match_data->pmatch[i].rm_so) {
	len += p - last_match_index + (match_data->pmatch[i].rm_eo - match_data->pmatch[i].rm_so);
	last_match_index = p + 2;
	}
	p++;
	}
	p++;
	}
	len += p - last_match_index + 1;

	/* FIXME: Excessive? */
	if (len - 1 <= 0) {
	return NULL;
	}

	p_dst = result = calloc(1, len);
	p = string;

	while (*p) {
	if (p == '%' && (p + 1) && isdigit(*(p + 1))) {
	i = *(p + 1) - '0';
	if (match_data->nregs >= i && match_data->pmatch[i].rm_so != -1 &&
	match_data->pmatch[i].rm_eo > match_data->pmatch[i].rm_so) {
	/* rm_eo can be equal to rm_so, but then there is nothing to do */
	int match_len = match_data->pmatch[i].rm_eo - match_data->pmatch[i].rm_so;
	memcpy(p_dst, match_data->string + match_data->pmatch[i].rm_so, match_len);
	p_dst += match_len;
	}
	p++;
	} else {
	(p_dst) = (p);
	p_dst++;
	}
	p++;
	}

	return result;
	}

	/*!
	* \brief Evaluate rules
	*
	* \param[in,out] ruleset XML possibly containing rule sub-elements
	* \param[in] rule_data
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if there are no rules or
	*/
	gboolean
	pe_eval_rules(xmlNode ruleset, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	// If there are no rules, pass by default
	gboolean ruleset_default = TRUE;

	for (xmlNode *rule = first_named_child(ruleset, XML_TAG_RULE);
	rule != NULL; rule = crm_next_same_xml(rule)) {

	ruleset_default = FALSE;
	if (pe_eval_expr(rule, rule_data, next_change)) {
	/* Only the deprecated "lifetime" element of location constraints
	* may contain more than one rule at the top level -- the schema
	* limits a block of nvpairs to a single top-level rule. So, this
	* effectively means that a lifetime is active if any rule it
	* contains is active.
	*/
	return TRUE;
	}
	}

	return ruleset_default;
	}

	/*!
	* \brief Evaluate all of a rule's expressions
	*
	* \param[in,out] rule XML containing a rule definition or its id-ref
	* \param[in] rule_data Matching parameters to check against rule
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if \p rule_data passes \p rule, otherwise FALSE
	*/
	gboolean
	pe_eval_expr(xmlNode rule, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	xmlNode *expr = NULL;
	gboolean test = TRUE;
	gboolean empty = TRUE;
	gboolean passed = TRUE;
	gboolean do_and = TRUE;
	const char *value = NULL;

	rule = expand_idref(rule, NULL);
	value = crm_element_value(rule, XML_RULE_ATTR_BOOLEAN_OP);
	if (pcmk__str_eq(value, "or", pcmk__str_casei)) {
	do_and = FALSE;
	passed = FALSE;
	}

	crm_trace("Testing rule %s", ID(rule));
	for (expr = pcmk__xe_first_child(rule); expr != NULL;
	expr = pcmk__xe_next(expr)) {

	test = pe_eval_subexpr(expr, rule_data, next_change);
	empty = FALSE;

	if (test && do_and == FALSE) {
	crm_trace("Expression %s/%s passed", ID(rule), ID(expr));
	return TRUE;

	} else if (test == FALSE && do_and) {
	crm_trace("Expression %s/%s failed", ID(rule), ID(expr));
	return FALSE;
	}
	}

	if (empty) {
	- crm_err("Invalid Rule %s: rules must contain at least one expression", ID(rule));
	+ pcmk__config_err("Ignoring rule %s because it contains no expressions",
	+ ID(rule));
	}

	crm_trace("Rule %s %s", ID(rule), passed ? "passed" : "failed");
	return passed;
	}

	/*!
	* \brief Evaluate a single rule expression, including any subexpressions
	*
	* \param[in,out] expr XML containing a rule expression
	* \param[in] rule_data Matching parameters to check against expression
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if \p rule_data passes \p expr, otherwise FALSE
	*/
	gboolean
	pe_eval_subexpr(xmlNode expr, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	gboolean accept = FALSE;
	const char *uname = NULL;

	switch (find_expression_type(expr)) {
	case pcmk__subexpr_rule:
	accept = pe_eval_expr(expr, rule_data, next_change);
	break;
	case pcmk__subexpr_attribute:
	case pcmk__subexpr_location:
	/* these expressions can never succeed if there is
	* no node to compare with
	*/
	if (rule_data->node_hash != NULL) {
	accept = pe__eval_attr_expr(expr, rule_data);
	}
	break;

	case pcmk__subexpr_datetime:
	switch (pe__eval_date_expr(expr, rule_data, next_change)) {
	case pcmk_rc_within_range:
	case pcmk_rc_ok:
	accept = TRUE;
	break;

	default:
	accept = FALSE;
	break;
	}
	break;

	case pcmk__subexpr_role:
	accept = pe__eval_role_expr(expr, rule_data);
	break;

	case pcmk__subexpr_resource:
	accept = pe__eval_rsc_expr(expr, rule_data);
	break;

	case pcmk__subexpr_operation:
	accept = pe__eval_op_expr(expr, rule_data);
	break;

	default:
	CRM_CHECK(FALSE /* bad type */ , return FALSE);
	accept = FALSE;
	}
	if (rule_data->node_hash) {
	uname = g_hash_table_lookup(rule_data->node_hash, CRM_ATTR_UNAME);
	}

	crm_trace("Expression %s %s on %s",
	ID(expr), accept ? "passed" : "failed", uname ? uname : "all nodes");
	return accept;
	}

	/*!
	* \internal
	* \brief Compare two values in a rule's node attribute expression
	*
	* \param[in] l_val Value on left-hand side of comparison
	* \param[in] r_val Value on right-hand side of comparison
	* \param[in] type How to interpret the values (allowed values:
	* \c "string", \c "integer", \c "number",
	* \c "version", \c NULL)
	* \param[in] op Type of comparison
	*
	* \return -1 if <tt>(l_val < r_val)</tt>,
	* 0 if <tt>(l_val == r_val)</tt>,
	* 1 if <tt>(l_val > r_val)</tt>
	*/
	static int
	compare_attr_expr_vals(const char l_val, const char r_val, const char *type,
	const char *op)
	{
	int cmp = 0;

	if (l_val != NULL && r_val != NULL) {
	if (type == NULL) {
	if (pcmk__strcase_any_of(op, "lt", "lte", "gt", "gte", NULL)) {
	if (pcmk__char_in_any_str('.', l_val, r_val, NULL)) {
	type = "number";
	} else {
	type = "integer";
	}

	} else {
	type = "string";
	}
	crm_trace("Defaulting to %s based comparison for '%s' op", type, op);
	}

	if (pcmk__str_eq(type, "string", pcmk__str_casei)) {
	cmp = strcasecmp(l_val, r_val);

	} else if (pcmk__str_eq(type, "integer", pcmk__str_casei)) {
	long long l_val_num;
	int rc1 = pcmk__scan_ll(l_val, &l_val_num, 0LL);

	long long r_val_num;
	int rc2 = pcmk__scan_ll(r_val, &r_val_num, 0LL);

	if ((rc1 == pcmk_rc_ok) && (rc2 == pcmk_rc_ok)) {
	if (l_val_num < r_val_num) {
	cmp = -1;
	} else if (l_val_num > r_val_num) {
	cmp = 1;
	} else {
	cmp = 0;
	}

	} else {
	crm_debug("Integer parse error. Comparing %s and %s as strings",
	l_val, r_val);
	cmp = compare_attr_expr_vals(l_val, r_val, "string", op);
	}

	} else if (pcmk__str_eq(type, "number", pcmk__str_casei)) {
	double l_val_num;
	double r_val_num;

	int rc1 = pcmk__scan_double(l_val, &l_val_num, NULL, NULL);
	int rc2 = pcmk__scan_double(r_val, &r_val_num, NULL, NULL);

	if (rc1 == pcmk_rc_ok && rc2 == pcmk_rc_ok) {
	if (l_val_num < r_val_num) {
	cmp = -1;
	} else if (l_val_num > r_val_num) {
	cmp = 1;
	} else {
	cmp = 0;
	}

	} else {
	crm_debug("Floating-point parse error. Comparing %s and %s as "
	"strings", l_val, r_val);
	cmp = compare_attr_expr_vals(l_val, r_val, "string", op);
	}

	} else if (pcmk__str_eq(type, "version", pcmk__str_casei)) {
	cmp = compare_version(l_val, r_val);

	}

	} else if (l_val == NULL && r_val == NULL) {
	cmp = 0;
	} else if (r_val == NULL) {
	cmp = 1;
	} else { // l_val == NULL && r_val != NULL
	cmp = -1;
	}

	return cmp;
	}

	/*!
	* \internal
	* \brief Check whether an attribute expression evaluates to \c true
	*
	* \param[in] l_val Value on left-hand side of comparison
	* \param[in] r_val Value on right-hand side of comparison
	* \param[in] type How to interpret the values (allowed values:
	* \c "string", \c "integer", \c "number",
	* \c "version", \c NULL)
	* \param[in] op Type of comparison.
	*
	* \return \c true if expression evaluates to \c true, \c false
	* otherwise
	*/
	static bool
	accept_attr_expr(const char l_val, const char r_val, const char *type,
	const char *op)
	{
	int cmp;

	if (pcmk__str_eq(op, "defined", pcmk__str_casei)) {
	return (l_val != NULL);

	} else if (pcmk__str_eq(op, "not_defined", pcmk__str_casei)) {
	return (l_val == NULL);

	}

	cmp = compare_attr_expr_vals(l_val, r_val, type, op);

	if (pcmk__str_eq(op, "eq", pcmk__str_casei)) {
	return (cmp == 0);

	} else if (pcmk__str_eq(op, "ne", pcmk__str_casei)) {
	return (cmp != 0);

	} else if (l_val == NULL \|\| r_val == NULL) {
	// The comparison is meaningless from this point on
	return false;

	} else if (pcmk__str_eq(op, "lt", pcmk__str_casei)) {
	return (cmp < 0);

	} else if (pcmk__str_eq(op, "lte", pcmk__str_casei)) {
	return (cmp <= 0);

	} else if (pcmk__str_eq(op, "gt", pcmk__str_casei)) {
	return (cmp > 0);

	} else if (pcmk__str_eq(op, "gte", pcmk__str_casei)) {
	return (cmp >= 0);
	}

	return false; // Should never reach this point
	}

	/*!
	* \internal
	* \brief Get correct value according to value-source
	*
	* \param[in] value value given in rule expression
	* \param[in] value_source value-source given in rule expressions
	* \param[in] match_data If not NULL, resource back-references and params
	*/
	static const char *
	expand_value_source(const char value, const char value_source,
	const pe_match_data_t *match_data)
	{
	GHashTable *table = NULL;

	if (pcmk__str_empty(value)) {
	return NULL; // value_source is irrelevant

	} else if (pcmk__str_eq(value_source, "param", pcmk__str_casei)) {
	table = match_data->params;

	} else if (pcmk__str_eq(value_source, "meta", pcmk__str_casei)) {
	table = match_data->meta;

	} else { // literal
	return value;
	}

	if (table == NULL) {
	return NULL;
	}
	return (const char *) g_hash_table_lookup(table, value);
	}

	/*!
	* \internal
	* \brief Evaluate a node attribute expression based on #uname, #id, #kind,
	* or a generic node attribute
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data The match_data and node_hash members are used
	*
	* \return TRUE if rule_data satisfies the expression, FALSE otherwise
	*/
	gboolean
	pe__eval_attr_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	gboolean attr_allocated = FALSE;
	const char *h_val = NULL;

	const char *op = NULL;
	const char *type = NULL;
	const char *attr = NULL;
	const char *value = NULL;
	const char *value_source = NULL;

	attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE);
	op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION);
	value = crm_element_value(expr, XML_EXPR_ATTR_VALUE);
	type = crm_element_value(expr, XML_EXPR_ATTR_TYPE);
	value_source = crm_element_value(expr, XML_EXPR_ATTR_VALUE_SOURCE);

	if (attr == NULL) {
	pcmk__config_err("Expression %s invalid: " XML_EXPR_ATTR_ATTRIBUTE
	" not specified", pcmk__s(ID(expr), "without ID"));
	return FALSE;
	} else if (op == NULL) {
	pcmk__config_err("Expression %s invalid: " XML_EXPR_ATTR_OPERATION
	" not specified", pcmk__s(ID(expr), "without ID"));
	}

	if (rule_data->match_data != NULL) {
	// Expand any regular expression submatches (%0-%9) in attribute name
	if (rule_data->match_data->re != NULL) {
	char *resolved_attr = pe_expand_re_matches(attr, rule_data->match_data->re);

	if (resolved_attr != NULL) {
	attr = (const char *) resolved_attr;
	attr_allocated = TRUE;
	}
	}

	// Get value appropriate to value-source
	value = expand_value_source(value, value_source, rule_data->match_data);
	}

	if (rule_data->node_hash != NULL) {
	h_val = (const char *)g_hash_table_lookup(rule_data->node_hash, attr);
	}

	if (attr_allocated) {
	free((char *)attr);
	attr = NULL;
	}

	return accept_attr_expr(h_val, value, type, op);
	}

	/*!
	* \internal
	* \brief Evaluate a date_expression
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data Only the now member is used
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return Standard Pacemaker return code
	*/
	int
	pe__eval_date_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	crm_time_t *start = NULL;
	crm_time_t *end = NULL;
	const char *value = NULL;
	const char *op = crm_element_value(expr, "operation");

	xmlNode *duration_spec = NULL;
	xmlNode *date_spec = NULL;

	// "undetermined" will also be returned for parsing errors
	int rc = pcmk_rc_undetermined;

	crm_trace("Testing expression: %s", ID(expr));

	duration_spec = first_named_child(expr, "duration");
	date_spec = first_named_child(expr, "date_spec");

	value = crm_element_value(expr, "start");
	if (value != NULL) {
	start = crm_time_new(value);
	}
	value = crm_element_value(expr, "end");
	if (value != NULL) {
	end = crm_time_new(value);
	}

	if (start != NULL && end == NULL && duration_spec != NULL) {
	end = parse_xml_duration(start, duration_spec);
	}

	if (pcmk__str_eq(op, "in_range", pcmk__str_null_matches \| pcmk__str_casei)) {
	if ((start == NULL) && (end == NULL)) {
	// in_range requires at least one of start or end
	} else if ((start != NULL) && (crm_time_compare(rule_data->now, start) < 0)) {
	rc = pcmk_rc_before_range;
	crm_time_set_if_earlier(next_change, start);
	} else if ((end != NULL) && (crm_time_compare(rule_data->now, end) > 0)) {
	rc = pcmk_rc_after_range;
	} else {
	rc = pcmk_rc_within_range;
	if (end && next_change) {
	// Evaluation doesn't change until second after end
	crm_time_add_seconds(end, 1);
	crm_time_set_if_earlier(next_change, end);
	}
	}

	} else if (pcmk__str_eq(op, "date_spec", pcmk__str_casei)) {
	rc = pe_cron_range_satisfied(rule_data->now, date_spec);
	// @TODO set next_change appropriately

	} else if (pcmk__str_eq(op, "gt", pcmk__str_casei)) {
	if (start == NULL) {
	// gt requires start
	} else if (crm_time_compare(rule_data->now, start) > 0) {
	rc = pcmk_rc_within_range;
	} else {
	rc = pcmk_rc_before_range;

	// Evaluation doesn't change until second after start
	crm_time_add_seconds(start, 1);
	crm_time_set_if_earlier(next_change, start);
	}

	} else if (pcmk__str_eq(op, "lt", pcmk__str_casei)) {
	if (end == NULL) {
	// lt requires end
	} else if (crm_time_compare(rule_data->now, end) < 0) {
	rc = pcmk_rc_within_range;
	crm_time_set_if_earlier(next_change, end);
	} else {
	rc = pcmk_rc_after_range;
	}
	}

	crm_time_free(start);
	crm_time_free(end);
	return rc;
	}

	gboolean
	pe__eval_op_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	const char *name = crm_element_value(expr, XML_NVPAIR_ATTR_NAME);
	const char *interval_s = crm_element_value(expr, XML_LRM_ATTR_INTERVAL);
	guint interval;

	crm_trace("Testing op_defaults expression: %s", ID(expr));

	if (rule_data->op_data == NULL) {
	crm_trace("No operations data provided");
	return FALSE;
	}

	interval = crm_parse_interval_spec(interval_s);
	if (interval == 0 && errno != 0) {
	crm_trace("Could not parse interval: %s", interval_s);
	return FALSE;
	}

	if (interval_s != NULL && interval != rule_data->op_data->interval) {
	crm_trace("Interval doesn't match: %d != %d", interval, rule_data->op_data->interval);
	return FALSE;
	}

	if (!pcmk__str_eq(name, rule_data->op_data->op_name, pcmk__str_none)) {
	crm_trace("Name doesn't match: %s != %s", name, rule_data->op_data->op_name);
	return FALSE;
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Check whether a resource role matches a rule role
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data Only the role member is used
	*
	* \return true if role matches, otherwise false
	*/
	static bool
	role_matches(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	const char *value = crm_element_value(expr, XML_EXPR_ATTR_VALUE);
	enum rsc_role_e role = text2role(value);

	if (role == pcmk_role_unknown) {
	pcmk__config_err("Invalid role %s in rule expression", value);
	return false;
	}
	return role == rule_data->role;
	}

	/*!
	* \internal
	* \brief Evaluate a node attribute expression based on #role
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data Only the role member is used
	*
	* \return TRUE if rule_data->role satisfies the expression, FALSE otherwise
	*/
	gboolean
	pe__eval_role_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	const char *op = NULL;

	if (rule_data->role == pcmk_role_unknown) { // Skip role checks for now
	return FALSE;
	}

	op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION);

	// @TODO Why are only Unpromoted/Promoted considered defined?

	if (pcmk__str_eq(op, "defined", pcmk__str_casei)) {
	if (rule_data->role > pcmk_role_started) {
	return TRUE;
	}

	} else if (pcmk__str_eq(op, "not_defined", pcmk__str_casei)) {
	if ((rule_data->role > pcmk_role_unknown)
	&& (rule_data->role < pcmk_role_unpromoted)) {
	return TRUE;
	}

	} else if (pcmk__str_eq(op, "eq", pcmk__str_casei)) {
	return role_matches(expr, rule_data)? TRUE : FALSE;

	} else if (pcmk__str_eq(op, "ne", pcmk__str_casei)
	// Test "ne" only with promotable clone roles (@TODO Why?)
	&& (rule_data->role >= pcmk_role_unpromoted)) {
	return role_matches(expr, rule_data)? FALSE : TRUE;

	} else {
	pcmk__config_err("Operation '%s' is not valid with " CRM_ATTR_ROLE
	" comparisons", op);
	}
	return FALSE;
	}

	gboolean
	pe__eval_rsc_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	const char *class = crm_element_value(expr, XML_AGENT_ATTR_CLASS);
	const char *provider = crm_element_value(expr, XML_AGENT_ATTR_PROVIDER);
	const char *type = crm_element_value(expr, XML_EXPR_ATTR_TYPE);

	crm_trace("Testing rsc_defaults expression: %s", ID(expr));

	if (rule_data->rsc_data == NULL) {
	crm_trace("No resource data provided");
	return FALSE;
	}

	if (class != NULL &&
	!pcmk__str_eq(class, rule_data->rsc_data->standard, pcmk__str_none)) {
	crm_trace("Class doesn't match: %s != %s", class, rule_data->rsc_data->standard);
	return FALSE;
	}

	if ((provider == NULL && rule_data->rsc_data->provider != NULL) \|\|
	(provider != NULL && rule_data->rsc_data->provider == NULL) \|\|
	!pcmk__str_eq(provider, rule_data->rsc_data->provider, pcmk__str_none)) {
	crm_trace("Provider doesn't match: %s != %s", provider, rule_data->rsc_data->provider);
	return FALSE;
	}

	if (type != NULL &&
	!pcmk__str_eq(type, rule_data->rsc_data->agent, pcmk__str_none)) {
	crm_trace("Agent doesn't match: %s != %s", type, rule_data->rsc_data->agent);
	return FALSE;
	}

	return TRUE;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/pengine/rules_compat.h>

	gboolean
	test_ruleset(xmlNode ruleset, GHashTable node_hash, crm_time_t *now)
	{
	return pe_evaluate_rules(ruleset, node_hash, now, NULL);
	}

	gboolean
	test_rule(xmlNode * rule, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now)
	{
	return pe_test_rule(rule, node_hash, role, now, NULL, NULL);
	}

	gboolean
	pe_test_rule_re(xmlNode * rule, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now, pe_re_match_data_t * re_match_data)
	{
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = NULL,
	.meta = NULL,
	};
	return pe_test_rule(rule, node_hash, role, now, NULL, &match_data);
	}

	gboolean
	pe_test_rule_full(xmlNode rule, GHashTable node_hash, enum rsc_role_e role,
	crm_time_t now, pe_match_data_t match_data)
	{
	return pe_test_rule(rule, node_hash, role, now, NULL, match_data);
	}

	gboolean
	test_expression(xmlNode * expr, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now)
	{
	return pe_test_expression(expr, node_hash, role, now, NULL, NULL);
	}

	gboolean
	pe_test_expression_re(xmlNode * expr, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now, pe_re_match_data_t * re_match_data)
	{
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = NULL,
	.meta = NULL,
	};
	return pe_test_expression(expr, node_hash, role, now, NULL, &match_data);
	}

	gboolean
	pe_test_expression_full(xmlNode expr, GHashTable node_hash,
	enum rsc_role_e role, crm_time_t *now,
	pe_match_data_t *match_data)
	{
	return pe_test_expression(expr, node_hash, role, now, NULL, match_data);
	}

	void
	unpack_instance_attributes(xmlNode top, xmlNode xml_obj, const char *set_name,
	GHashTable node_hash, GHashTable hash,
	const char *always_first, gboolean overwrite,
	crm_time_t *now)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe_eval_nvpairs(top, xml_obj, set_name, &rule_data, hash, always_first,
	overwrite, NULL);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/pengine/rules_alerts.c b/lib/pengine/rules_alerts.c
	index 9eed7ff890..30eaf59b5c 100644
	--- a/lib/pengine/rules_alerts.c
	+++ b/lib/pengine/rules_alerts.c
	@@ -1,294 +1,302 @@
	/*
	* Copyright 2015-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/rules.h>
	#include <crm/common/alerts_internal.h>
	#include <crm/common/xml_internal.h>
	#include <crm/pengine/rules_internal.h>

	/*!
	* \internal
	* \brief Unpack an alert's or alert recipient's meta attributes
	*
	* \param[in,out] basenode Alert or recipient XML
	* \param[in,out] entry Where to store unpacked values
	* \param[in,out] max_timeout Max timeout of all alerts and recipients thus far
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_meta_attrs_from_cib(xmlNode basenode, pcmk__alert_t entry,
	guint *max_timeout)
	{
	GHashTable *config_hash = pcmk__strkey_table(free, free);
	crm_time_t *now = crm_time_new(NULL);
	const char *value = NULL;
	int rc = pcmk_rc_ok;

	pe_unpack_nvpairs(basenode, basenode, XML_TAG_META_SETS, NULL, config_hash,
	NULL, FALSE, now, NULL);
	crm_time_free(now);

	value = g_hash_table_lookup(config_hash, PCMK_META_ENABLED);
	if ((value != NULL) && !crm_is_true(value)) {
	// No need to continue unpacking
	rc = pcmk_rc_disabled;
	goto done;
	}

	value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_TIMEOUT);
	if (value) {
	entry->timeout = crm_get_msec(value);
	if (entry->timeout <= 0) {
	if (entry->timeout == 0) {
	crm_trace("Alert %s uses default timeout of %dmsec",
	entry->id, PCMK__ALERT_DEFAULT_TIMEOUT_MS);
	} else {
	- crm_warn("Alert %s has invalid timeout value '%s', using default %dmsec",
	- entry->id, (char*)value, PCMK__ALERT_DEFAULT_TIMEOUT_MS);
	+ pcmk__config_warn("Alert %s has invalid timeout value '%s', "
	+ "using default (%d ms)",
	+ entry->id, value,
	+ PCMK__ALERT_DEFAULT_TIMEOUT_MS);
	}
	entry->timeout = PCMK__ALERT_DEFAULT_TIMEOUT_MS;
	} else {
	crm_trace("Alert %s uses timeout of %dmsec",
	entry->id, entry->timeout);
	}
	if (entry->timeout > *max_timeout) {
	*max_timeout = entry->timeout;
	}
	}
	value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_TSTAMP_FORMAT);
	if (value) {
	/* hard to do any checks here as merely anything can
	* can be a valid time-format-string
	*/
	entry->tstamp_format = strdup(value);
	crm_trace("Alert %s uses timestamp format '%s'",
	entry->id, entry->tstamp_format);
	}

	done:
	g_hash_table_destroy(config_hash);
	return rc;
	}

	static void
	get_envvars_from_cib(xmlNode basenode, pcmk__alert_t entry)
	{
	xmlNode *child;

	if ((basenode == NULL) \|\| (entry == NULL)) {
	return;
	}

	child = first_named_child(basenode, XML_TAG_ATTR_SETS);
	if (child == NULL) {
	return;
	}

	if (entry->envvars == NULL) {
	entry->envvars = pcmk__strkey_table(free, free);
	}

	for (child = first_named_child(child, XML_CIB_TAG_NVPAIR); child != NULL;
	child = crm_next_same_xml(child)) {

	const char *name = crm_element_value(child, XML_NVPAIR_ATTR_NAME);
	const char *value = crm_element_value(child, XML_NVPAIR_ATTR_VALUE);

	if (value == NULL) {
	value = "";
	}
	g_hash_table_insert(entry->envvars, strdup(name), strdup(value));
	crm_trace("Alert %s: added environment variable %s='%s'",
	entry->id, name, value);
	}
	}

	static void
	unpack_alert_filter(xmlNode basenode, pcmk__alert_t entry)
	{
	xmlNode *select = first_named_child(basenode, XML_CIB_TAG_ALERT_SELECT);
	xmlNode *event_type = NULL;
	uint32_t flags = pcmk__alert_none;

	for (event_type = pcmk__xe_first_child(select); event_type != NULL;
	event_type = pcmk__xe_next(event_type)) {

	if (pcmk__xe_is(event_type, XML_CIB_TAG_ALERT_FENCING)) {
	flags \|= pcmk__alert_fencing;

	} else if (pcmk__xe_is(event_type, XML_CIB_TAG_ALERT_NODES)) {
	flags \|= pcmk__alert_node;

	} else if (pcmk__xe_is(event_type, XML_CIB_TAG_ALERT_RESOURCES)) {
	flags \|= pcmk__alert_resource;

	} else if (pcmk__xe_is(event_type, XML_CIB_TAG_ALERT_ATTRIBUTES)) {
	xmlNode *attr;
	const char *attr_name;
	int nattrs = 0;

	flags \|= pcmk__alert_attribute;
	for (attr = first_named_child(event_type, XML_CIB_TAG_ALERT_ATTR);
	attr != NULL;
	attr = crm_next_same_xml(attr)) {

	attr_name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
	if (attr_name) {
	if (nattrs == 0) {
	g_strfreev(entry->select_attribute_name);
	entry->select_attribute_name = NULL;
	}
	++nattrs;
	entry->select_attribute_name = pcmk__realloc(entry->select_attribute_name,
	(nattrs + 1) * sizeof(char*));
	entry->select_attribute_name[nattrs - 1] = strdup(attr_name);
	entry->select_attribute_name[nattrs] = NULL;
	}
	}
	}
	}

	if (flags != pcmk__alert_none) {
	entry->flags = flags;
	crm_debug("Alert %s receives events: attributes:%s%s%s%s",
	entry->id,
	(pcmk_is_set(flags, pcmk__alert_attribute)?
	(entry->select_attribute_name? "some" : "all") : "none"),
	(pcmk_is_set(flags, pcmk__alert_fencing)? " fencing" : ""),
	(pcmk_is_set(flags, pcmk__alert_node)? " nodes" : ""),
	(pcmk_is_set(flags, pcmk__alert_resource)? " resources" : ""));
	}
	}

	/*!
	* \internal
	* \brief Unpack an alert or an alert recipient
	*
	* \param[in,out] alert Alert or recipient XML
	* \param[in,out] entry Where to store unpacked values
	* \param[in,out] max_timeout Max timeout of all alerts and recipients thus far
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_alert(xmlNode alert, pcmk__alert_t entry, guint *max_timeout)
	{
	int rc = pcmk_rc_ok;

	get_envvars_from_cib(alert, entry);
	rc = get_meta_attrs_from_cib(alert, entry, max_timeout);
	if (rc == pcmk_rc_ok) {
	unpack_alert_filter(alert, entry);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Unpack a CIB alerts section
	*
	* \param[in] alerts XML of alerts section
	*
	* \return List of unpacked alert entries
	*
	* \note Unlike most unpack functions, this is not used by the scheduler itself,
	* but is supplied for use by daemons that need to send alerts.
	*/
	GList *
	pe_unpack_alerts(const xmlNode *alerts)
	{
	xmlNode *alert;
	pcmk__alert_t *entry;
	guint max_timeout = 0;
	GList *alert_list = NULL;

	if (alerts == NULL) {
	return alert_list;
	}

	for (alert = first_named_child(alerts, XML_CIB_TAG_ALERT);
	alert != NULL; alert = crm_next_same_xml(alert)) {

	xmlNode *recipient;
	int recipients = 0;
	const char *alert_id = ID(alert);
	const char *alert_path = crm_element_value(alert, XML_ALERT_ATTR_PATH);

	/* The schema should enforce this, but to be safe ... */
	- if ((alert_id == NULL) \|\| (alert_path == NULL)) {
	- crm_warn("Ignoring invalid alert without id and path");
	+ if (alert_id == NULL) {
	+ pcmk__config_warn("Ignoring invalid alert without " XML_ATTR_ID);
	+ crm_log_xml_info(alert, "missing-id");
	+ continue;
	+ }
	+ if (alert_path == NULL) {
	+ pcmk__config_warn("Ignoring alert %s: No " XML_ALERT_ATTR_PATH,
	+ alert_id);
	continue;
	}

	entry = pcmk__alert_new(alert_id, alert_path);

	if (unpack_alert(alert, entry, &max_timeout) != pcmk_rc_ok) {
	// Don't allow recipients to override if entire alert is disabled
	crm_debug("Alert %s is disabled", entry->id);
	pcmk__free_alert(entry);
	continue;
	}

	if (entry->tstamp_format == NULL) {
	entry->tstamp_format = strdup(PCMK__ALERT_DEFAULT_TSTAMP_FORMAT);
	}

	crm_debug("Alert %s: path=%s timeout=%dms tstamp-format='%s' %u vars",
	entry->id, entry->path, entry->timeout, entry->tstamp_format,
	(entry->envvars? g_hash_table_size(entry->envvars) : 0));

	for (recipient = first_named_child(alert, XML_CIB_TAG_ALERT_RECIPIENT);
	recipient != NULL; recipient = crm_next_same_xml(recipient)) {

	pcmk__alert_t *recipient_entry = pcmk__dup_alert(entry);

	recipients++;
	recipient_entry->recipient = strdup(crm_element_value(recipient,
	XML_ALERT_ATTR_REC_VALUE));

	if (unpack_alert(recipient, recipient_entry,
	&max_timeout) != pcmk_rc_ok) {
	crm_debug("Alert %s: recipient %s is disabled",
	entry->id, recipient_entry->id);
	pcmk__free_alert(recipient_entry);
	continue;
	}
	alert_list = g_list_prepend(alert_list, recipient_entry);
	crm_debug("Alert %s has recipient %s with value %s and %d envvars",
	entry->id, ID(recipient), recipient_entry->recipient,
	(recipient_entry->envvars?
	g_hash_table_size(recipient_entry->envvars) : 0));
	}

	if (recipients == 0) {
	alert_list = g_list_prepend(alert_list, entry);
	} else {
	pcmk__free_alert(entry);
	}
	}
	return alert_list;
	}

	/*!
	* \internal
	* \brief Free an alert list generated by pe_unpack_alerts()
	*
	* \param[in,out] alert_list Alert list to free
	*/
	void
	pe_free_alert_list(GList *alert_list)
	{
	if (alert_list) {
	g_list_free_full(alert_list, (GDestroyNotify) pcmk__free_alert);
	}
	}
	diff --git a/lib/pengine/status.c b/lib/pengine/status.c
	index ae1fbce361..739bae58d6 100644
	--- a/lib/pengine/status.c
	+++ b/lib/pengine/status.c
	@@ -1,483 +1,484 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	/*!
	* \brief Create a new object to hold scheduler data
	*
	* \return New, initialized scheduler data on success, else NULL (and set errno)
	* \note Only pcmk_scheduler_t objects created with this function (as opposed
	* to statically declared or directly allocated) should be used with the
	* functions in this library, to allow for future extensions to the
	* data type. The caller is responsible for freeing the memory with
	* pe_free_working_set() when the instance is no longer needed.
	*/
	pcmk_scheduler_t *
	pe_new_working_set(void)
	{
	pcmk_scheduler_t *scheduler = calloc(1, sizeof(pcmk_scheduler_t));

	if (scheduler != NULL) {
	set_working_set_defaults(scheduler);
	}
	return scheduler;
	}

	/*!
	* \brief Free scheduler data
	*
	* \param[in,out] scheduler Scheduler data to free
	*/
	void
	pe_free_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler != NULL) {
	pe_reset_working_set(scheduler);
	scheduler->priv = NULL;
	free(scheduler);
	}
	}

	/*
	* Unpack everything
	* At the end you'll have:
	* - A list of nodes
	* - A list of resources (each with any dependencies on other resources)
	* - A list of constraints between resources and nodes
	* - A list of constraints between start/stop actions
	* - A list of nodes that need to be stonith'd
	* - A list of nodes that need to be shutdown
	* - A list of the possible stop/start actions (without dependencies)
	*/
	gboolean
	cluster_status(pcmk_scheduler_t * scheduler)
	{
	xmlNode *section = NULL;

	if ((scheduler == NULL) \|\| (scheduler->input == NULL)) {
	return FALSE;
	}

	crm_trace("Beginning unpack");

	if (scheduler->failed != NULL) {
	free_xml(scheduler->failed);
	}
	scheduler->failed = create_xml_node(NULL, "failed-ops");

	if (scheduler->now == NULL) {
	scheduler->now = crm_time_new(NULL);
	}

	if (scheduler->dc_uuid == NULL) {
	scheduler->dc_uuid = crm_element_value_copy(scheduler->input,
	XML_ATTR_DC_UUID);
	}

	if (pcmk__xe_attr_is_true(scheduler->input, XML_ATTR_HAVE_QUORUM)) {
	pe__set_working_set_flags(scheduler, pcmk_sched_quorate);
	} else {
	pe__clear_working_set_flags(scheduler, pcmk_sched_quorate);
	}

	scheduler->op_defaults = get_xpath_object("//" XML_CIB_TAG_OPCONFIG,
	scheduler->input, LOG_NEVER);
	scheduler->rsc_defaults = get_xpath_object("//" XML_CIB_TAG_RSCCONFIG,
	scheduler->input, LOG_NEVER);

	section = get_xpath_object("//" XML_CIB_TAG_CRMCONFIG, scheduler->input,
	LOG_TRACE);
	unpack_config(section, scheduler);

	if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_location_only\|pcmk_sched_quorate)
	&& (scheduler->no_quorum_policy != pcmk_no_quorum_ignore)) {
	- crm_warn("Fencing and resource management disabled due to lack of quorum");
	+ pcmk__sched_warn("Fencing and resource management disabled "
	+ "due to lack of quorum");
	}

	section = get_xpath_object("//" XML_CIB_TAG_NODES, scheduler->input,
	LOG_TRACE);
	unpack_nodes(section, scheduler);

	section = get_xpath_object("//" XML_CIB_TAG_RESOURCES, scheduler->input,
	LOG_TRACE);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	unpack_remote_nodes(section, scheduler);
	}
	unpack_resources(section, scheduler);

	section = get_xpath_object("//" XML_CIB_TAG_TAGS, scheduler->input,
	LOG_NEVER);
	unpack_tags(section, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	section = get_xpath_object("//"XML_CIB_TAG_STATUS, scheduler->input,
	LOG_TRACE);
	unpack_status(section, scheduler);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_no_counts)) {
	for (GList *item = scheduler->resources; item != NULL;
	item = item->next) {
	((pcmk_resource_t *) (item->data))->fns->count(item->data);
	}
	crm_trace("Cluster resource count: %d (%d disabled, %d blocked)",
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	pe__set_working_set_flags(scheduler, pcmk_sched_have_status);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Free a list of pcmk_resource_t
	*
	* \param[in,out] resources List to free
	*
	* \note When the scheduler's resource list is freed, that includes the original
	* storage for the uname and id of any Pacemaker Remote nodes in the
	* scheduler's node list, so take care not to use those afterward.
	* \todo Refactor pcmk_node_t to strdup() the node name.
	*/
	static void
	pe_free_resources(GList *resources)
	{
	pcmk_resource_t *rsc = NULL;
	GList *iterator = resources;

	while (iterator != NULL) {
	rsc = (pcmk_resource_t *) iterator->data;
	iterator = iterator->next;
	rsc->fns->free(rsc);
	}
	if (resources != NULL) {
	g_list_free(resources);
	}
	}

	static void
	pe_free_actions(GList *actions)
	{
	GList *iterator = actions;

	while (iterator != NULL) {
	pe_free_action(iterator->data);
	iterator = iterator->next;
	}
	if (actions != NULL) {
	g_list_free(actions);
	}
	}

	static void
	pe_free_nodes(GList *nodes)
	{
	for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) {
	pcmk_node_t node = (pcmk_node_t ) iterator->data;

	// Shouldn't be possible, but to be safe ...
	if (node == NULL) {
	continue;
	}
	if (node->details == NULL) {
	free(node);
	continue;
	}

	/* This is called after pe_free_resources(), which means that we can't
	* use node->details->uname for Pacemaker Remote nodes.
	*/
	crm_trace("Freeing node %s", (pe__is_guest_or_remote_node(node)?
	"(guest or remote)" : pe__node_name(node)));

	if (node->details->attrs != NULL) {
	g_hash_table_destroy(node->details->attrs);
	}
	if (node->details->utilization != NULL) {
	g_hash_table_destroy(node->details->utilization);
	}
	if (node->details->digest_cache != NULL) {
	g_hash_table_destroy(node->details->digest_cache);
	}
	g_list_free(node->details->running_rsc);
	g_list_free(node->details->allocated_rsc);
	free(node->details);
	free(node);
	}
	if (nodes != NULL) {
	g_list_free(nodes);
	}
	}

	static void
	pe__free_ordering(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__action_relation_t *order = iterator->data;

	iterator = iterator->next;

	free(order->task1);
	free(order->task2);
	free(order);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	static void
	pe__free_location(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__location_t *cons = iterator->data;

	iterator = iterator->next;

	g_list_free_full(cons->nodes, free);
	free(cons->id);
	free(cons);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	/*!
	* \brief Reset scheduler data to defaults without freeing it or constraints
	*
	* \param[in,out] scheduler Scheduler data to reset
	*
	* \deprecated This function is deprecated as part of the API;
	* pe_reset_working_set() should be used instead.
	*/
	void
	cleanup_calculations(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	pe__clear_working_set_flags(scheduler, pcmk_sched_have_status);
	if (scheduler->config_hash != NULL) {
	g_hash_table_destroy(scheduler->config_hash);
	}

	if (scheduler->singletons != NULL) {
	g_hash_table_destroy(scheduler->singletons);
	}

	if (scheduler->tickets) {
	g_hash_table_destroy(scheduler->tickets);
	}

	if (scheduler->template_rsc_sets) {
	g_hash_table_destroy(scheduler->template_rsc_sets);
	}

	if (scheduler->tags) {
	g_hash_table_destroy(scheduler->tags);
	}

	free(scheduler->dc_uuid);

	crm_trace("deleting resources");
	pe_free_resources(scheduler->resources);

	crm_trace("deleting actions");
	pe_free_actions(scheduler->actions);

	crm_trace("deleting nodes");
	pe_free_nodes(scheduler->nodes);

	pe__free_param_checks(scheduler);
	g_list_free(scheduler->stop_needed);
	free_xml(scheduler->graph);
	crm_time_free(scheduler->now);
	free_xml(scheduler->input);
	free_xml(scheduler->failed);

	set_working_set_defaults(scheduler);

	CRM_CHECK(scheduler->ordering_constraints == NULL,;
	);
	CRM_CHECK(scheduler->placement_constraints == NULL,;
	);
	}

	/*!
	* \brief Reset scheduler data to default state without freeing it
	*
	* \param[in,out] scheduler Scheduler data to reset
	*/
	void
	pe_reset_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	crm_trace("Deleting %d ordering constraints",
	g_list_length(scheduler->ordering_constraints));
	pe__free_ordering(scheduler->ordering_constraints);
	scheduler->ordering_constraints = NULL;

	crm_trace("Deleting %d location constraints",
	g_list_length(scheduler->placement_constraints));
	pe__free_location(scheduler->placement_constraints);
	scheduler->placement_constraints = NULL;

	crm_trace("Deleting %d colocation constraints",
	g_list_length(scheduler->colocation_constraints));
	g_list_free_full(scheduler->colocation_constraints, free);
	scheduler->colocation_constraints = NULL;

	crm_trace("Deleting %d ticket constraints",
	g_list_length(scheduler->ticket_constraints));
	g_list_free_full(scheduler->ticket_constraints, free);
	scheduler->ticket_constraints = NULL;

	cleanup_calculations(scheduler);
	}

	void
	set_working_set_defaults(pcmk_scheduler_t *scheduler)
	{
	void *priv = scheduler->priv;

	memset(scheduler, 0, sizeof(pcmk_scheduler_t));

	scheduler->priv = priv;
	scheduler->order_id = 1;
	scheduler->action_id = 1;
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;

	scheduler->flags = 0x0ULL;

	pe__set_working_set_flags(scheduler,
	pcmk_sched_symmetric_cluster
	\|pcmk_sched_stop_removed_resources
	\|pcmk_sched_cancel_removed_actions);
	if (!strcmp(PCMK__CONCURRENT_FENCING_DEFAULT, "true")) {
	pe__set_working_set_flags(scheduler, pcmk_sched_concurrent_fencing);
	}
	}

	pcmk_resource_t *
	pe_find_resource(GList rsc_list, const char id)
	{
	return pe_find_resource_with_flags(rsc_list, id, pcmk_rsc_match_history);
	}

	pcmk_resource_t *
	pe_find_resource_with_flags(GList rsc_list, const char id, enum pe_find flags)
	{
	GList *rIter = NULL;

	for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
	pcmk_resource_t *parent = rIter->data;

	pcmk_resource_t *match =
	parent->fns->find_rsc(parent, id, NULL, flags);
	if (match != NULL) {
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	/*!
	* \brief Find a node by name or ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id If not NULL, ID of node to find
	* \param[in] node_name If not NULL, name of node to find
	*
	* \return Node from \p nodes that matches \p id if any,
	* otherwise node from \p nodes that matches \p uname if any,
	* otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_any(const GList nodes, const char id, const char *uname)
	{
	pcmk_node_t *match = NULL;

	if (id != NULL) {
	match = pe_find_node_id(nodes, id);
	}
	if ((match == NULL) && (uname != NULL)) {
	match = pe_find_node(nodes, uname);
	}
	return match;
	}

	/*!
	* \brief Find a node by ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id ID of node to find
	*
	* \return Node from \p nodes that matches \p id if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_id(const GList nodes, const char id)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	/* @TODO Whether node IDs should be considered case-sensitive should
	* probably depend on the node type, so functionizing the comparison
	* would be worthwhile
	*/
	if (pcmk__str_eq(node->details->id, id, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}

	/*!
	* \brief Find a node by name in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] node_name Name of node to find
	*
	* \return Node from \p nodes that matches \p node_name if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node(const GList nodes, const char node_name)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	if (pcmk__str_eq(node->details->uname, node_name, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index b7cbc8b822..3e4410530e 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,5090 +1,5100 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <glib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	CRM_TRACE_INIT_DATA(pe_status);

	// A (parsed) resource action history entry
	struct action_history {
	pcmk_resource_t *rsc; // Resource that history is for
	pcmk_node_t *node; // Node that history is for
	xmlNode *xml; // History entry XML

	// Parsed from entry XML
	const char *id; // XML ID of history entry
	const char *key; // Operation key of action
	const char *task; // Action name
	const char *exit_reason; // Exit reason given for result
	guint interval_ms; // Action interval
	int call_id; // Call ID of action
	int expected_exit_status; // Expected exit status of action
	int exit_status; // Actual exit status of action
	int execution_status; // Execution status of action
	};

	/* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
	* use pe__set_working_set_flags()/pe__clear_working_set_flags() so that the
	* flag is stringified more readably in log messages.
	*/
	#define set_config_flag(scheduler, option, flag) do { \
	const char *scf_value = pe_pref((scheduler)->config_hash, (option)); \
	if (scf_value != NULL) { \
	if (crm_is_true(scf_value)) { \
	(scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} else { \
	(scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} \
	} \
	} while(0)

	static void unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node,
	xmlNode xml_op, xmlNode *last_failure,
	enum action_fail_response *failed);
	static void determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node);
	static void add_node_attrs(const xmlNode xml_obj, pcmk_node_t node,
	bool overwrite, pcmk_scheduler_t *scheduler);
	static void determine_online_status(const xmlNode *node_state,
	pcmk_node_t *this_node,
	pcmk_scheduler_t *scheduler);

	static void unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler);


	static gboolean
	is_dangling_guest_node(pcmk_node_t *node)
	{
	/* we are looking for a remote-node that was supposed to be mapped to a
	* container resource, but all traces of that container have disappeared
	* from both the config and the status section. */
	if (pe__is_guest_or_remote_node(node) &&
	node->details->remote_rsc &&
	node->details->remote_rsc->container == NULL &&
	pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_removed_filler)) {
	return TRUE;
	}

	return FALSE;
	}

	/*!
	* \brief Schedule a fence action for a node
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in,out] node Node to fence
	* \param[in] reason Text description of why fencing is needed
	* \param[in] priority_delay Whether to consider `priority-fencing-delay`
	*/
	void
	pe_fence_node(pcmk_scheduler_t scheduler, pcmk_node_t node,
	const char *reason, bool priority_delay)
	{
	CRM_CHECK(node, return);

	/* A guest node is fenced by marking its container as failed */
	if (pe__is_guest_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc->container;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing guest node %s "
	"(otherwise would because %s): "
	"its guest resource %s is unmanaged",
	pe__node_name(node), reason, rsc->id);
	} else {
	- crm_warn("Guest node %s will be fenced "
	- "(by recovering its guest resource %s): %s",
	- pe__node_name(node), rsc->id, reason);
	+ pcmk__sched_warn("Guest node %s will be fenced "
	+ "(by recovering its guest resource %s): %s",
	+ pe__node_name(node), rsc->id, reason);

	/* We don't mark the node as unclean because that would prevent the
	* node from running resources. We want to allow it to run resources
	* in this transition if the recovery succeeds.
	*/
	node->details->remote_requires_reset = TRUE;
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	}

	} else if (is_dangling_guest_node(node)) {
	crm_info("Cleaning up dangling connection for guest node %s: "
	"fencing was already done because %s, "
	"and guest resource no longer exists",
	pe__node_name(node), reason);
	pe__set_resource_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);

	} else if (pe__is_remote_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc;

	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing remote node %s "
	"(otherwise would because %s): connection is unmanaged",
	pe__node_name(node), reason);
	} else if(node->details->remote_requires_reset == FALSE) {
	node->details->remote_requires_reset = TRUE;
	- crm_warn("Remote node %s %s: %s",
	- pe__node_name(node),
	- pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	- reason);
	+ pcmk__sched_warn("Remote node %s %s: %s",
	+ pe__node_name(node),
	+ pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	+ reason);
	}
	node->details->unclean = TRUE;
	// No need to apply `priority-fencing-delay` for remote nodes
	pe_fence_op(node, NULL, TRUE, reason, FALSE, scheduler);

	} else if (node->details->unclean) {
	crm_trace("Cluster node %s %s because %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "would also be fenced" : "also is unclean",
	reason);

	} else {
	- crm_warn("Cluster node %s %s: %s",
	- pe__node_name(node),
	- pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	- reason);
	+ pcmk__sched_warn("Cluster node %s %s: %s",
	+ pe__node_name(node),
	+ pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	+ reason);
	node->details->unclean = TRUE;
	pe_fence_op(node, NULL, TRUE, reason, priority_delay, scheduler);
	}
	}

	// @TODO xpaths can't handle templates, rules, or id-refs

	// nvpair with provides or requires set to unfencing
	#define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR \
	"[(@" XML_NVPAIR_ATTR_NAME "='" PCMK_STONITH_PROVIDES "'" \
	"or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
	"and @" XML_NVPAIR_ATTR_VALUE "='" PCMK__VALUE_UNFENCING "']"

	// unfencing in rsc_defaults or any resource
	#define XPATH_ENABLE_UNFENCING \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES \
	"//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR \
	"\|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG \
	"/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR

	static void
	set_if_xpath(uint64_t flag, const char xpath, pcmk_scheduler_t scheduler)
	{
	xmlXPathObjectPtr result = NULL;

	if (!pcmk_is_set(scheduler->flags, flag)) {
	result = xpath_search(scheduler->input, xpath);
	if (result && (numXpathResults(result) > 0)) {
	pe__set_working_set_flags(scheduler, flag);
	}
	freeXpathObject(result);
	}
	}

	gboolean
	unpack_config(xmlNode config, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	GHashTable *config_hash = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	scheduler->config_hash = config_hash;

	pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
	CIB_OPTIONS_FIRST, FALSE, scheduler);

	verify_pe_options(scheduler->config_hash);

	set_config_flag(scheduler, "enable-startup-probes",
	pcmk_sched_probe_resources);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pe_pref(scheduler->config_hash, XML_ATTR_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_info("Watchdog-based self-fencing will be performed via SBD if "
	"fencing is required and stonith-watchdog-timeout is nonzero");
	pe__set_working_set_flags(scheduler, pcmk_sched_have_fencing);
	}

	/* Set certain flags via xpath here, so they can be used before the relevant
	* configuration sections are unpacked.
	*/
	set_if_xpath(pcmk_sched_enable_unfencing, XPATH_ENABLE_UNFENCING,
	scheduler);

	value = pe_pref(scheduler->config_hash, "stonith-timeout");
	scheduler->stonith_timeout = (int) crm_parse_interval_spec(value);
	crm_debug("STONITH timeout: %d", scheduler->stonith_timeout);

	set_config_flag(scheduler, "stonith-enabled", pcmk_sched_fencing_enabled);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	crm_debug("STONITH of failed nodes is enabled");
	} else {
	crm_debug("STONITH of failed nodes is disabled");
	}

	scheduler->stonith_action = pe_pref(scheduler->config_hash,
	"stonith-action");
	if (!strcmp(scheduler->stonith_action, "poweroff")) {
	pe_warn_once(pcmk__wo_poweroff,
	"Support for stonith-action of 'poweroff' is deprecated "
	"and will be removed in a future release (use 'off' instead)");
	scheduler->stonith_action = PCMK_ACTION_OFF;
	}
	crm_trace("STONITH will %s nodes", scheduler->stonith_action);

	set_config_flag(scheduler, "concurrent-fencing",
	pcmk_sched_concurrent_fencing);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	crm_debug("Concurrent fencing is enabled");
	} else {
	crm_debug("Concurrent fencing is disabled");
	}

	value = pe_pref(scheduler->config_hash,
	XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
	if (value) {
	scheduler->priority_fencing_delay = crm_parse_interval_spec(value)
	/ 1000;
	crm_trace("Priority fencing delay is %ds",
	scheduler->priority_fencing_delay);
	}

	set_config_flag(scheduler, "stop-all-resources", pcmk_sched_stop_all);
	crm_debug("Stop all active resources: %s",
	pcmk__btoa(pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)));

	set_config_flag(scheduler, "symmetric-cluster",
	pcmk_sched_symmetric_cluster);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pe_pref(scheduler->config_hash, "no-quorum-policy");

	if (pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_ignore;

	} else if (pcmk__str_eq(value, "freeze", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_freeze;

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_demote;

	} else if (pcmk__str_eq(value, "suicide", pcmk__str_casei)) {
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	int do_panic = 0;

	crm_element_value_int(scheduler->input, XML_ATTR_QUORUM_PANIC,
	&do_panic);
	if (do_panic \|\| pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_fence;
	} else {
	crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}
	} else {
	pcmk__config_err("Resetting no-quorum-policy to 'stop' because "
	"fencing is disabled");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	} else {
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	crm_debug("On loss of quorum: Freeze resources");
	break;
	case pcmk_no_quorum_stop:
	crm_debug("On loss of quorum: Stop ALL resources");
	break;
	case pcmk_no_quorum_demote:
	crm_debug("On loss of quorum: "
	"Demote promotable resources and stop other resources");
	break;
	case pcmk_no_quorum_fence:
	crm_notice("On loss of quorum: Fence all remaining nodes");
	break;
	case pcmk_no_quorum_ignore:
	crm_notice("On loss of quorum: Ignore");
	break;
	}

	set_config_flag(scheduler, "stop-orphan-resources",
	pcmk_sched_stop_removed_resources);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	crm_trace("Orphan resources are stopped");
	} else {
	crm_trace("Orphan resources are ignored");
	}

	set_config_flag(scheduler, "stop-orphan-actions",
	pcmk_sched_cancel_removed_actions);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_cancel_removed_actions)) {
	crm_trace("Orphan resource actions are stopped");
	} else {
	crm_trace("Orphan resource actions are ignored");
	}

	value = pe_pref(scheduler->config_hash, "remove-after-stop");
	if (value != NULL) {
	if (crm_is_true(value)) {
	pe__set_working_set_flags(scheduler, pcmk_sched_remove_after_stop);
	#ifndef PCMK__COMPAT_2_0
	pe_warn_once(pcmk__wo_remove_after,
	"Support for the remove-after-stop cluster property is"
	" deprecated and will be removed in a future release");
	#endif
	} else {
	pe__clear_working_set_flags(scheduler,
	pcmk_sched_remove_after_stop);
	}
	}

	set_config_flag(scheduler, "maintenance-mode", pcmk_sched_in_maintenance);
	crm_trace("Maintenance mode: %s",
	pcmk__btoa(pcmk_is_set(scheduler->flags,
	pcmk_sched_in_maintenance)));

	set_config_flag(scheduler, "start-failure-is-fatal",
	pcmk_sched_start_failure_fatal);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
	crm_trace("Start failures are always fatal");
	} else {
	crm_trace("Start failures are handled by failcount");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	set_config_flag(scheduler, "startup-fencing",
	pcmk_sched_startup_fencing);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	crm_trace("Unseen nodes will be fenced");
	} else {
	pe_warn_once(pcmk__wo_blind, "Blind faith: not fencing unseen nodes");
	}

	pe__unpack_node_health_scores(scheduler);

	scheduler->placement_strategy = pe_pref(scheduler->config_hash,
	"placement-strategy");
	crm_trace("Placement strategy: %s", scheduler->placement_strategy);

	set_config_flag(scheduler, "shutdown-lock", pcmk_sched_shutdown_lock);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	value = pe_pref(scheduler->config_hash,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT);
	scheduler->shutdown_lock = crm_parse_interval_spec(value) / 1000;
	crm_trace("Resources will be locked to nodes that were cleanly "
	"shut down (locks expire after %s)",
	pcmk__readable_interval(scheduler->shutdown_lock));
	} else {
	crm_trace("Resources will not be locked to nodes that were cleanly "
	"shut down");
	}

	value = pe_pref(scheduler->config_hash,
	XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT);
	scheduler->node_pending_timeout = crm_parse_interval_spec(value) / 1000;
	if (scheduler->node_pending_timeout == 0) {
	crm_trace("Do not fence pending nodes");
	} else {
	crm_trace("Fence pending nodes after %s",
	pcmk__readable_interval(scheduler->node_pending_timeout
	* 1000));
	}

	return TRUE;
	}

	pcmk_node_t *
	pe_create_node(const char id, const char uname, const char *type,
	const char score, pcmk_scheduler_t scheduler)
	{
	pcmk_node_t *new_node = NULL;

	if (pe_find_node(scheduler->nodes, uname) != NULL) {
	pcmk__config_warn("More than one node entry has name '%s'", uname);
	}

	new_node = calloc(1, sizeof(pcmk_node_t));
	if (new_node == NULL) {
	+ pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	new_node->weight = char2score(score);
	new_node->details = calloc(1, sizeof(struct pe_node_shared_s));

	if (new_node->details == NULL) {
	free(new_node);
	+ pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->rsc_discovery_enabled = TRUE;
	new_node->details->running_rsc = NULL;
	new_node->details->data_set = scheduler;

	if (pcmk__str_eq(type, "member", pcmk__str_null_matches \| pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_cluster;

	} else if (pcmk__str_eq(type, "remote", pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_remote;
	pe__set_working_set_flags(scheduler, pcmk_sched_have_remote_nodes);

	} else {
	/* @COMPAT 'ping' is the default for backward compatibility, but it
	* should be changed to 'member' at a compatibility break
	*/
	if (!pcmk__str_eq(type, "ping", pcmk__str_casei)) {
	pcmk__config_warn("Node %s has unrecognized type '%s', "
	"assuming 'ping'", pcmk__s(uname, "without name"),
	type);
	}
	pe_warn_once(pcmk__wo_ping_node,
	"Support for nodes of type 'ping' (such as %s) is "
	"deprecated and will be removed in a future release",
	pcmk__s(uname, "unnamed node"));
	new_node->details->type = node_ping;
	}

	new_node->details->attrs = pcmk__strkey_table(free, free);

	if (pe__is_guest_or_remote_node(new_node)) {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("remote"));
	} else {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("cluster"));
	}

	new_node->details->utilization = pcmk__strkey_table(free, free);
	new_node->details->digest_cache = pcmk__strkey_table(free,
	pe__free_digests);

	scheduler->nodes = g_list_insert_sorted(scheduler->nodes, new_node,
	pe__cmp_node_name);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, pcmk_scheduler_t *data)
	{
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = ID(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	const char *is_managed = NULL;

	for (attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (!pcmk__str_eq((const char *)attr_set->name, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	continue;
	}

	for (attr = pcmk__xe_first_child(attr_set); attr != NULL;
	attr = pcmk__xe_next(attr)) {
	const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
	const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);

	if (pcmk__str_eq(name, XML_RSC_ATTR_REMOTE_NODE, pcmk__str_casei)) {
	remote_name = value;
	} else if (pcmk__str_eq(name, "remote-addr", pcmk__str_casei)) {
	remote_server = value;
	} else if (pcmk__str_eq(name, "remote-port", pcmk__str_casei)) {
	remote_port = value;
	} else if (pcmk__str_eq(name, "remote-connect-timeout", pcmk__str_casei)) {
	connect_timeout = value;
	} else if (pcmk__str_eq(name, "remote-allow-migrate", pcmk__str_casei)) {
	remote_allow_migrate=value;
	} else if (pcmk__str_eq(name, XML_RSC_ATTR_MANAGED, pcmk__str_casei)) {
	is_managed = value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (pe_find_resource(data->resources, remote_name) != NULL) {
	return NULL;
	}

	pe_create_remote_xml(parent, remote_name, container_id,
	remote_allow_migrate, is_managed,
	connect_timeout, remote_server, remote_port);
	return remote_name;
	}

	static void
	handle_startup_fencing(pcmk_scheduler_t scheduler, pcmk_node_t new_node)
	{
	if ((new_node->details->type == pcmk_node_variant_remote)
	&& (new_node->details->remote_rsc == NULL)) {
	/* Ignore fencing for remote nodes that don't have a connection resource
	* associated with them. This happens when remote node entries get left
	* in the nodes section after the connection resource is removed.
	*/
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	// All nodes are unclean until we've seen their status entry
	new_node->details->unclean = TRUE;

	} else {
	// Blind faith ...
	new_node->details->unclean = FALSE;
	}

	/* We need to be able to determine if a node's status section
	* exists or not separate from whether the node is unclean. */
	new_node->details->unseen = TRUE;
	}

	gboolean
	unpack_nodes(xmlNode xml_nodes, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	pcmk_node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_nodes); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, pcmk__str_none)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
	type = crm_element_value(xml_obj, XML_ATTR_TYPE);
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
	"> entry in configuration without id");
	continue;
	}
	new_node = pe_create_node(id, uname, type, score, scheduler);

	if (new_node == NULL) {
	return FALSE;
	}

	handle_startup_fencing(scheduler, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, scheduler);

	crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
	}
	}

	if (scheduler->localhost
	&& (pe_find_node(scheduler->nodes, scheduler->localhost) == NULL)) {
	crm_info("Creating a fake local node");
	pe_create_node(scheduler->localhost, scheduler->localhost, NULL, 0,
	scheduler);
	}

	return TRUE;
	}

	static void
	setup_container(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *container_id = NULL;

	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) setup_container, scheduler);
	return;
	}

	container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
	if (container_id && !pcmk__str_eq(container_id, rsc->id, pcmk__str_casei)) {
	pcmk_resource_t *container = pe_find_resource(scheduler->resources,
	container_id);

	if (container) {
	rsc->container = container;
	pe__set_resource_flags(container, pcmk_rsc_has_filler);
	container->fillers = g_list_append(container->fillers, rsc);
	pcmk__rsc_trace(rsc, "Resource %s's container is %s",
	rsc->id, container_id);
	} else {
	pcmk__config_err("Resource %s: Unknown resource container (%s)",
	rsc->id, container_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	/* Create remote nodes and guest nodes from the resource configuration
	* before unpacking resources.
	*/
	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	const char *new_node_id = NULL;

	/* Check for remote nodes, which are defined by ocf:pacemaker:remote
	* primitives.
	*/
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = ID(xml_obj);
	/* The "pe_find_node" check is here to make sure we don't iterate over
	* an expanded node that has already been added to the node list. */
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found remote node %s defined by resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes, which are defined by special meta-attributes
	* of a primitive of any type (for example, VirtualDomain or Xen).
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, pcmk__str_none)) {
	/* This will add an ocf:pacemaker:remote primitive to the
	* configuration for the guest node's connection, to be unpacked
	* later.
	*/
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources,
	scheduler);
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes inside a group. Clones are currently not
	* supported as guest nodes.
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, pcmk__str_none)) {
	xmlNode *xml_obj2 = NULL;
	for (xml_obj2 = pcmk__xe_first_child(xml_obj); xml_obj2 != NULL;
	xml_obj2 = pcmk__xe_next(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources,
	scheduler);

	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s inside group %s",
	new_node_id, ID(xml_obj2), ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	}
	}
	}
	return TRUE;
	}

	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the scheduler calculations.
	*/
	static void
	link_rsc2remotenode(pcmk_scheduler_t scheduler, pcmk_resource_t new_rsc)
	{
	pcmk_node_t *remote_node = NULL;

	if (new_rsc->is_remote_node == FALSE) {
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	remote_node = pe_find_node(scheduler->nodes, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return);

	pcmk__rsc_trace(new_rsc, "Linking remote connection resource %s to %s",
	new_rsc->id, pe__node_name(remote_node));
	remote_node->details->remote_rsc = new_rsc;

	if (new_rsc->container == NULL) {
	/* Handle start-up fencing for remote nodes (as opposed to guest nodes)
	* the same as is done for cluster nodes.
	*/
	handle_startup_fencing(scheduler, remote_node);

	} else {
	/* pe_create_node() marks the new node as "remote" or "cluster"; now
	* that we know the node is a guest node, update it correctly.
	*/
	g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("container"));
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	pcmk_tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	/*!
	* \internal
	* \brief Parse configuration XML for resource information
	*
	* \param[in] xml_resources Top of resource configuration XML
	* \param[in,out] scheduler Scheduler data
	*
	* \return TRUE
	*
	* \note unpack_remote_nodes() MUST be called before this, so that the nodes can
	* be used when pe__unpack_resource() calls resource_location()
	*/
	gboolean
	unpack_resources(const xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	GList *gIter = NULL;

	scheduler->template_rsc_sets = pcmk__strkey_table(free, destroy_tag);

	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	pcmk_resource_t *new_rsc = NULL;
	const char *id = ID(xml_obj);

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring <%s> resource without ID",
	xml_obj->name);
	continue;
	}

	if (pcmk__str_eq((const char *) xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE,
	pcmk__str_none)) {
	if (g_hash_table_lookup_extended(scheduler->template_rsc_sets, id,
	NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	g_hash_table_insert(scheduler->template_rsc_sets, strdup(id),
	NULL);
	}
	continue;
	}

	crm_trace("Unpacking <%s " XML_ATTR_ID "='%s'>",
	xml_obj->name, id);
	if (pe__unpack_resource(xml_obj, &new_rsc, NULL,
	scheduler) == pcmk_rc_ok) {
	scheduler->resources = g_list_append(scheduler->resources, new_rsc);
	pcmk__rsc_trace(new_rsc, "Added resource %s", new_rsc->id);

	} else {
	pcmk__config_err("Ignoring <%s> resource '%s' "
	"because configuration is invalid",
	xml_obj->name, id);
	}
	}

	for (gIter = scheduler->resources; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	setup_container(rsc, scheduler);
	link_rsc2remotenode(scheduler, rsc);
	}

	scheduler->resources = g_list_sort(scheduler->resources,
	pe__cmp_rsc_priority);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* Ignore */

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {

	pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
	pcmk__config_err("Either configure some or disable STONITH with the stonith-enabled option");
	pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode xml_tags, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_tag = NULL;

	scheduler->tags = pcmk__strkey_table(free, destroy_tag);

	for (xml_tag = pcmk__xe_first_child(xml_tags); xml_tag != NULL;
	xml_tag = pcmk__xe_next(xml_tag)) {

	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = ID(xml_tag);

	if (!pcmk__str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, pcmk__str_none)) {
	continue;
	}

	if (tag_id == NULL) {
	pcmk__config_err("Ignoring <%s> without " XML_ATTR_ID,
	(const char *) xml_tag->name);
	continue;
	}

	for (xml_obj_ref = pcmk__xe_first_child(xml_tag); xml_obj_ref != NULL;
	xml_obj_ref = pcmk__xe_next(xml_obj_ref)) {

	const char *obj_ref = ID(xml_obj_ref);

	if (!pcmk__str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, pcmk__str_none)) {
	continue;
	}

	if (obj_ref == NULL) {
	pcmk__config_err("Ignoring <%s> for tag '%s' without " XML_ATTR_ID,
	xml_obj_ref->name, tag_id);
	continue;
	}

	if (add_tag_ref(scheduler->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode xml_ticket, pcmk_scheduler_t scheduler)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	pcmk_ticket_t *ticket = NULL;

	ticket_id = ID(xml_ticket);
	if (pcmk__str_empty(ticket_id)) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, scheduler);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = pcmk__xml_attr_value(xIter);

	if (pcmk__str_eq(prop_name, XML_ATTR_ID, pcmk__str_none)) {
	continue;
	}
	g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
	}

	granted = g_hash_table_lookup(ticket->state, "granted");
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, "last-granted");
	if (last_granted) {
	long long last_granted_ll;

	pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
	ticket->last_granted = (time_t) last_granted_ll;
	}

	standby = g_hash_table_lookup(ticket->state, "standby");
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode xml_tickets, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_tickets); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (!pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, pcmk__str_none)) {
	continue;
	}
	unpack_ticket_state(xml_obj, scheduler);
	}

	return TRUE;
	}

	static void
	unpack_handle_remote_attrs(pcmk_node_t this_node, const xmlNode state,
	pcmk_scheduler_t *scheduler)
	{
	const char *resource_discovery_enabled = NULL;
	const xmlNode *attrs = NULL;
	pcmk_resource_t *rsc = NULL;

	if (!pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	return;
	}

	if ((this_node == NULL) \|\| !pe__is_guest_or_remote_node(this_node)) {
	return;
	}
	crm_trace("Processing Pacemaker Remote node %s", pe__node_name(this_node));

	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_MAINTENANCE),
	&(this_node->details->remote_maintenance), 0);

	rsc = this_node->details->remote_rsc;
	if (this_node->details->remote_requires_reset == FALSE) {
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	}
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, scheduler);

	if (pe__shutdown_requested(this_node)) {
	crm_info("%s is shutting down", pe__node_name(this_node));
	this_node->details->shutdown = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) \|\|
	((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed))) {
	crm_info("%s is in maintenance mode", pe__node_name(this_node));
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	if (pe__is_remote_node(this_node)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	- crm_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	- " attribute on Pacemaker Remote node %s"
	- " because fencing is disabled",
	- pe__node_name(this_node));
	+ pcmk__config_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	+ " attribute on Pacemaker Remote node %s"
	+ " because fencing is disabled",
	+ pe__node_name(this_node));
	} else {
	/* This is either a remote node with fencing enabled, or a guest
	* node. We don't care whether fencing is enabled when fencing guest
	* nodes, because they are "fenced" by recovering their containing
	* resource.
	*/
	crm_info("%s has resource discovery disabled",
	pe__node_name(this_node));
	this_node->details->rsc_discovery_enabled = FALSE;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack a cluster node's transient attributes
	*
	* \param[in] state CIB node state XML
	* \param[in,out] node Cluster node whose attributes are being unpacked
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_transient_attributes(const xmlNode state, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	const char *discovery = NULL;
	const xmlNode *attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS,
	FALSE);

	add_node_attrs(attrs, node, TRUE, scheduler);

	if (crm_is_true(pe_node_attribute_raw(node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(node));
	node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(node, "maintenance"))) {
	crm_info("%s is in maintenance mode", pe__node_name(node));
	node->details->maintenance = TRUE;
	}

	discovery = pe_node_attribute_raw(node, XML_NODE_ATTR_RSC_DISCOVERY);
	if ((discovery != NULL) && !crm_is_true(discovery)) {
	- crm_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	- " attribute for %s because disabling resource discovery "
	- "is not allowed for cluster nodes", pe__node_name(node));
	+ pcmk__config_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY " attribute "
	+ "for %s because disabling resource discovery is "
	+ "not allowed for cluster nodes", pe__node_name(node));
	}
	}

	/*!
	* \internal
	* \brief Unpack a node state entry (first pass)
	*
	* Unpack one node state entry from status. This unpacks information from the
	* node_state element itself and node attributes inside it, but not the
	* resource history inside it. Multiple passes through the status are needed to
	* fully unpack everything.
	*
	* \param[in] state CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_state(const xmlNode state, pcmk_scheduler_t scheduler)
	{
	const char *id = NULL;
	const char *uname = NULL;
	pcmk_node_t *this_node = NULL;

	id = crm_element_value(state, XML_ATTR_ID);
	if (id == NULL) {
	- crm_warn("Ignoring malformed " XML_CIB_TAG_STATE " entry without "
	- XML_ATTR_ID);
	+ pcmk__config_err("Ignoring invalid " XML_CIB_TAG_STATE " entry without "
	+ XML_ATTR_ID);
	+ crm_log_xml_info(state, "missing-id");
	return;
	}

	uname = crm_element_value(state, XML_ATTR_UNAME);
	if (uname == NULL) {
	/* If a joining peer makes the cluster acquire the quorum from corosync
	* meanwhile it has not joined CPG membership of pacemaker-controld yet,
	* it's possible that the created node_state entry doesn't have an uname
	* yet. We should recognize the node as `pending` and wait for it to
	* join CPG.
	*/
	crm_trace("Handling " XML_CIB_TAG_STATE " entry with id=\"%s\" without "
	XML_ATTR_UNAME, id);
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	pcmk__config_warn("Ignoring recorded node state for id=\"%s\" (%s) "
	"because it is no longer in the configuration",
	id, pcmk__s(uname, "uname unknown"));
	return;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	/* We can't determine the online status of Pacemaker Remote nodes until
	* after all resource history has been unpacked. In this first pass, we
	* do need to mark whether the node has been fenced, as this plays a
	* role during unpacking cluster node resource state.
	*/
	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_FENCED),
	&(this_node->details->remote_was_fenced), 0);
	return;
	}

	unpack_transient_attributes(state, this_node, scheduler);

	/* Provisionally mark this cluster node as clean. We have at least seen it
	* in the current cluster's lifetime.
	*/
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;

	crm_trace("Determining online status of cluster node %s (id %s)",
	pe__node_name(this_node), id);
	determine_online_status(state, this_node, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	&& this_node->details->online
	&& (scheduler->no_quorum_policy == pcmk_no_quorum_fence)) {
	/* Everything else should flow from this automatically
	* (at least until the scheduler becomes able to migrate off
	* healthy resources)
	*/
	pe_fence_node(scheduler, this_node, "cluster does not have quorum",
	FALSE);
	}
	}

	/*!
	* \internal
	* \brief Unpack nodes' resource history as much as possible
	*
	* Unpack as many nodes' resource history as possible in one pass through the
	* status. We need to process Pacemaker Remote nodes' connections/containers
	* before unpacking their history; the connection/container history will be
	* in another node's history, so it might take multiple passes to unpack
	* everything.
	*
	* \param[in] status CIB XML status section
	* \param[in] fence If true, treat any not-yet-unpacked nodes as unseen
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
	* or EAGAIN if more unpacking remains to be done)
	*/
	static int
	unpack_node_history(const xmlNode *status, bool fence,
	pcmk_scheduler_t *scheduler)
	{
	int rc = pcmk_rc_ok;

	// Loop through all node_state entries in CIB status
	for (const xmlNode *state = first_named_child(status, XML_CIB_TAG_STATE);
	state != NULL; state = crm_next_same_xml(state)) {

	const char *id = ID(state);
	const char *uname = crm_element_value(state, XML_ATTR_UNAME);
	pcmk_node_t *this_node = NULL;

	if ((id == NULL) \|\| (uname == NULL)) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history from malformed "
	XML_CIB_TAG_STATE " without id and/or uname");
	continue;
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history for node %s because "
	"no longer in configuration", id);
	continue;
	}

	if (this_node->details->unpacked) {
	crm_trace("Not unpacking resource history for node %s because "
	"already unpacked", id);
	continue;
	}

	if (fence) {
	// We're processing all remaining nodes

	} else if (pe__is_guest_node(this_node)) {
	/* We can unpack a guest node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection and containing resource are both up.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL) \|\| (rsc->role != pcmk_role_started)
	\|\| (rsc->container->role != pcmk_role_started)) {
	crm_trace("Not unpacking resource history for guest node %s "
	"because container and connection are not known to "
	"be up", id);
	continue;
	}

	} else if (pe__is_remote_node(this_node)) {
	/* We can unpack a remote node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection is up, with the exception of when shutdown locks are
	* in use.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL)
	\|\| (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)
	&& (rsc->role != pcmk_role_started))) {
	crm_trace("Not unpacking resource history for remote node %s "
	"because connection is not known to be up", id);
	continue;
	}

	/* If fencing and shutdown locks are disabled and we're not processing
	* unseen nodes, then we don't want to unpack offline nodes until online
	* nodes have been unpacked. This allows us to number active clone
	* instances first.
	*/
	} else if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_fencing_enabled
	\|pcmk_sched_shutdown_lock)
	&& !this_node->details->online) {
	crm_trace("Not unpacking resource history for offline "
	"cluster node %s", id);
	continue;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);
	unpack_handle_remote_attrs(this_node, state, scheduler);
	}

	crm_trace("Unpacking resource history for %snode %s",
	(fence? "unseen " : ""), id);

	this_node->details->unpacked = TRUE;
	unpack_node_lrm(this_node, state, scheduler);

	rc = EAGAIN; // Other node histories might depend on this one
	}
	return rc;
	}

	/* remove nodes that are down, stopping */
	/* create positive rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode status, pcmk_scheduler_t scheduler)
	{
	xmlNode *state = NULL;

	crm_trace("Beginning unpack");

	if (scheduler->tickets == NULL) {
	scheduler->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	for (state = pcmk__xe_first_child(status); state != NULL;
	state = pcmk__xe_next(state)) {

	if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
	unpack_tickets_state((xmlNode *) state, scheduler);

	} else if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	unpack_node_state(state, scheduler);
	}
	}

	while (unpack_node_history(status, FALSE, scheduler) == EAGAIN) {
	crm_trace("Another pass through node resource histories is needed");
	}

	// Now catch any nodes we didn't see
	unpack_node_history(status,
	pcmk_is_set(scheduler->flags,
	pcmk_sched_fencing_enabled),
	scheduler);

	/* Now that we know where resources are, we can schedule stops of containers
	* with failed bundle connections
	*/
	if (scheduler->stop_needed != NULL) {
	for (GList *item = scheduler->stop_needed; item; item = item->next) {
	pcmk_resource_t *container = item->data;
	pcmk_node_t *node = pe__current_node(container);

	if (node) {
	stop_action(container, node, FALSE);
	}
	}
	g_list_free(scheduler->stop_needed);
	scheduler->stop_needed = NULL;
	}

	/* Now that we know status of all Pacemaker Remote connections and nodes,
	* we can stop connections for node shutdowns, and check the online status
	* of remote/guest nodes that didn't have any node history to unpack.
	*/
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *this_node = gIter->data;

	if (!pe__is_guest_or_remote_node(this_node)) {
	continue;
	}
	if (this_node->details->shutdown
	&& (this_node->details->remote_rsc != NULL)) {
	pe__set_next_role(this_node->details->remote_rsc, pcmk_role_stopped,
	"remote shutdown");
	}
	if (!this_node->details->unpacked) {
	determine_remote_online_status(scheduler, this_node);
	}
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became a member at the cluster layer
	*
	* \param[in] node_state Node's node_state entry
	* \param[in,out] scheduler Scheduler data
	*
	* \return Epoch time when node became a cluster member
	* (or scheduler effective time for legacy entries) if a member,
	* 0 if not a member, or -1 if no valid information available
	*/
	static long long
	unpack_node_member(const xmlNode node_state, pcmk_scheduler_t scheduler)
	{
	const char *member_time = crm_element_value(node_state, PCMK__XA_IN_CCM);
	int member = 0;

	if (member_time == NULL) {
	return -1LL;

	} else if (crm_str_to_boolean(member_time, &member) == 1) {
	/* If in_ccm=0, we'll return 0 here. If in_ccm=1, either the entry was
	* recorded as a boolean for a DC < 2.1.7, or the node is pending
	* shutdown and has left the CPG, in which case it was set to 1 to avoid
	* fencing for node-pending-timeout.
	*
	* We return the effective time for in_ccm=1 because what's important to
	* avoid fencing is that effective time minus this value is less than
	* the pending node timeout.
	*/
	return member? (long long) get_effective_time(scheduler) : 0LL;

	} else {
	long long when_member = 0LL;

	if ((pcmk__scan_ll(member_time, &when_member,
	0LL) != pcmk_rc_ok) \|\| (when_member < 0LL)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_IN_CCM
	" in " XML_CIB_TAG_STATE " entry", member_time);
	return -1LL;
	}
	return when_member;
	}
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became online in process group
	*
	* \param[in] node_state Node's node_state entry
	*
	* \return Epoch time when node became online in process group (or 0 if not
	* online, or 1 for legacy online entries)
	*/
	static long long
	unpack_node_online(const xmlNode *node_state)
	{
	const char *peer_time = crm_element_value(node_state, PCMK__XA_CRMD);

	// @COMPAT Entries recorded for DCs < 2.1.7 have "online" or "offline"
	if (pcmk__str_eq(peer_time, OFFLINESTATUS,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	return 0LL;

	} else if (pcmk__str_eq(peer_time, ONLINESTATUS, pcmk__str_casei)) {
	return 1LL;

	} else {
	long long when_online = 0LL;

	if ((pcmk__scan_ll(peer_time, &when_online, 0LL) != pcmk_rc_ok)
	\|\| (when_online < 0)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_CRMD " in "
	XML_CIB_TAG_STATE " entry, assuming offline", peer_time);
	return 0LL;
	}
	return when_online;
	}
	}

	/*!
	* \internal
	* \brief Unpack node attribute for user-requested fencing
	*
	* \param[in] node Node to check
	* \param[in] node_state Node's node_state entry in CIB status
	*
	* \return \c true if fencing has been requested for \p node, otherwise \c false
	*/
	static bool
	unpack_node_terminate(const pcmk_node_t node, const xmlNode node_state)
	{
	long long value = 0LL;
	int value_i = 0;
	const char *value_s = pe_node_attribute_raw(node, PCMK_NODE_ATTR_TERMINATE);

	// Value may be boolean or an epoch time
	if (crm_str_to_boolean(value_s, &value_i) == 1) {
	return (value_i != 0);
	}
	if (pcmk__scan_ll(value_s, &value, 0LL) == pcmk_rc_ok) {
	return (value > 0);
	}
	crm_warn("Ignoring unrecognized value '%s' for " PCMK_NODE_ATTR_TERMINATE
	"node attribute for %s", value_s, pe__node_name(node));
	return false;
	}

	static gboolean
	determine_online_status_no_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	if (when_member <= 0) {
	crm_trace("Node %s is %sdown", pe__node_name(this_node),
	((when_member < 0)? "presumed " : ""));

	} else if (when_online > 0) {
	if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	online = TRUE;
	} else {
	crm_debug("Node %s is not ready to run resources: %s",
	pe__node_name(this_node), join);
	}

	} else if (this_node->details->expected_up == FALSE) {
	crm_trace("Node %s controller is down: "
	"member@%lld online@%lld join=%s expected=%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));

	} else {
	/* mark it unclean */
	pe_fence_node(scheduler, this_node, "peer is unexpectedly down", FALSE);
	crm_info("Node %s member@%lld online@%lld join=%s expected=%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
	}
	return online;
	}

	/*!
	* \internal
	* \brief Check whether a node has taken too long to join controller group
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node to check
	* \param[in] when_member Epoch time when node became a cluster member
	* \param[in] when_online Epoch time when node joined controller group
	*
	* \return true if node has been pending (on the way up) longer than
	* node-pending-timeout, otherwise false
	* \note This will also update the cluster's recheck time if appropriate.
	*/
	static inline bool
	pending_too_long(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	long long when_member, long long when_online)
	{
	if ((scheduler->node_pending_timeout > 0)
	&& (when_member > 0) && (when_online <= 0)) {
	// There is a timeout on pending nodes, and node is pending

	time_t timeout = when_member + scheduler->node_pending_timeout;

	if (get_effective_time(node->details->data_set) >= timeout) {
	return true; // Node has timed out
	}

	// Node is pending, but still has time
	pe__update_recheck_time(timeout, scheduler, "pending node timeout");
	}
	return false;
	}

	static bool
	determine_online_status_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	bool termination_requested = unpack_node_terminate(this_node, node_state);
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	/*
	- PCMK__XA_JOIN ::= member\|down\|pending\|banned
	- PCMK__XA_EXPECTED ::= member\|down

	@COMPAT with entries recorded for DCs < 2.1.7
	- PCMK__XA_IN_CCM ::= true\|false
	- PCMK__XA_CRMD ::= online\|offline

	Since crm_feature_set 3.18.0 (pacemaker-2.1.7):
	- PCMK__XA_IN_CCM ::= <timestamp>\|0
	Since when node has been a cluster member. A value 0 of means the node is not
	a cluster member.

	- PCMK__XA_CRMD ::= <timestamp>\|0
	Since when peer has been online in CPG. A value 0 means the peer is offline
	in CPG.
	*/

	crm_trace("Node %s member@%lld online@%lld join=%s expected=%s%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"),
	(termination_requested? " (termination requested)" : ""));

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", pe__node_name(this_node));

	/* Slightly different criteria since we can't shut down a dead peer */
	return (when_online > 0);
	}

	if (when_member < 0) {
	pe_fence_node(scheduler, this_node,
	"peer has not been seen by the cluster", FALSE);
	return false;
	}

	if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_none)) {
	pe_fence_node(scheduler, this_node,
	"peer failed Pacemaker membership criteria", FALSE);

	} else if (termination_requested) {
	if ((when_member <= 0) && (when_online <= 0)
	&& pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_none)) {
	crm_info("%s was fenced as requested", pe__node_name(this_node));
	return false;
	}
	pe_fence_node(scheduler, this_node, "fencing was requested", false);

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN,
	pcmk__str_null_matches)) {

	if (pending_too_long(scheduler, this_node, when_member, when_online)) {
	pe_fence_node(scheduler, this_node,
	"peer pending timed out on joining the process group",
	FALSE);

	} else if ((when_member > 0) \|\| (when_online > 0)) {
	crm_info("- %s is not ready to run resources",
	pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up",
	pe__node_name(this_node));
	}

	} else if (when_member <= 0) {
	// Consider `priority-fencing-delay` for lost nodes
	pe_fence_node(scheduler, this_node,
	"peer is no longer part of the cluster", TRUE);

	} else if (when_online <= 0) {
	pe_fence_node(scheduler, this_node,
	"peer process is no longer available", FALSE);

	/* Everything is running at this point, now check join state */

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_none)) {
	crm_info("%s is active", pe__node_name(this_node));

	} else if (pcmk__str_any_of(join, CRMD_JOINSTATE_PENDING,
	CRMD_JOINSTATE_DOWN, NULL)) {
	crm_info("%s is not ready to run resources", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(scheduler, this_node, "peer was in an unknown state",
	FALSE);
	}

	return (when_member > 0);
	}

	static void
	determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node)
	{
	pcmk_resource_t *rsc = this_node->details->remote_rsc;
	pcmk_resource_t *container = NULL;
	pcmk_node_t *host = NULL;

	/* If there is a node state entry for a (former) Pacemaker Remote node
	* but no resource creating that node, the node's connection resource will
	* be NULL. Consider it an offline remote node in that case.
	*/
	if (rsc == NULL) {
	this_node->details->online = FALSE;
	goto remote_online_done;
	}

	container = rsc->container;

	if (container && pcmk__list_of_1(rsc->running_on)) {
	host = rsc->running_on->data;
	}

	/* If the resource is currently started, mark it online. */
	if (rsc->role == pcmk_role_started) {
	crm_trace("%s node %s presumed ONLINE because connection resource is started",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if ((rsc->role == pcmk_role_started)
	&& (rsc->next_role == pcmk_role_stopped)) {

	crm_trace("%s node %s shutting down because connection resource is stopping",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if(container && pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	crm_trace("Guest node %s UNCLEAN because guest resource failed",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	crm_trace("%s node %s OFFLINE because connection resource failed",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;

	} else if ((rsc->role == pcmk_role_stopped)
	\|\| ((container != NULL)
	&& (container->role == pcmk_role_stopped))) {

	crm_trace("%s node %s OFFLINE because its resource is stopped",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = FALSE;

	} else if (host && (host->details->online == FALSE)
	&& host->details->unclean) {
	crm_trace("Guest node %s UNCLEAN because host is unclean",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;
	}

	remote_online_done:
	crm_trace("Remote node %s online=%s",
	this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
	}

	static void
	determine_online_status(const xmlNode node_state, pcmk_node_t this_node,
	pcmk_scheduler_t *scheduler)
	{
	gboolean online = FALSE;
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);

	CRM_CHECK(this_node != NULL, return);

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;

	if (pe__shutdown_requested(this_node)) {
	this_node->details->shutdown = TRUE;

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	this_node->details->expected_up = TRUE;
	}

	if (this_node->details->type == node_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	online = determine_online_status_no_fencing(scheduler, node_state,
	this_node);

	} else {
	online = determine_online_status_fencing(scheduler, node_state,
	this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* don't run resources here */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (this_node->details->type == node_ping) {
	crm_info("%s is not a Pacemaker node", pe__node_name(this_node));

	} else if (this_node->details->unclean) {
	pcmk__sched_warn("%s is unclean", pe__node_name(this_node));

	} else if (this_node->details->online) {
	crm_info("%s is %s", pe__node_name(this_node),
	this_node->details->shutdown ? "shutting down" :
	this_node->details->pending ? "pending" :
	this_node->details->standby ? "standby" :
	this_node->details->maintenance ? "maintenance" : "online");

	} else {
	crm_trace("%s is offline", pe__node_name(this_node));
	}
	}

	/*!
	* \internal
	* \brief Find the end of a resource's name, excluding any clone suffix
	*
	* \param[in] id Resource ID to check
	*
	* \return Pointer to last character of resource's base name
	*/
	const char *
	pe_base_name_end(const char *id)
	{
	if (!pcmk__str_empty(id)) {
	const char *end = id + strlen(id) - 1;

	for (const char *s = end; s > id; --s) {
	switch (*s) {
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	return (s == end)? s : (s - 1);
	default:
	return end;
	}
	}
	return end;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get a resource name excluding any clone suffix
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_strip(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	char *basename = NULL;

	CRM_ASSERT(end);
	basename = strndup(last_rsc_id, end - last_rsc_id + 1);
	CRM_ASSERT(basename);
	return basename;
	}

	/*!
	* \internal
	* \brief Get the name of the first instance of a cloned resource
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name plus :0
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_zero(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	size_t base_name_len = end - last_rsc_id + 1;
	char *zero = NULL;

	CRM_ASSERT(end);
	zero = calloc(base_name_len + 3, sizeof(char));
	CRM_ASSERT(zero);
	memcpy(zero, last_rsc_id, base_name_len);
	zero[base_name_len] = ':';
	zero[base_name_len + 1] = '0';
	return zero;
	}

	static pcmk_resource_t *
	create_fake_resource(const char rsc_id, const xmlNode rsc_entry,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (pe__unpack_resource(xml_rsc, &rsc, NULL, scheduler) != pcmk_rc_ok) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	pcmk_node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	node = pe_find_node(scheduler->nodes, rsc_id);
	if (node == NULL) {
	node = pe_create_node(rsc_id, rsc_id, "remote", NULL, scheduler);
	}
	link_rsc2remotenode(scheduler, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
	/* This orphaned rsc needs to be mapped to a container. */
	crm_trace("Detected orphaned container filler %s", rsc_id);
	pe__set_resource_flags(rsc, pcmk_rsc_removed_filler);
	}
	pe__set_resource_flags(rsc, pcmk_rsc_removed);
	scheduler->resources = g_list_append(scheduler->resources, rsc);
	return rsc;
	}

	/*!
	* \internal
	* \brief Create orphan instance for anonymous clone resource history
	*
	* \param[in,out] parent Clone resource that orphan will be added to
	* \param[in] rsc_id Orphan's resource ID
	* \param[in] node Where orphan is active (for logging only)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly added orphaned instance of \p parent
	*/
	static pcmk_resource_t *
	create_anonymous_orphan(pcmk_resource_t parent, const char rsc_id,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *top = pe__create_clone_child(parent, scheduler);

	// find_rsc() because we might be a cloned group
	pcmk_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	pcmk__rsc_debug(parent, "Created orphan %s for %s: %s on %s",
	top->id, parent->id, rsc_id, pe__node_name(node));
	return orphan;
	}

	/*!
	* \internal
	* \brief Check a node for an instance of an anonymous clone
	*
	* Return a child instance of the specified anonymous clone, in order of
	* preference: (1) the instance running on the specified node, if any;
	* (2) an inactive instance (i.e. within the total of clone-max instances);
	* (3) a newly created orphan (i.e. clone-max instances are already active).
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node on which to check for instance
	* \param[in,out] parent Clone to check
	* \param[in] rsc_id Name of cloned resource in history (no instance)
	*/
	static pcmk_resource_t *
	find_anonymous_clone(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	pcmk_resource_t parent, const char rsc_id)
	{
	GList *rIter = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *inactive_instance = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(pe_rsc_is_clone(parent));
	CRM_ASSERT(!pcmk_is_set(parent->flags, pcmk_rsc_unique));

	// Check for active (or partially active, for cloned groups) instance
	pcmk__rsc_trace(parent, "Looking for %s on %s in %s",
	rsc_id, pe__node_name(node), parent->id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GList *locations = NULL;
	pcmk_resource_t *child = rIter->data;

	/* Check whether this instance is already known to be active or pending
	* anywhere, at this stage of unpacking. Because this function is called
	* for a resource before the resource's individual operation history
	* entries are unpacked, locations will generally not contain the
	* desired node.
	*
	* However, there are three exceptions:
	* (1) when child is a cloned group and we have already unpacked the
	* history of another member of the group on the same node;
	* (2) when we've already unpacked the history of another numbered
	* instance on the same node (which can happen if globally-unique
	* was flipped from true to false); and
	* (3) when we re-run calculations on the same scheduler data as part of
	* a simulation.
	*/
	child->fns->location(child, &locations, 2);
	if (locations) {
	/* We should never associate the same numbered anonymous clone
	* instance with multiple nodes, and clone instances can't migrate,
	* so there must be only one location, regardless of history.
	*/
	CRM_LOG_ASSERT(locations->next == NULL);

	if (((pcmk_node_t *) locations->data)->details == node->details) {
	/* This child instance is active on the requested node, so check
	* for a corresponding configured resource. We use find_rsc()
	* instead of child because child may be a cloned group, and we
	* need the particular member corresponding to rsc_id.
	*
	* If the history entry is orphaned, rsc will be NULL.
	*/
	rsc = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);
	if (rsc) {
	/* If there are multiple instance history entries for an
	* anonymous clone in a single node's history (which can
	* happen if globally-unique is switched from true to
	* false), we want to consider the instances beyond the
	* first as orphans, even if there are inactive instance
	* numbers available.
	*/
	if (rsc->running_on) {
	crm_notice("Active (now-)anonymous clone %s has "
	"multiple (orphan) instance histories on %s",
	parent->id, pe__node_name(node));
	skip_inactive = TRUE;
	rsc = NULL;
	} else {
	pcmk__rsc_trace(parent, "Resource %s, active", rsc->id);
	}
	}
	}
	g_list_free(locations);

	} else {
	pcmk__rsc_trace(parent, "Resource %s, skip inactive", child->id);
	if (!skip_inactive && !inactive_instance
	&& !pcmk_is_set(child->flags, pcmk_rsc_blocked)) {
	// Remember one inactive instance in case we don't find active
	inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	/* ... but don't use it if it was already associated with a
	* pending action on another node
	*/
	if (inactive_instance && inactive_instance->pending_node
	&& (inactive_instance->pending_node->details != node->details)) {
	inactive_instance = NULL;
	}
	}
	}
	}

	if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
	pcmk__rsc_trace(parent, "Resource %s, empty slot",
	inactive_instance->id);
	rsc = inactive_instance;
	}

	/* If the resource has "requires" set to "quorum" or "nothing", and we don't
	* have a clone instance for every node, we don't want to consume a valid
	* instance number for unclean nodes. Such instances may appear to be active
	* according to the history, but should be considered inactive, so we can
	* start an instance elsewhere. Treat such instances as orphans.
	*
	* An exception is instances running on guest nodes -- since guest node
	* "fencing" is actually just a resource stop, requires shouldn't apply.
	*
	* @TODO Ideally, we'd use an inactive instance number if it is not needed
	* for any clean instances. However, we don't know that at this point.
	*/
	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)
	&& (!node->details->online \|\| node->details->unclean)
	&& !pe__is_guest_node(node)
	&& !pe__is_universal_clone(parent, scheduler)) {

	rsc = NULL;
	}

	if (rsc == NULL) {
	rsc = create_anonymous_orphan(parent, rsc_id, node, scheduler);
	pcmk__rsc_trace(parent, "Resource %s, orphan", rsc->id);
	}
	return rsc;
	}

	static pcmk_resource_t *
	unpack_find_resource(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	const char *rsc_id)
	{
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	/* If we didn't find the resource by its name in the operation history,
	* check it again as a clone instance. Even when clone-max=0, we create
	* a single :0 orphan to match against here.
	*/
	char *clone0_id = clone_zero(rsc_id);
	pcmk_resource_t *clone0 = pe_find_resource(scheduler->resources,
	clone0_id);

	if (clone0 && !pcmk_is_set(clone0->flags, pcmk_rsc_unique)) {
	rsc = clone0;
	parent = uber_parent(clone0);
	crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
	} else {
	crm_trace("%s is not known as %s either (orphan)",
	rsc_id, clone0_id);
	}
	free(clone0_id);

	} else if (rsc->variant > pcmk_rsc_variant_primitive) {
	crm_trace("Resource history for %s is orphaned because it is no longer primitive",
	rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (pe_rsc_is_anon_clone(parent)) {

	if (pe_rsc_is_bundled(parent)) {
	rsc = pe__find_bundle_replica(parent->parent, node);
	} else {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(scheduler, node, parent, base);
	free(base);
	CRM_ASSERT(rsc != NULL);
	}
	}

	if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_casei)
	&& !pcmk__str_eq(rsc_id, rsc->clone_name, pcmk__str_casei)) {

	pcmk__str_update(&rsc->clone_name, rsc_id);
	pcmk__rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, pe__node_name(node), rsc->id,
	pcmk_is_set(rsc->flags, pcmk_rsc_removed)? " (ORPHAN)" : "");
	}
	return rsc;
	}

	static pcmk_resource_t *
	process_orphan_resource(const xmlNode rsc_entry, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	crm_debug("Detected orphan resource %s on %s", rsc_id, pe__node_name(node));
	rsc = create_fake_resource(rsc_id, rsc_entry, scheduler);
	if (rsc == NULL) {
	return NULL;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);

	} else {
	CRM_CHECK(rsc != NULL, return NULL);
	pcmk__rsc_trace(rsc, "Added orphan %s", rsc->id);
	resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__",
	scheduler);
	}
	return rsc;
	}

	static void
	process_rsc_state(pcmk_resource_t rsc, pcmk_node_t node,
	enum action_fail_response on_fail)
	{
	pcmk_node_t *tmpnode = NULL;
	char *reason = NULL;
	enum action_fail_response save_on_fail = pcmk_on_fail_ignore;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role), pe__node_name(node),
	fail2text(on_fail));

	/* process current state */
	if (rsc->role != pcmk_role_unknown) {
	pcmk_resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
	pcmk_node_t *n = pe__copy_node(node);

	pcmk__rsc_trace(rsc, "%s%s%s known on %s",
	rsc->id,
	((rsc->clone_name == NULL)? "" : " also known as "),
	((rsc->clone_name == NULL)? "" : rsc->clone_name),
	pe__node_name(n));
	g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
	}
	if (pcmk_is_set(iter->flags, pcmk_rsc_unique)) {
	break;
	}
	iter = iter->parent;
	}
	}

	/* If a managed resource is believed to be running, but node is down ... */
	if ((rsc->role > pcmk_role_stopped)
	&& node->details->online == FALSE
	&& node->details->maintenance == FALSE
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {

	gboolean should_fence = FALSE;

	/* If this is a guest node, fence it (regardless of whether fencing is
	* enabled, because guest node fencing is done by recovery of the
	* container resource rather than by the fencer). Mark the resource
	* we're processing as failed. When the guest comes back up, its
	* operation history in the CIB will be cleared, freeing the affected
	* resource to run again once we are sure we know its state.
	*/
	if (pe__is_guest_node(node)) {
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	should_fence = TRUE;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& !pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_failed)) {

	/* Setting unseen means that fencing of the remote node will
	* occur only if the connection resource is not going to start
	* somewhere. This allows connection resources on a failed
	* cluster node to move to another node without requiring the
	* remote nodes to be fenced as well.
	*/
	node->details->unseen = TRUE;
	reason = crm_strdup_printf("%s is active there (fencing will be"
	" revoked if remote connection can "
	"be re-established elsewhere)",
	rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
	}
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	}
	free(reason);
	}

	/* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
	save_on_fail = on_fail;

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = pcmk_on_fail_ignore;
	}

	switch (on_fail) {
	case pcmk_on_fail_ignore:
	/* nothing to do */
	break;

	case pcmk_on_fail_demote:
	pe__set_resource_flags(rsc, pcmk_rsc_failed);
	demote_action(rsc, node, FALSE);
	break;

	case pcmk_on_fail_fence_node:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	reason = crm_strdup_printf("%s failed there", rsc->id);
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	free(reason);
	break;

	case pcmk_on_fail_standby_node:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case pcmk_on_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);
	pe__set_resource_flags(rsc, pcmk_rsc_blocked);
	break;

	case pcmk_on_fail_ban:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY, "__action_migration_auto__",
	rsc->cluster);
	break;

	case pcmk_on_fail_stop:
	pe__set_next_role(rsc, pcmk_role_stopped, "on-fail=stop");
	break;

	case pcmk_on_fail_restart:
	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_restart_container:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (rsc->container && pe_rsc_is_bundled(rsc)) {
	/* A bundle's remote connection can run on a different node than
	* the bundle's container. We don't necessarily know where the
	* container is running yet, so remember it and add a stop
	* action for it later.
	*/
	rsc->cluster->stop_needed =
	g_list_prepend(rsc->cluster->stop_needed, rsc->container);
	} else if (rsc->container) {
	stop_action(rsc->container, node, FALSE);
	} else if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_reset_remote:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	tmpnode = NULL;
	if (rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	}
	if (tmpnode &&
	pe__is_remote_node(tmpnode) &&
	tmpnode->details->remote_was_fenced == 0) {

	/* The remote connection resource failed in a way that
	* should result in fencing the remote node.
	*/
	pe_fence_node(rsc->cluster, tmpnode,
	"remote connection is unrecoverable", FALSE);
	}
	}

	/* require the stop action regardless if fencing is occurring or not. */
	if (rsc->role > pcmk_role_stopped) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->remote_reconnect_ms) {
	pe__set_next_role(rsc, pcmk_role_stopped, "remote reset");
	}
	break;
	}

	/* ensure a remote-node connection failure forces an unclean remote-node
	* to be fenced. By setting unseen = FALSE, the remote-node failure will
	* result in a fencing operation regardless if we're going to attempt to
	* reconnect to the remote-node in this transition or not. */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed) && rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	tmpnode->details->unseen = FALSE;
	}
	}

	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__config_warn("Detected active orphan %s running on %s",
	rsc->id, pe__node_name(node));
	} else {
	pcmk__config_warn("Resource '%s' must be stopped manually on "
	"%s because cluster is configured not to "
	"stop active orphans",
	rsc->id, pe__node_name(node));
	}
	}

	native_add_running(rsc, node, rsc->cluster,
	(save_on_fail != pcmk_on_fail_ignore));
	switch (on_fail) {
	case pcmk_on_fail_ignore:
	break;
	case pcmk_on_fail_demote:
	case pcmk_on_fail_block:
	pe__set_resource_flags(rsc, pcmk_rsc_failed);
	break;
	default:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	break;
	}

	} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pcmk__rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)",
	rsc->clone_name, rsc->id);
	free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	GList *possible_matches = pe__resource_actions(rsc, node,
	PCMK_ACTION_STOP, FALSE);
	GList *gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t stop = (pcmk_action_t ) gIter->data;

	pe__set_action_flags(stop, pcmk_action_optional);
	}

	g_list_free(possible_matches);
	}

	/* A successful stop after migrate_to on the migration source doesn't make
	* the partially migrated resource stopped on the migration target.
	*/
	if ((rsc->role == pcmk_role_stopped)
	&& rsc->partial_migration_source
	&& rsc->partial_migration_source->details == node->details
	&& rsc->partial_migration_target
	&& rsc->running_on) {

	rsc->role = pcmk_role_started;
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(pcmk_node_t node, pcmk_resource_t rsc,
	int start_index, int stop_index,
	GList sorted_op_list, pcmk_scheduler_t scheduler)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GList *gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "%s: Start index %d, stop index = %d",
	rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	guint interval_ms = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);

	counter++;

	if (node->details->online == FALSE) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: node is offline",
	rsc->id, pe__node_name(node));
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: resource is not active",
	id, pe__node_name(node));
	continue;

	} else if (counter < start_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: old %d",
	id, pe__node_name(node), counter);
	continue;
	}

	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	if (interval_ms == 0) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: non-recurring",
	id, pe__node_name(node));
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: status",
	id, pe__node_name(node));
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = pcmk__op_key(rsc->id, task, interval_ms);
	pcmk__rsc_trace(rsc, "Creating %s on %s", key, pe__node_name(node));
	custom_action(rsc, key, task, node, TRUE, scheduler);
	}
	}

	void
	calculate_active_ops(const GList sorted_op_list, int start_index,
	int *stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_clone_start = -1;
	const char *task = NULL;
	const char *status = NULL;

	*stop_index = -1;
	*start_index = -1;

	for (const GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	const xmlNode rsc_op = (const xmlNode ) iter->data;

	counter++;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& pcmk__str_eq(status, "0", pcmk__str_casei)) {
	*stop_index = counter;

	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_START,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR,
	pcmk__str_casei)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);

	if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
	implied_monitor_start = counter;
	}
	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_DEMOTE, NULL)) {
	implied_clone_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_clone_start != -1) {
	*start_index = implied_clone_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	// If resource history entry has shutdown lock, remember lock node and time
	static void
	unpack_shutdown_lock(const xmlNode rsc_entry, pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	time_t lock_time = 0; // When lock started (i.e. node shutdown time)

	if ((crm_element_value_epoch(rsc_entry, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	&lock_time) == pcmk_ok) && (lock_time != 0)) {

	if ((scheduler->shutdown_lock > 0)
	&& (get_effective_time(scheduler)
	> (lock_time + scheduler->shutdown_lock))) {
	pcmk__rsc_info(rsc, "Shutdown lock for %s on %s expired",
	rsc->id, pe__node_name(node));
	pe__clear_resource_history(rsc, node);
	} else {
	/* @COMPAT I don't like breaking const signatures, but
	* rsc->lock_node should really be const -- we just can't change it
	* until the next API compatibility break.
	*/
	rsc->lock_node = (pcmk_node_t *) node;
	rsc->lock_time = lock_time;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack one lrm_resource entry from a node's CIB status
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] rsc_entry lrm_resource XML being unpacked
	* \param[in,out] scheduler Scheduler data
	*
	* \return Resource corresponding to the entry, or NULL if no operation history
	*/
	static pcmk_resource_t *
	unpack_lrm_resource(pcmk_node_t node, const xmlNode lrm_resource,
	pcmk_scheduler_t *scheduler)
	{
	GList *gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = pcmk_role_unknown;

	const char *rsc_id = ID(lrm_resource);

	pcmk_resource_t *rsc = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	xmlNode *rsc_op = NULL;
	xmlNode *last_failure = NULL;

	enum action_fail_response on_fail = pcmk_on_fail_ignore;
	enum rsc_role_e saved_role = pcmk_role_unknown;

	if (rsc_id == NULL) {
	- crm_warn("Ignoring malformed " XML_LRM_TAG_RESOURCE
	- " entry without id");
	+ pcmk__config_err("Ignoring invalid " XML_LRM_TAG_RESOURCE
	+ " entry: No " XML_ATTR_ID);
	+ crm_log_xml_info(lrm_resource, "missing-id");
	return NULL;
	}
	crm_trace("Unpacking " XML_LRM_TAG_RESOURCE " for %s on %s",
	rsc_id, pe__node_name(node));

	// Build a list of individual lrm_rsc_op entries, so we can sort them
	for (rsc_op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {

	op_list = g_list_prepend(op_list, rsc_op);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	}
	}

	/* find the resource */
	rsc = unpack_find_resource(scheduler, node, rsc_id);
	if (rsc == NULL) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	} else {
	rsc = process_orphan_resource(lrm_resource, node, scheduler);
	}
	}
	CRM_ASSERT(rsc != NULL);

	// Check whether the resource is "shutdown-locked" to this node
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	unpack_shutdown_lock(lrm_resource, rsc, node, scheduler);
	}

	/* process operations */
	saved_role = rsc->role;
	rsc->role = pcmk_role_unknown;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list,
	scheduler);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail);

	if (get_target_role(rsc, &req_role)) {
	if ((rsc->next_role == pcmk_role_unknown)
	\|\| (req_role < rsc->next_role)) {

	pe__set_next_role(rsc, req_role, XML_RSC_ATTR_TARGET_ROLE);

	} else if (req_role > rsc->next_role) {
	pcmk__rsc_info(rsc,
	"%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role),
	role2text(req_role));
	}
	}

	if (saved_role > rsc->role) {
	rsc->role = saved_role;
	}

	return rsc;
	}

	static void
	handle_orphaned_container_fillers(const xmlNode *lrm_rsc_list,
	pcmk_scheduler_t *scheduler)
	{
	for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rsc_list);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next(rsc_entry)) {

	pcmk_resource_t *rsc;
	pcmk_resource_t *container;
	const char *rsc_id;
	const char *container_id;

	if (!pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_casei)) {
	continue;
	}

	container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	if (container_id == NULL \|\| rsc_id == NULL) {
	continue;
	}

	container = pe_find_resource(scheduler->resources, container_id);
	if (container == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources, rsc_id);
	if ((rsc == NULL) \|\| (rsc->container != NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	continue;
	}

	pcmk__rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
	rsc->id, container_id);
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	}
	}

	/*!
	* \internal
	* \brief Unpack one node's lrm status section
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] xml CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler)
	{
	bool found_orphaned_container_filler = false;

	// Drill down to lrm_resources section
	xml = find_xml_node(xml, XML_CIB_TAG_LRM, FALSE);
	if (xml == NULL) {
	return;
	}
	xml = find_xml_node(xml, XML_LRM_TAG_RESOURCES, FALSE);
	if (xml == NULL) {
	return;
	}

	// Unpack each lrm_resource entry
	for (const xmlNode *rsc_entry = first_named_child(xml, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	pcmk_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, scheduler);

	if ((rsc != NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	found_orphaned_container_filler = true;
	}
	}

	/* Now that all resource state has been unpacked for this node, map any
	* orphaned container fillers to their container resource.
	*/
	if (found_orphaned_container_filler) {
	handle_orphaned_container_fillers(xml, scheduler);
	}
	}

	static void
	set_active(pcmk_resource_t *rsc)
	{
	const pcmk_resource_t *top = pe__const_top_resource(rsc, false);

	if (top && pcmk_is_set(top->flags, pcmk_rsc_promotable)) {
	rsc->role = pcmk_role_unpromoted;
	} else {
	rsc->role = pcmk_role_started;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk_node_t *node = value;
	int *score = user_data;

	node->weight = *score;
	}

	#define XPATH_NODE_STATE "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE
	#define SUB_XPATH_LRM_RESOURCE "/" XML_CIB_TAG_LRM \
	"/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE
	#define SUB_XPATH_LRM_RSC_OP "/" XML_LRM_TAG_RSC_OP

	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	int target_rc, pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((resource != NULL) && (op != NULL) && (node != NULL),
	return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node, "']"
	SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", resource, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_TASK "='", op, "'",
	NULL);

	/* Need to check against transition_magic too? */
	if ((source != NULL) && (strcmp(op, PCMK_ACTION_MIGRATE_TO) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_TARGET "='", source, "']",
	NULL);

	} else if ((source != NULL)
	&& (strcmp(op, PCMK_ACTION_MIGRATE_FROM) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_SOURCE "='", source, "']",
	NULL);
	} else {
	g_string_append_c(xpath, ']');
	}

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);
	g_string_free(xpath, TRUE);

	if (xml && target_rc >= 0) {
	int rc = PCMK_OCF_UNKNOWN_ERROR;
	int status = PCMK_EXEC_ERROR;

	crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
	crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
	if ((rc != target_rc) \|\| (status != PCMK_EXEC_DONE)) {
	return NULL;
	}
	}
	return xml;
	}

	static xmlNode *
	find_lrm_resource(const char rsc_id, const char node_name,
	pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((rsc_id != NULL) && (node_name != NULL), return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", rsc_id, "']",
	NULL);

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);

	g_string_free(xpath, TRUE);
	return xml;
	}

	/*!
	* \internal
	* \brief Check whether a resource has no completed action history on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node_name Node to check
	*
	* \return true if \p rsc_id is unknown on \p node_name, otherwise false
	*/
	static bool
	unknown_on_node(pcmk_resource_t rsc, const char node_name)
	{
	bool result = false;
	xmlXPathObjectPtr search;
	GString *xpath = g_string_sized_new(256);

	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", rsc->id, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_RC "!='193']",
	NULL);
	search = xpath_search(rsc->cluster->input, (const char *) xpath->str);
	result = (numXpathResults(search) == 0);
	freeXpathObject(search);
	g_string_free(xpath, TRUE);
	return result;
	}

	/*!
	* \brief Check whether a probe/monitor indicating the resource was not running
	* on a node happened after some event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a monitor happened after event, false otherwise
	*/
	static bool
	monitor_not_running_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	/* Any probe/monitor operation on the node indicating it was not running
	* there
	*/
	xmlNode *monitor = find_lrm_op(rsc_id, PCMK_ACTION_MONITOR, node_name,
	NULL, PCMK_OCF_NOT_RUNNING, scheduler);

	return (monitor && pe__is_newer_op(monitor, xml_op, same_node) > 0);
	}

	/*!
	* \brief Check whether any non-monitor operation on a node happened after some
	* event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that non-monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	non_monitor_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *lrm_resource = NULL;

	lrm_resource = find_lrm_resource(rsc_id, node_name, scheduler);
	if (lrm_resource == NULL) {
	return false;
	}

	for (xmlNode *op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	op != NULL; op = crm_next_same_xml(op)) {
	const char * task = NULL;

	if (op == xml_op) {
	continue;
	}

	task = crm_element_value(op, XML_LRM_ATTR_TASK);

	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	NULL)
	&& pe__is_newer_op(op, xml_op, same_node) > 0) {
	return true;
	}
	}

	return false;
	}

	/*!
	* \brief Check whether the resource has newer state on a node after a migration
	* attempt
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] migrate_to Any migrate_to event that is being compared to
	* \param[in] migrate_from Any migrate_from event that is being compared to
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	newer_state_after_migrate(const char rsc_id, const char node_name,
	const xmlNode *migrate_to,
	const xmlNode *migrate_from,
	pcmk_scheduler_t *scheduler)
	{
	const xmlNode *xml_op = migrate_to;
	const char *source = NULL;
	const char *target = NULL;
	bool same_node = false;

	if (migrate_from) {
	xml_op = migrate_from;
	}

	source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	/* It's preferred to compare to the migrate event on the same node if
	* existing, since call ids are more reliable.
	*/
	if (pcmk__str_eq(node_name, target, pcmk__str_casei)) {
	if (migrate_from) {
	xml_op = migrate_from;
	same_node = true;

	} else {
	xml_op = migrate_to;
	}

	} else if (pcmk__str_eq(node_name, source, pcmk__str_casei)) {
	if (migrate_to) {
	xml_op = migrate_to;
	same_node = true;

	} else {
	xml_op = migrate_from;
	}
	}

	/* If there's any newer non-monitor operation on the node, or any newer
	* probe/monitor operation on the node indicating it was not running there,
	* the migration events potentially no longer matter for the node.
	*/
	return non_monitor_after(rsc_id, node_name, xml_op, same_node, scheduler)
	\|\| monitor_not_running_after(rsc_id, node_name, xml_op, same_node,
	scheduler);
	}

	/*!
	* \internal
	* \brief Parse migration source and target node names from history entry
	*
	* \param[in] entry Resource history entry for a migration action
	* \param[in] source_node If not NULL, source must match this node
	* \param[in] target_node If not NULL, target must match this node
	* \param[out] source_name Where to store migration source node name
	* \param[out] target_name Where to store migration target node name
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_migration_node_names(const xmlNode entry, const pcmk_node_t source_node,
	const pcmk_node_t *target_node,
	const char source_name, const char target_name)
	{
	*source_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_SOURCE);
	*target_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_TARGET);
	if ((source_name == NULL) \|\| (target_name == NULL)) {
	- crm_err("Ignoring resource history entry %s without "
	- XML_LRM_ATTR_MIGRATE_SOURCE " and " XML_LRM_ATTR_MIGRATE_TARGET,
	- ID(entry));
	+ pcmk__config_err("Ignoring resource history entry %s without "
	+ XML_LRM_ATTR_MIGRATE_SOURCE " and "
	+ XML_LRM_ATTR_MIGRATE_TARGET, ID(entry));
	return pcmk_rc_unpack_error;
	}

	if ((source_node != NULL)
	&& !pcmk__str_eq(*source_name, source_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	- crm_err("Ignoring resource history entry %s because "
	- XML_LRM_ATTR_MIGRATE_SOURCE "='%s' does not match %s",
	- ID(entry), *source_name, pe__node_name(source_node));
	+ pcmk__config_err("Ignoring resource history entry %s because "
	+ XML_LRM_ATTR_MIGRATE_SOURCE "='%s' does not match %s",
	+ ID(entry), *source_name, pe__node_name(source_node));
	return pcmk_rc_unpack_error;
	}

	if ((target_node != NULL)
	&& !pcmk__str_eq(*target_name, target_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	- crm_err("Ignoring resource history entry %s because "
	- XML_LRM_ATTR_MIGRATE_TARGET "='%s' does not match %s",
	- ID(entry), *target_name, pe__node_name(target_node));
	+ pcmk__config_err("Ignoring resource history entry %s because "
	+ XML_LRM_ATTR_MIGRATE_TARGET "='%s' does not match %s",
	+ ID(entry), *target_name, pe__node_name(target_node));
	return pcmk_rc_unpack_error;
	}

	return pcmk_rc_ok;
	}

	/*
	* \internal
	* \brief Add a migration source to a resource's list of dangling migrations
	*
	* If the migrate_to and migrate_from actions in a live migration both
	* succeeded, but there is no stop on the source, the migration is considered
	* "dangling." Add the source to the resource's dangling migration list, which
	* will be used to schedule a stop on the source without affecting the target.
	*
	* \param[in,out] rsc Resource involved in migration
	* \param[in] node Migration source
	*/
	static void
	add_dangling_migration(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk__rsc_trace(rsc, "Dangling migration of %s requires stop on %s",
	rsc->id, pe__node_name(node));
	rsc->role = pcmk_role_stopped;
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations,
	(gpointer) node);
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a successful migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_success(struct action_history *history)
	{
	/* A complete migration sequence is:
	* 1. migrate_to on source node (which succeeded if we get to this function)
	* 2. migrate_from on target node
	* 3. stop on source node
	*
	* If no migrate_from has happened, the migration is considered to be
	* "partial". If the migrate_from succeeded but no stop has happened, the
	* migration is considered to be "dangling".
	*
	* If a successful migrate_to and stop have happened on the source node, we
	* still need to check for a partial migration, due to scenarios (easier to
	* produce with batch-limit=1) like:
	*
	* - A resource is migrating from node1 to node2, and a migrate_to is
	* initiated for it on node1.
	*
	* - node2 goes into standby mode while the migrate_to is pending, which
	* aborts the transition.
	*
	* - Upon completion of the migrate_to, a new transition schedules a stop
	* on both nodes and a start on node1.
	*
	* - If the new transition is aborted for any reason while the resource is
	* stopping on node1, the transition after that stop completes will see
	* the migrate_to and stop on the source, but it's still a partial
	* migration, and the resource must be stopped on node2 because it is
	* potentially active there due to the migrate_to.
	*
	* We also need to take into account that either node's history may be
	* cleared at any point in the migration process.
	*/
	int from_rc = PCMK_OCF_OK;
	int from_status = PCMK_EXEC_PENDING;
	pcmk_node_t *target_node = NULL;
	xmlNode *migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;
	bool source_newer_op = false;
	bool target_newer_state = false;
	bool active_on_target = false;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	// Check for newer state on the source
	source_newer_op = non_monitor_after(history->rsc->id, source, history->xml,
	true, history->rsc->cluster);

	// Check for a migrate_from action from this source on the target
	migrate_from = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_FROM,
	target, source, -1, history->rsc->cluster);
	if (migrate_from != NULL) {
	if (source_newer_op) {
	/* There's a newer non-monitor operation on the source and a
	* migrate_from on the target, so this migrate_to is irrelevant to
	* the resource's state.
	*/
	return;
	}
	crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
	crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS,
	&from_status);
	}

	/* If the resource has newer state on both the source and target after the
	* migration events, this migrate_to is irrelevant to the resource's state.
	*/
	target_newer_state = newer_state_after_migrate(history->rsc->id, target,
	history->xml, migrate_from,
	history->rsc->cluster);
	if (source_newer_op && target_newer_state) {
	return;
	}

	/* Check for dangling migration (migrate_from succeeded but stop not done).
	* We know there's no stop because we already returned if the target has a
	* migrate_from and the source has any newer non-monitor operation.
	*/
	if ((from_rc == PCMK_OCF_OK) && (from_status == PCMK_EXEC_DONE)) {
	add_dangling_migration(history->rsc, history->node);
	return;
	}

	/* Without newer state, this migrate_to implies the resource is active.
	* (Clones are not allowed to migrate, so role can't be promoted.)
	*/
	history->rsc->role = pcmk_role_started;

	target_node = pe_find_node(history->rsc->cluster->nodes, target);
	active_on_target = !target_newer_state && (target_node != NULL)
	&& target_node->details->online;

	if (from_status != PCMK_EXEC_PENDING) { // migrate_from failed on target
	if (active_on_target) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	TRUE);
	} else {
	// Mark resource as failed, require recovery, and prevent migration
	pe__set_resource_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pe__clear_resource_flags(history->rsc, pcmk_rsc_migratable);
	}
	return;
	}

	// The migrate_from is pending, complete but erased, or to be scheduled

	/* If there is no history at all for the resource on an online target, then
	* it was likely cleaned. Just return, and we'll schedule a probe. Once we
	* have the probe result, it will be reflected in target_newer_state.
	*/
	if ((target_node != NULL) && target_node->details->online
	&& unknown_on_node(history->rsc, target)) {
	return;
	}

	if (active_on_target) {
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	if ((source_node != NULL) && source_node->details->online) {
	/* This is a partial migration: the migrate_to completed
	* successfully on the source, but the migrate_from has not
	* completed. Remember the source and target; if the newly
	* chosen target remains the same when we schedule actions
	* later, we may continue with the migration.
	*/
	history->rsc->partial_migration_target = target_node;
	history->rsc->partial_migration_source = source_node;
	}

	} else if (!source_newer_op) {
	// Mark resource as failed, require recovery, and prevent migration
	pe__set_resource_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pe__clear_resource_flags(history->rsc, pcmk_rsc_migratable);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_failure(struct action_history *history)
	{
	xmlNode *target_migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for migrate_from on the target
	target_migrate_from = find_lrm_op(history->rsc->id,
	PCMK_ACTION_MIGRATE_FROM, target, source,
	PCMK_OCF_OK, history->rsc->cluster);

	if (/* If the resource state is unknown on the target, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, target)
	/* If the resource has newer state on the target after the migration
	* events, this migrate_to no longer matters for the target.
	*/
	&& !newer_state_after_migrate(history->rsc->id, target, history->xml,
	target_migrate_from,
	history->rsc->cluster)) {
	/* The resource has no newer state on the target, so assume it's still
	* active there.
	* (if it is up).
	*/
	pcmk_node_t *target_node = pe_find_node(history->rsc->cluster->nodes,
	target);

	if (target_node && target_node->details->online) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	}

	} else if (!non_monitor_after(history->rsc->id, source, history->xml, true,
	history->rsc->cluster)) {
	/* We know the resource has newer state on the target, but this
	* migrate_to still matters for the source as long as there's no newer
	* non-monitor operation there.
	*/

	// Mark node as having dangling migration so we can force a stop later
	history->rsc->dangling_migrations =
	g_list_prepend(history->rsc->dangling_migrations,
	(gpointer) history->node);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_from action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_from_failure(struct action_history *history)
	{
	xmlNode *source_migrate_to = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, NULL, history->node, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for a migrate_to on the source
	source_migrate_to = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_TO,
	source, target, PCMK_OCF_OK,
	history->rsc->cluster);

	if (/* If the resource state is unknown on the source, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, source)
	/* If the resource has newer state on the source after the migration
	* events, this migrate_from no longer matters for the source.
	*/
	&& !newer_state_after_migrate(history->rsc->id, source,
	source_migrate_to, history->xml,
	history->rsc->cluster)) {
	/* The resource has no newer state on the source, so assume it's still
	* active there (if it is up).
	*/
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	if (source_node && source_node->details->online) {
	native_add_running(history->rsc, source_node, history->rsc->cluster,
	TRUE);
	}
	}
	}

	/*!
	* \internal
	* \brief Add an action to cluster's list of failed actions
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	record_failed_op(struct action_history *history)
	{
	if (!(history->node->details->online)) {
	return;
	}

	for (const xmlNode *xIter = history->rsc->cluster->failed->children;
	xIter != NULL; xIter = xIter->next) {

	const char *key = pe__xe_history_key(xIter);
	const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);

	if (pcmk__str_eq(history->key, key, pcmk__str_none)
	&& pcmk__str_eq(uname, history->node->details->uname,
	pcmk__str_casei)) {
	crm_trace("Skipping duplicate entry %s on %s",
	history->key, pe__node_name(history->node));
	return;
	}
	}

	crm_trace("Adding entry for %s on %s to failed action list",
	history->key, pe__node_name(history->node));
	crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);
	crm_xml_add(history->xml, XML_LRM_ATTR_RSCID, history->rsc->id);
	add_node_copy(history->rsc->cluster->failed, history->xml);
	}

	static char *
	last_change_str(const xmlNode *xml_op)
	{
	time_t when;
	char *result = NULL;

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&when) == pcmk_ok) {
	char *when_s = pcmk__epoch2str(&when, 0);
	const char *p = strchr(when_s, ' ');

	// Skip day of week to make message shorter
	if ((p != NULL) && (*(++p) != '\0')) {
	result = strdup(p);
	CRM_ASSERT(result != NULL);
	}
	free(when_s);
	}

	if (result == NULL) {
	result = strdup("unknown time");
	CRM_ASSERT(result != NULL);
	}

	return result;
	}

	/*!
	* \internal
	* \brief Compare two on-fail values
	*
	* \param[in] first One on-fail value to compare
	* \param[in] second The other on-fail value to compare
	*
	* \return A negative number if second is more severe than first, zero if they
	* are equal, or a positive number if first is more severe than second.
	* \note This is only needed until the action_fail_response values can be
	* renumbered at the next API compatibility break.
	*/
	static int
	cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
	{
	switch (first) {
	case pcmk_on_fail_demote:
	switch (second) {
	case pcmk_on_fail_ignore:
	return 1;
	case pcmk_on_fail_demote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_reset_remote:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return 1;
	case pcmk_on_fail_reset_remote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return 1;
	case pcmk_on_fail_restart_container:
	return 0;
	default:
	return -1;
	}
	break;

	default:
	break;
	}
	switch (second) {
	case pcmk_on_fail_demote:
	return (first == pcmk_on_fail_ignore)? -1 : 1;

	case pcmk_on_fail_reset_remote:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return -1;
	default:
	return 1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return -1;
	default:
	return 1;
	}
	break;

	default:
	break;
	}
	return first - second;
	}

	/*!
	* \internal
	* \brief Ban a resource (or its clone if an anonymous instance) from all nodes
	*
	* \param[in,out] rsc Resource to ban
	*/
	static void
	ban_from_all_nodes(pcmk_resource_t *rsc)
	{
	int score = -INFINITY;
	pcmk_resource_t *fail_rsc = rsc;

	if (fail_rsc->parent != NULL) {
	pcmk_resource_t *parent = uber_parent(fail_rsc);

	if (pe_rsc_is_anon_clone(parent)) {
	/* For anonymous clones, if an operation with on-fail=stop fails for
	* any instance, the entire clone must stop.
	*/
	fail_rsc = parent;
	}
	}

	// Ban the resource from all nodes
	crm_notice("%s will not be started under current conditions", fail_rsc->id);
	if (fail_rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(fail_rsc->allowed_nodes);
	}
	fail_rsc->allowed_nodes = pe__node_list2table(rsc->cluster->nodes);
	g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
	}

	/*!
	* \internal
	* \brief Get configured failure handling and role after failure for an action
	*
	* \param[in,out] history Unpacked action history entry
	* \param[out] on_fail Where to set configured failure handling
	* \param[out] fail_role Where to set to role after failure
	*/
	static void
	unpack_failure_handling(struct action_history *history,
	enum action_fail_response *on_fail,
	enum rsc_role_e *fail_role)
	{
	xmlNode *config = pcmk__find_action_config(history->rsc, history->task,
	history->interval_ms, true);

	GHashTable *meta = pcmk__unpack_action_meta(history->rsc, history->node,
	history->task,
	history->interval_ms, config);

	const char *on_fail_str = g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);

	*on_fail = pcmk__parse_on_fail(history->rsc, history->task,
	history->interval_ms, on_fail_str);
	fail_role = pcmk__role_after_failure(history->rsc, history->task, on_fail,
	meta);
	g_hash_table_destroy(meta);
	}

	/*!
	* \internal
	* \brief Update resource role, failure handling, etc., after a failed action
	*
	* \param[in,out] history Parsed action result history
	* \param[in] config_on_fail Action failure handling from configuration
	* \param[in] fail_role Resource's role after failure of this action
	* \param[out] last_failure This will be set to the history XML
	* \param[in,out] on_fail Actual handling of action result
	*/
	static void
	unpack_rsc_op_failure(struct action_history *history,
	enum action_fail_response config_on_fail,
	enum rsc_role_e fail_role, xmlNode **last_failure,
	enum action_fail_response *on_fail)
	{
	bool is_probe = false;
	char *last_change_s = NULL;

	*last_failure = history->xml;

	is_probe = pcmk_xe_is_probe(history->xml);
	last_change_s = last_change_str(history->xml);

	if (!pcmk_is_set(history->rsc->cluster->flags, pcmk_sched_symmetric_cluster)
	&& (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	crm_trace("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pe__node_name(history->node), last_change_s,
	history->exit_status, history->id);
	} else {
	- crm_warn("Unexpected result (%s%s%s) was recorded for "
	- "%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
	- services_ocf_exitcode_str(history->exit_status),
	- (pcmk__str_empty(history->exit_reason)? "" : ": "),
	- pcmk__s(history->exit_reason, ""),
	- (is_probe? "probe" : history->task), history->rsc->id,
	- pe__node_name(history->node), last_change_s,
	- history->exit_status, history->id);
	+ pcmk__sched_warn("Unexpected result (%s%s%s) was recorded for %s of "
	+ "%s on %s at %s " CRM_XS " exit-status=%d id=%s",
	+ services_ocf_exitcode_str(history->exit_status),
	+ (pcmk__str_empty(history->exit_reason)? "" : ": "),
	+ pcmk__s(history->exit_reason, ""),
	+ (is_probe? "probe" : history->task), history->rsc->id,
	+ pe__node_name(history->node), last_change_s,
	+ history->exit_status, history->id);

	if (is_probe && (history->exit_status != PCMK_OCF_OK)
	&& (history->exit_status != PCMK_OCF_NOT_RUNNING)
	&& (history->exit_status != PCMK_OCF_RUNNING_PROMOTED)) {

	/* A failed (not just unexpected) probe result could mean the user
	* didn't know resources will be probed even where they can't run.
	*/
	crm_notice("If it is not possible for %s to run on %s, see "
	"the resource-discovery option for location constraints",
	history->rsc->id, pe__node_name(history->node));
	}

	record_failed_op(history);
	}

	free(last_change_s);

	if (cmp_on_fail(*on_fail, config_on_fail) < 0) {
	pcmk__rsc_trace(history->rsc, "on-fail %s -> %s for %s",
	fail2text(*on_fail), fail2text(config_on_fail),
	history->key);
	*on_fail = config_on_fail;
	}

	if (strcmp(history->task, PCMK_ACTION_STOP) == 0) {
	resource_location(history->rsc, history->node, -INFINITY,
	"__stop_fail__", history->rsc->cluster);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0) {
	unpack_migrate_to_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_FROM) == 0) {
	unpack_migrate_from_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if (strcmp(history->task, PCMK_ACTION_DEMOTE) == 0) {
	if (config_on_fail == pcmk_on_fail_block) {
	history->rsc->role = pcmk_role_promoted;
	pe__set_next_role(history->rsc, pcmk_role_stopped,
	"demote with on-fail=block");

	} else if (history->exit_status == PCMK_OCF_NOT_RUNNING) {
	history->rsc->role = pcmk_role_stopped;

	} else {
	/* Staying in the promoted role would put the scheduler and
	* controller into a loop. Setting the role to unpromoted is not
	* dangerous because the resource will be stopped as part of
	* recovery, and any promotion will be ordered after that stop.
	*/
	history->rsc->role = pcmk_role_unpromoted;
	}
	}

	if (is_probe && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	/* leave stopped */
	pcmk__rsc_trace(history->rsc, "Leaving %s stopped", history->rsc->id);
	history->rsc->role = pcmk_role_stopped;

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "Setting %s active", history->rsc->id);
	set_active(history->rsc);
	}

	pcmk__rsc_trace(history->rsc,
	"Resource %s: role=%s unclean=%s on_fail=%s fail_role=%s",
	history->rsc->id, role2text(history->rsc->role),
	pcmk__btoa(history->node->details->unclean),
	fail2text(config_on_fail), role2text(fail_role));

	if ((fail_role != pcmk_role_started)
	&& (history->rsc->next_role < fail_role)) {
	pe__set_next_role(history->rsc, fail_role, "failure");
	}

	if (fail_role == pcmk_role_stopped) {
	ban_from_all_nodes(history->rsc);
	}
	}

	/*!
	* \internal
	* \brief Block a resource with a failed action if it cannot be recovered
	*
	* If resource action is a failed stop and fencing is not possible, mark the
	* resource as unmanaged and blocked, since recovery cannot be done.
	*
	* \param[in,out] history Parsed action history entry
	*/
	static void
	block_if_unrecoverable(struct action_history *history)
	{
	char *last_change_s = NULL;

	if (strcmp(history->task, PCMK_ACTION_STOP) != 0) {
	return; // All actions besides stop are always recoverable
	}
	if (pe_can_fence(history->node->details->data_set, history->node)) {
	return; // Failed stops are recoverable via fencing
	}

	last_change_s = last_change_str(history->xml);
	pcmk__sched_err("No further recovery can be attempted for %s "
	"because %s on %s failed (%s%s%s) at %s "
	CRM_XS " rc=%d id=%s",
	history->rsc->id, history->task,
	pe__node_name(history->node),
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	last_change_s, history->exit_status, history->id);

	free(last_change_s);

	pe__clear_resource_flags(history->rsc, pcmk_rsc_managed);
	pe__set_resource_flags(history->rsc, pcmk_rsc_blocked);
	}

	/*!
	* \internal
	* \brief Update action history's execution status and why
	*
	* \param[in,out] history Parsed action history entry
	* \param[out] why Where to store reason for update
	* \param[in] value New value
	* \param[in] reason Description of why value was changed
	*/
	static inline void
	remap_because(struct action_history history, const char *why, int value,
	const char *reason)
	{
	if (history->execution_status != value) {
	history->execution_status = value;
	*why = reason;
	}
	}

	/*!
	* \internal
	* \brief Remap informational monitor results and operation status
	*
	* For the monitor results, certain OCF codes are for providing extended information
	* to the user about services that aren't yet failed but not entirely healthy either.
	* These must be treated as the "normal" result by Pacemaker.
	*
	* For operation status, the action result can be used to determine an appropriate
	* status for the purposes of responding to the action. The status provided by the
	* executor is not directly usable since the executor does not know what was expected.
	*
	* \param[in,out] history Parsed action history entry
	* \param[in,out] on_fail What should be done about the result
	* \param[in] expired Whether result is expired
	*
	* \note If the result is remapped and the node is not shutting down or failed,
	* the operation will be recorded in the scheduler data's list of failed
	* operations to highlight it for the user.
	*
	* \note This may update the resource's current and next role.
	*/
	static void
	remap_operation(struct action_history *history,
	enum action_fail_response *on_fail, bool expired)
	{
	bool is_probe = false;
	int orig_exit_status = history->exit_status;
	int orig_exec_status = history->execution_status;
	const char *why = NULL;
	const char *task = history->task;

	// Remap degraded results to their successful counterparts
	history->exit_status = pcmk__effective_rc(history->exit_status);
	if (history->exit_status != orig_exit_status) {
	why = "degraded result";
	if (!expired && (!history->node->details->shutdown
	\|\| history->node->details->online)) {
	record_failed_op(history);
	}
	}

	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& ((history->execution_status != PCMK_EXEC_DONE)
	\|\| (history->exit_status != PCMK_OCF_NOT_RUNNING))) {
	history->execution_status = PCMK_EXEC_DONE;
	history->exit_status = PCMK_OCF_NOT_RUNNING;
	why = "equivalent probe result";
	}

	/* If the executor reported an execution status of anything but done or
	* error, consider that final. But for done or error, we know better whether
	* it should be treated as a failure or not, because we know the expected
	* result.
	*/
	switch (history->execution_status) {
	case PCMK_EXEC_DONE:
	case PCMK_EXEC_ERROR:
	break;

	// These should be treated as node-fatal
	case PCMK_EXEC_NO_FENCE_DEVICE:
	case PCMK_EXEC_NO_SECRETS:
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"node-fatal error");
	goto remap_done;

	default:
	goto remap_done;
	}

	is_probe = pcmk_xe_is_probe(history->xml);
	if (is_probe) {
	task = "probe";
	}

	if (history->expected_exit_status < 0) {
	/* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
	* Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
	* expected exit status in the transition key, which (along with the
	* similar case of a corrupted transition key in the CIB) will be
	* reported to this function as -1. Pacemaker 2.0+ does not support
	* rolling upgrades from those versions or processing of saved CIB files
	* from those versions, so we do not need to care much about this case.
	*/
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"obsolete history format");
	- crm_warn("Expected result not found for %s on %s "
	- "(corrupt or obsolete CIB?)",
	- history->key, pe__node_name(history->node));
	+ pcmk__config_warn("Expected result not found for %s on %s "
	+ "(corrupt or obsolete CIB?)",
	+ history->key, pe__node_name(history->node));

	} else if (history->exit_status == history->expected_exit_status) {
	remap_because(history, &why, PCMK_EXEC_DONE, "expected result");

	} else {
	remap_because(history, &why, PCMK_EXEC_ERROR, "unexpected result");
	pcmk__rsc_debug(history->rsc,
	"%s on %s: expected %d (%s), got %d (%s%s%s)",
	history->key, pe__node_name(history->node),
	history->expected_exit_status,
	services_ocf_exitcode_str(history->expected_exit_status),
	history->exit_status,
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""));
	}

	switch (history->exit_status) {
	case PCMK_OCF_OK:
	if (is_probe
	&& (history->expected_exit_status == PCMK_OCF_NOT_RUNNING)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe
	\|\| (history->expected_exit_status == history->exit_status)
	\|\| !pcmk_is_set(history->rsc->flags, pcmk_rsc_managed)) {

	/* For probes, recurring monitors for the Stopped role, and
	* unmanaged resources, "not running" is not considered a
	* failure.
	*/
	remap_because(history, &why, PCMK_EXEC_DONE, "exit status");
	history->rsc->role = pcmk_role_stopped;
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"not running");
	}
	break;

	case PCMK_OCF_RUNNING_PROMOTED:
	if (is_probe
	&& (history->exit_status != history->expected_exit_status)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active and promoted on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	if (!expired
	\|\| (history->exit_status == history->expected_exit_status)) {
	history->rsc->role = pcmk_role_promoted;
	}
	break;

	case PCMK_OCF_FAILED_PROMOTED:
	if (!expired) {
	history->rsc->role = pcmk_role_promoted;
	}
	remap_because(history, &why, PCMK_EXEC_ERROR, "exit status");
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	remap_because(history, &why, PCMK_EXEC_ERROR_FATAL, "exit status");
	break;

	case PCMK_OCF_UNIMPLEMENT_FEATURE:
	{
	guint interval_ms = 0;
	crm_element_value_ms(history->xml, XML_LRM_ATTR_INTERVAL_MS,
	&interval_ms);

	if (interval_ms == 0) {
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"exit status");
	} else {
	remap_because(history, &why, PCMK_EXEC_NOT_SUPPORTED,
	"exit status");
	}
	}
	break;

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD, "exit status");
	break;

	default:
	if (history->execution_status == PCMK_EXEC_DONE) {
	char *last_change_s = last_change_str(history->xml);

	crm_info("Treating unknown exit status %d from %s of %s "
	"on %s at %s as failure",
	history->exit_status, task, history->rsc->id,
	pe__node_name(history->node), last_change_s);
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"unknown exit status");
	free(last_change_s);
	}
	break;
	}

	remap_done:
	if (why != NULL) {
	pcmk__rsc_trace(history->rsc,
	"Remapped %s result from [%s: %s] to [%s: %s] "
	"because of %s",
	history->key, pcmk_exec_status_str(orig_exec_status),
	crm_exit_str(orig_exit_status),
	pcmk_exec_status_str(history->execution_status),
	crm_exit_str(history->exit_status), why);
	}
	}

	// return TRUE if start or monitor last failure but parameters changed
	static bool
	should_clear_for_param_change(const xmlNode xml_op, const char task,
	pcmk_resource_t rsc, pcmk_node_t node)
	{
	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_MONITOR, NULL)) {
	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't allocated resources yet, so we can't reliably
	* substitute addr parameters for the REMOTE_CONTAINER_HACK.
	* When that's needed, defer the check until later.
	*/
	pe__add_param_check(xml_op, rsc, node, pcmk__check_last_failure,
	rsc->cluster);

	} else {
	pcmk__op_digest_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
	rsc->cluster);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s"
	" has no digest to compare",
	rsc->id, pe__xe_history_key(xml_op),
	node->details->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	// Order action after fencing of remote node, given connection rsc
	static void
	order_after_remote_fencing(pcmk_action_t action, pcmk_resource_t remote_conn,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_node_t *remote_node = pe_find_node(scheduler->nodes, remote_conn->id);

	if (remote_node) {
	pcmk_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
	FALSE, scheduler);

	order_actions(fence, action, pcmk__ar_first_implies_then);
	}
	}

	static bool
	should_ignore_failure_timeout(const pcmk_resource_t rsc, const char task,
	guint interval_ms, bool is_last_failure)
	{
	/* Clearing failures of recurring monitors has special concerns. The
	* executor reports only changes in the monitor result, so if the
	* monitor is still active and still getting the same failure result,
	* that will go undetected after the failure is cleared.
	*
	* Also, the operation history will have the time when the recurring
	* monitor result changed to the given code, not the time when the
	* result last happened.
	*
	* @TODO We probably should clear such failures only when the failure
	* timeout has passed since the last occurrence of the failed result.
	* However we don't record that information. We could maybe approximate
	* that by clearing only if there is a more recent successful monitor or
	* stop result, but we don't even have that information at this point
	* since we are still unpacking the resource's operation history.
	*
	* This is especially important for remote connection resources with a
	* reconnect interval, so in that case, we skip clearing failures
	* if the remote node hasn't been fenced.
	*/
	if (rsc->remote_reconnect_ms
	&& pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)
	&& (interval_ms != 0)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {

	pcmk_node_t *remote_node = pe_find_node(rsc->cluster->nodes, rsc->id);

	if (remote_node && !remote_node->details->remote_was_fenced) {
	if (is_last_failure) {
	crm_info("Waiting to clear monitor failure for remote node %s"
	" until fencing has occurred", rsc->id);
	}
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Check operation age and schedule failure clearing when appropriate
	*
	* This function has two distinct purposes. The first is to check whether an
	* operation history entry is expired (i.e. the resource has a failure timeout,
	* the entry is older than the timeout, and the resource either has no fail
	* count or its fail count is entirely older than the timeout). The second is to
	* schedule fail count clearing when appropriate (i.e. the operation is expired
	* and either the resource has an expired fail count or the operation is a
	* last_failure for a remote connection resource with a reconnect interval,
	* or the operation is a last_failure for a start or monitor operation and the
	* resource's parameters have changed since the operation).
	*
	* \param[in,out] history Parsed action result history
	*
	* \return true if operation history entry is expired, otherwise false
	*/
	static bool
	check_operation_expiry(struct action_history *history)
	{
	bool expired = false;
	bool is_last_failure = pcmk__ends_with(history->id, "_last_failure_0");
	time_t last_run = 0;
	int unexpired_fail_count = 0;
	const char *clear_reason = NULL;

	if (history->execution_status == PCMK_EXEC_NOT_INSTALLED) {
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not expired: "
	"Not Installed does not expire",
	history->id, pe__node_name(history->node));
	return false; // "Not installed" must always be cleared manually
	}

	if ((history->rsc->failure_timeout > 0)
	&& (crm_element_value_epoch(history->xml, XML_RSC_OP_LAST_CHANGE,
	&last_run) == 0)) {

	// Resource has a failure-timeout, and history entry has a timestamp

	time_t now = get_effective_time(history->rsc->cluster);
	time_t last_failure = 0;

	// Is this particular operation history older than the failure timeout?
	if ((now >= (last_run + history->rsc->failure_timeout))
	&& !should_ignore_failure_timeout(history->rsc, history->task,
	history->interval_ms,
	is_last_failure)) {
	expired = true;
	}

	// Does the resource as a whole have an unexpired fail count?
	unexpired_fail_count = pe_get_failcount(history->node, history->rsc,
	&last_failure,
	pcmk__fc_effective,
	history->xml);

	// Update scheduler recheck time according to last failure
	crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
	" last-failure@%lld",
	history->id, (long long) last_run, (expired? "" : "not "),
	(long long) now, unexpired_fail_count,
	history->rsc->failure_timeout, (long long) last_failure);
	last_failure += history->rsc->failure_timeout + 1;
	if (unexpired_fail_count && (now < last_failure)) {
	pe__update_recheck_time(last_failure, history->rsc->cluster,
	"fail count expiration");
	}
	}

	if (expired) {
	if (pe_get_failcount(history->node, history->rsc, NULL,
	pcmk__fc_default, history->xml)) {
	// There is a fail count ignoring timeout

	if (unexpired_fail_count == 0) {
	// There is no fail count considering timeout
	clear_reason = "it expired";

	} else {
	/* This operation is old, but there is an unexpired fail count.
	* In a properly functioning cluster, this should only be
	* possible if this operation is not a failure (otherwise the
	* fail count should be expired too), so this is really just a
	* failsafe.
	*/
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Unexpired fail count",
	history->id, pe__node_name(history->node));
	expired = false;
	}

	} else if (is_last_failure
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* Clear any expired last failure when reconnect interval is set,
	* even if there is no fail count.
	*/
	clear_reason = "reconnect interval is set";
	}
	}

	if (!expired && is_last_failure
	&& should_clear_for_param_change(history->xml, history->task,
	history->rsc, history->node)) {
	clear_reason = "resource parameters have changed";
	}

	if (clear_reason != NULL) {
	pcmk_action_t *clear_op = NULL;

	// Schedule clearing of the fail count
	clear_op = pe__clear_failcount(history->rsc, history->node,
	clear_reason, history->rsc->cluster);

	if (pcmk_is_set(history->rsc->cluster->flags,
	pcmk_sched_fencing_enabled)
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* If we're clearing a remote connection due to a reconnect
	* interval, we want to wait until any scheduled fencing
	* completes.
	*
	* We could limit this to remote_node->details->unclean, but at
	* this point, that's always true (it won't be reliable until
	* after unpack_node_history() is done).
	*/
	crm_info("Clearing %s failure will wait until any scheduled "
	"fencing of %s completes",
	history->task, history->rsc->id);
	order_after_remote_fencing(clear_op, history->rsc,
	history->rsc->cluster);
	}
	}

	if (expired && (history->interval_ms == 0)
	&& pcmk__str_eq(history->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	switch (history->exit_status) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_PROMOTED:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_PROMOTED:
	// Don't expire probes that return these values
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Probe result",
	history->id, pe__node_name(history->node));
	expired = false;
	break;
	}
	}

	return expired;
	}

	int
	pe__target_rc_from_xml(const xmlNode *xml_op)
	{
	int target_rc = 0;
	const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);

	if (key == NULL) {
	return -1;
	}
	decode_transition_key(key, NULL, NULL, NULL, &target_rc);
	return target_rc;
	}

	/*!
	* \internal
	* \brief Update a resource's state for an action result
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] exit_status Exit status to base new state on
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	update_resource_state(struct action_history *history, int exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	bool clear_past_failure = false;

	if ((exit_status == PCMK_OCF_NOT_INSTALLED)
	\|\| (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml))) {
	history->rsc->role = pcmk_role_stopped;

	} else if (exit_status == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MONITOR,
	pcmk__str_none)) {
	if ((last_failure != NULL)
	&& pcmk__str_eq(history->key, pe__xe_history_key(last_failure),
	pcmk__str_none)) {
	clear_past_failure = true;
	}
	if (history->rsc->role < pcmk_role_started) {
	set_active(history->rsc);
	}

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_START, pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	history->rsc->role = pcmk_role_stopped;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_promoted;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	if (*on_fail == pcmk_on_fail_demote) {
	// Demote clears an error only if on-fail=demote
	clear_past_failure = true;
	}
	history->rsc->role = pcmk_role_unpromoted;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {
	unpack_migrate_to_success(history);

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "%s active on %s",
	history->rsc->id, pe__node_name(history->node));
	set_active(history->rsc);
	}

	if (!clear_past_failure) {
	return;
	}

	switch (*on_fail) {
	case pcmk_on_fail_stop:
	case pcmk_on_fail_ban:
	case pcmk_on_fail_standby_node:
	case pcmk_on_fail_fence_node:
	pcmk__rsc_trace(history->rsc,
	"%s (%s) is not cleared by a completed %s",
	history->rsc->id, fail2text(*on_fail),
	history->task);
	break;

	case pcmk_on_fail_block:
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_restart_container:
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures");
	break;

	case pcmk_on_fail_reset_remote:
	if (history->rsc->remote_reconnect_ms == 0) {
	/* With no reconnect interval, the connection is allowed to
	* start again after the remote node is fenced and
	* completely stopped. (With a reconnect interval, we wait
	* for the failure to be cleared entirely before attempting
	* to reconnect.)
	*/
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures and reset remote");
	}
	break;
	}
	}

	/*!
	* \internal
	* \brief Check whether a given history entry matters for resource state
	*
	* \param[in] history Parsed action history entry
	*
	* \return true if action can affect resource state, otherwise false
	*/
	static inline bool
	can_affect_state(struct action_history *history)
	{
	#if 0
	/* @COMPAT It might be better to parse only actions we know we're interested
	* in, rather than exclude a couple we don't. However that would be a
	* behavioral change that should be done at a major or minor series release.
	* Currently, unknown operations can affect whether a resource is considered
	* active and/or failed.
	*/
	return pcmk__str_any_of(history->task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	"asyncmon", NULL);
	#else
	return !pcmk__str_any_of(history->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_META_DATA, NULL);
	#endif
	}

	/*!
	* \internal
	* \brief Unpack execution/exit status and exit reason from a history entry
	*
	* \param[in,out] history Action history entry to unpack
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_action_result(struct action_history *history)
	{
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_OPSTATUS,
	&(history->execution_status)) < 0)
	\|\| (history->execution_status < PCMK_EXEC_PENDING)
	\|\| (history->execution_status > PCMK_EXEC_MAX)
	\|\| (history->execution_status == PCMK_EXEC_CANCELLED)) {
	- crm_err("Ignoring resource history entry %s for %s on %s "
	- "with invalid " XML_LRM_ATTR_OPSTATUS " '%s'",
	- history->id, history->rsc->id, pe__node_name(history->node),
	- pcmk__s(crm_element_value(history->xml, XML_LRM_ATTR_OPSTATUS),
	- ""));
	+ pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	+ "with invalid " XML_LRM_ATTR_OPSTATUS " '%s'",
	+ history->id, history->rsc->id,
	+ pe__node_name(history->node),
	+ pcmk__s(crm_element_value(history->xml,
	+ XML_LRM_ATTR_OPSTATUS),
	+ ""));
	return pcmk_rc_unpack_error;
	}
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_RC,
	&(history->exit_status)) < 0)
	\|\| (history->exit_status < 0) \|\| (history->exit_status > CRM_EX_MAX)) {
	#if 0
	/* @COMPAT We should ignore malformed entries, but since that would
	* change behavior, it should be done at a major or minor series
	* release.
	*/
	- crm_err("Ignoring resource history entry %s for %s on %s "
	- "with invalid " XML_LRM_ATTR_RC " '%s'",
	- history->id, history->rsc->id, pe__node_name(history->node),
	- pcmk__s(crm_element_value(history->xml, XML_LRM_ATTR_RC),
	- ""));
	+ pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	+ "with invalid " XML_LRM_ATTR_RC " '%s'",
	+ history->id, history->rsc->id,
	+ pe__node_name(history->node),
	+ pcmk__s(crm_element_value(history->xml,
	+ XML_LRM_ATTR_RC),
	+ ""));
	return pcmk_rc_unpack_error;
	#else
	history->exit_status = CRM_EX_ERROR;
	#endif
	}
	history->exit_reason = crm_element_value(history->xml,
	XML_LRM_ATTR_EXIT_REASON);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Process an action history entry whose result expired
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	*
	* \return Standard Pacemaker return code (in particular, pcmk_rc_ok means the
	* entry needs no further processing)
	*/
	static int
	process_expired_result(struct action_history *history, int orig_exit_status)
	{
	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& (orig_exit_status != history->expected_exit_status)) {

	if (history->rsc->role <= pcmk_role_stopped) {
	history->rsc->role = pcmk_role_unknown;
	}
	crm_trace("Ignoring resource history entry %s for probe of %s on %s: "
	"Masked failure expired",
	history->id, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->exit_status == history->expected_exit_status) {
	return pcmk_rc_undetermined; // Only failures expire
	}

	if (history->interval_ms == 0) {
	crm_notice("Ignoring resource history entry %s for %s of %s on %s: "
	"Expired failure",
	history->id, history->task, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->node->details->online && !history->node->details->unclean) {
	/* Reschedule the recurring action. schedule_cancel() won't work at
	* this stage, so as a hacky workaround, forcibly change the restart
	* digest so pcmk__check_action_config() does what we want later.
	*
	* @TODO We should skip this if there is a newer successful monitor.
	* Also, this causes rescheduling only if the history entry
	* has an op-digest (which the expire-non-blocked-failure
	* scheduler regression test doesn't, but that may not be a
	* realistic scenario in production).
	*/
	crm_notice("Rescheduling %s-interval %s of %s on %s "
	"after failure expired",
	pcmk__readable_interval(history->interval_ms), history->task,
	history->rsc->id, pe__node_name(history->node));
	crm_xml_add(history->xml, XML_LRM_ATTR_RESTART_DIGEST,
	"calculated-failure-timeout");
	return pcmk_rc_ok;
	}

	return pcmk_rc_undetermined;
	}

	/*!
	* \internal
	* \brief Process a masked probe failure
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	mask_probe_failure(struct action_history *history, int orig_exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	pcmk_resource_t *ban_rsc = history->rsc;

	if (!pcmk_is_set(history->rsc->flags, pcmk_rsc_unique)) {
	ban_rsc = uber_parent(history->rsc);
	}

	crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
	services_ocf_exitcode_str(orig_exit_status), history->rsc->id,
	pe__node_name(history->node));
	update_resource_state(history, history->expected_exit_status, last_failure,
	on_fail);
	crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);

	record_failed_op(history);
	resource_location(ban_rsc, history->node, -INFINITY, "masked-probe-failure",
	history->rsc->cluster);
	}

	/*!
	* \internal Check whether a given failure is for a given pending action
	*
	* \param[in] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*
	* \return true if \p last_failure is failure of pending action in \p history,
	* otherwise false
	* \note Both \p history and \p last_failure must come from the same
	* lrm_resource block, as node and resource are assumed to be the same.
	*/
	static bool
	failure_is_newer(const struct action_history *history,
	const xmlNode *last_failure)
	{
	guint failure_interval_ms = 0U;
	long long failure_change = 0LL;
	long long this_change = 0LL;

	if (last_failure == NULL) {
	return false; // Resource has no last_failure entry
	}

	if (!pcmk__str_eq(history->task,
	crm_element_value(last_failure, XML_LRM_ATTR_TASK),
	pcmk__str_none)) {
	return false; // last_failure is for different action
	}

	if ((crm_element_value_ms(last_failure, XML_LRM_ATTR_INTERVAL_MS,
	&failure_interval_ms) != pcmk_ok)
	\|\| (history->interval_ms != failure_interval_ms)) {
	return false; // last_failure is for action with different interval
	}

	if ((pcmk__scan_ll(crm_element_value(history->xml, XML_RSC_OP_LAST_CHANGE),
	&this_change, 0LL) != pcmk_rc_ok)
	\|\| (pcmk__scan_ll(crm_element_value(last_failure,
	XML_RSC_OP_LAST_CHANGE),
	&failure_change, 0LL) != pcmk_rc_ok)
	\|\| (failure_change < this_change)) {
	return false; // Failure is not known to be newer
	}

	return true;
	}

	/*!
	* \internal
	* \brief Update a resource's role etc. for a pending action
	*
	* \param[in,out] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*/
	static void
	process_pending_action(struct action_history *history,
	const xmlNode *last_failure)
	{
	/* For recurring monitors, a failure is recorded only in RSC_last_failure_0,
	* and there might be a RSC_monitor_INTERVAL entry with the last successful
	* or pending result.
	*
	* If last_failure contains the failure of the pending recurring monitor
	* we're processing here, and is newer, the action is no longer pending.
	* (Pending results have call ID -1, which sorts last, so the last failure
	* if any should be known.)
	*/
	if (failure_is_newer(history, last_failure)) {
	return;
	}

	if (strcmp(history->task, PCMK_ACTION_START) == 0) {
	pe__set_resource_flags(history->rsc, pcmk_rsc_start_pending);
	set_active(history->rsc);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if ((strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0)
	&& history->node->details->unclean) {
	/* A migrate_to action is pending on a unclean source, so force a stop
	* on the target.
	*/
	const char *migrate_target = NULL;
	pcmk_node_t *target = NULL;

	migrate_target = crm_element_value(history->xml,
	XML_LRM_ATTR_MIGRATE_TARGET);
	target = pe_find_node(history->rsc->cluster->nodes, migrate_target);
	if (target != NULL) {
	stop_action(history->rsc, target, FALSE);
	}
	}

	if (history->rsc->pending_task != NULL) {
	/* There should never be multiple pending actions, but as a failsafe,
	* just remember the first one processed for display purposes.
	*/
	return;
	}

	if (pcmk_is_probe(history->task, history->interval_ms)) {
	/* Pending probes are currently never displayed, even if pending
	* operations are requested. If we ever want to change that,
	* enable the below and the corresponding part of
	* native.c:native_pending_task().
	*/
	#if 0
	history->rsc->pending_task = strdup("probe");
	history->rsc->pending_node = history->node;
	#endif
	} else {
	history->rsc->pending_task = strdup(history->task);
	history->rsc->pending_node = history->node;
	}
	}

	static void
	unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node, xmlNode *xml_op,
	xmlNode *last_failure, enum action_fail_response on_fail)
	{
	int old_rc = 0;
	bool expired = false;
	pcmk_resource_t *parent = rsc;
	enum rsc_role_e fail_role = pcmk_role_unknown;
	enum action_fail_response failure_strategy = pcmk_on_fail_restart;

	struct action_history history = {
	.rsc = rsc,
	.node = node,
	.xml = xml_op,
	.execution_status = PCMK_EXEC_UNKNOWN,
	};

	CRM_CHECK(rsc && node && xml_op, return);

	history.id = ID(xml_op);
	if (history.id == NULL) {
	- crm_err("Ignoring resource history entry for %s on %s without ID",
	- rsc->id, pe__node_name(node));
	+ pcmk__config_err("Ignoring resource history entry for %s on %s "
	+ "without ID", rsc->id, pe__node_name(node));
	return;
	}

	// Task and interval
	history.task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (history.task == NULL) {
	- crm_err("Ignoring resource history entry %s for %s on %s without "
	- XML_LRM_ATTR_TASK, history.id, rsc->id, pe__node_name(node));
	+ pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	+ "without " XML_LRM_ATTR_TASK,
	+ history.id, rsc->id, pe__node_name(node));
	return;
	}
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS,
	&(history.interval_ms));
	if (!can_affect_state(&history)) {
	pcmk__rsc_trace(rsc,
	"Ignoring resource history entry %s for %s on %s "
	"with irrelevant action '%s'",
	history.id, rsc->id, pe__node_name(node), history.task);
	return;
	}

	if (unpack_action_result(&history) != pcmk_rc_ok) {
	return; // Error already logged
	}

	history.expected_exit_status = pe__target_rc_from_xml(xml_op);
	history.key = pe__xe_history_key(xml_op);
	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &(history.call_id));

	pcmk__rsc_trace(rsc, "Unpacking %s (%s call %d on %s): %s (%s)",
	history.id, history.task, history.call_id,
	pe__node_name(node),
	pcmk_exec_status_str(history.execution_status),
	crm_exit_str(history.exit_status));

	if (node->details->unclean) {
	pcmk__rsc_trace(rsc,
	"%s is running on %s, which is unclean (further action "
	"depends on value of stop's on-fail attribute)",
	rsc->id, pe__node_name(node));
	}

	expired = check_operation_expiry(&history);
	old_rc = history.exit_status;

	remap_operation(&history, on_fail, expired);

	if (expired && (process_expired_result(&history, old_rc) == pcmk_rc_ok)) {
	goto done;
	}

	if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
	mask_probe_failure(&history, old_rc, *last_failure, on_fail);
	goto done;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	parent = uber_parent(rsc);
	}

	switch (history.execution_status) {
	case PCMK_EXEC_PENDING:
	process_pending_action(&history, *last_failure);
	goto done;

	case PCMK_EXEC_DONE:
	update_resource_state(&history, history.exit_status, *last_failure,
	on_fail);
	goto done;

	case PCMK_EXEC_NOT_INSTALLED:
	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if (failure_strategy == pcmk_on_fail_ignore) {
	crm_warn("Cannot ignore failed %s of %s on %s: "
	"Resource agent doesn't exist "
	CRM_XS " status=%d rc=%d id=%s",
	history.task, rsc->id, pe__node_name(node),
	history.execution_status, history.exit_status,
	history.id);
	/* Also for printing it as "FAILED" by marking it as
	* pcmk_rsc_failed later
	*/
	*on_fail = pcmk_on_fail_ban;
	}
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);
	goto done;

	case PCMK_EXEC_NOT_CONNECTED:
	if (pe__is_guest_or_remote_node(node)
	&& pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_managed)) {
	/* We should never get into a situation where a managed remote
	* connection resource is considered OK but a resource action
	* behind the connection gets a "not connected" status. But as a
	* fail-safe in case a bug or unusual circumstances do lead to
	* that, ensure the remote connection is considered failed.
	*/
	pe__set_resource_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	break; // Not done, do error handling

	case PCMK_EXEC_ERROR:
	case PCMK_EXEC_ERROR_HARD:
	case PCMK_EXEC_ERROR_FATAL:
	case PCMK_EXEC_TIMEOUT:
	case PCMK_EXEC_NOT_SUPPORTED:
	case PCMK_EXEC_INVALID:
	break; // Not done, do error handling

	default: // No other value should be possible at this point
	break;
	}

	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if ((failure_strategy == pcmk_on_fail_ignore)
	\|\| ((failure_strategy == pcmk_on_fail_restart_container)
	&& (strcmp(history.task, PCMK_ACTION_STOP) == 0))) {

	char *last_change_s = last_change_str(xml_op);

	crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s succeeded "
	CRM_XS " %s",
	history.task, services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), rsc->id, pe__node_name(node),
	last_change_s, history.id);
	free(last_change_s);

	update_resource_state(&history, history.expected_exit_status,
	*last_failure, on_fail);
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	pe__set_resource_flags(rsc, pcmk_rsc_ignore_failure);

	record_failed_op(&history);

	if ((failure_strategy == pcmk_on_fail_restart_container)
	&& cmp_on_fail(*on_fail, pcmk_on_fail_restart) <= 0) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);

	if (history.execution_status == PCMK_EXEC_ERROR_HARD) {
	uint8_t log_level = LOG_ERR;

	if (history.exit_status == PCMK_OCF_NOT_INSTALLED) {
	log_level = LOG_NOTICE;
	}
	do_crm_log(log_level,
	"Preventing %s from restarting on %s because "
	"of hard failure (%s%s%s) " CRM_XS " %s",
	parent->id, pe__node_name(node),
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);

	} else if (history.execution_status == PCMK_EXEC_ERROR_FATAL) {
	- crm_err("Preventing %s from restarting anywhere because "
	- "of fatal failure (%s%s%s) " CRM_XS " %s",
	- parent->id, services_ocf_exitcode_str(history.exit_status),
	- (pcmk__str_empty(history.exit_reason)? "" : ": "),
	- pcmk__s(history.exit_reason, ""), history.id);
	+ pcmk__sched_err("Preventing %s from restarting anywhere because "
	+ "of fatal failure (%s%s%s) " CRM_XS " %s",
	+ parent->id,
	+ services_ocf_exitcode_str(history.exit_status),
	+ (pcmk__str_empty(history.exit_reason)? "" : ": "),
	+ pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, NULL, -INFINITY, "fatal-error",
	rsc->cluster);
	}
	}

	done:
	pcmk__rsc_trace(rsc, "%s role on %s after %s is %s (next %s)",
	rsc->id, pe__node_name(node), history.id,
	role2text(rsc->role), role2text(rsc->next_role));
	}

	static void
	add_node_attrs(const xmlNode xml_obj, pcmk_node_t node, bool overwrite,
	pcmk_scheduler_t *scheduler)
	{
	const char *cluster_name = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_UNAME), strdup(node->details->uname));

	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
	strdup(node->details->id));
	if (pcmk__str_eq(node->details->id, scheduler->dc_uuid, pcmk__str_casei)) {
	scheduler->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
	}

	cluster_name = g_hash_table_lookup(scheduler->config_hash, "cluster-name");
	if (cluster_name) {
	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
	strdup(cluster_name));
	}

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
	node->details->attrs, NULL, overwrite,
	scheduler);

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
	node->details->utilization, NULL,
	FALSE, scheduler);

	if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
	const char *site_name = pe_node_attribute_raw(node, "site-name");

	if (site_name) {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(site_name));

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(cluster_name));
	}
	}
	}

	static GList *
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GList *gIter = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP,
	pcmk__str_none)) {
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GList *
	find_operations(const char rsc, const char node, gboolean active_filter,
	pcmk_scheduler_t *scheduler)
	{
	GList *output = NULL;
	GList *intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(scheduler->input, XML_CIB_TAG_STATUS, TRUE);

	pcmk_node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	for (node_state = pcmk__xe_first_child(status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {

	if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);

	if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
	continue;
	}

	this_node = pe_find_node(scheduler->nodes, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);

	} else {
	determine_online_status(node_state, this_node, scheduler);
	}

	if (this_node->details->online
	\|\| pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	for (lrm_rsc = pcmk__xe_first_child(tmp); lrm_rsc != NULL;
	lrm_rsc = pcmk__xe_next(lrm_rsc)) {

	if (pcmk__str_eq((const char *)lrm_rsc->name,
	XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);

	if (rsc != NULL && !pcmk__str_eq(rsc_id, rsc, pcmk__str_casei)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}
	diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
	index 36682fc183..3a9dc367ed 100644
	--- a/lib/pengine/utils.c
	+++ b/lib/pengine/utils.c
	@@ -1,912 +1,913 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>

	#include "pe_status_private.h"

	extern bool pcmk__is_daemon;

	gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);

	/*!
	* \internal
	* \brief Check whether we can fence a particular node
	*
	* \param[in] scheduler Scheduler data
	* \param[in] node Name of node to check
	*
	* \return true if node can be fenced, false otherwise
	*/
	bool
	pe_can_fence(const pcmk_scheduler_t scheduler, const pcmk_node_t node)
	{
	if (pe__is_guest_node(node)) {
	/* Guest nodes are fenced by stopping their container resource. We can
	* do that if the container's host is either online or fenceable.
	*/
	pcmk_resource_t *rsc = node->details->remote_rsc->container;

	for (GList *n = rsc->running_on; n != NULL; n = n->next) {
	pcmk_node_t *container_node = n->data;

	if (!container_node->details->online
	&& !pe_can_fence(scheduler, container_node)) {
	return false;
	}
	}
	return true;

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	return false; /* Turned off */

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {
	return false; /* No devices */

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	return true;

	} else if (scheduler->no_quorum_policy == pcmk_no_quorum_ignore) {
	return true;

	} else if(node == NULL) {
	return false;

	} else if(node->details->online) {
	crm_notice("We can fence %s without quorum because they're in our membership",
	pe__node_name(node));
	return true;
	}

	crm_trace("Cannot fence %s", pe__node_name(node));
	return false;
	}

	/*!
	* \internal
	* \brief Copy a node object
	*
	* \param[in] this_node Node object to copy
	*
	* \return Newly allocated shallow copy of this_node
	* \note This function asserts on errors and is guaranteed to return non-NULL.
	*/
	pcmk_node_t *
	pe__copy_node(const pcmk_node_t *this_node)
	{
	pcmk_node_t *new_node = NULL;

	CRM_ASSERT(this_node != NULL);

	new_node = calloc(1, sizeof(pcmk_node_t));
	CRM_ASSERT(new_node != NULL);

	new_node->rsc_discover_mode = this_node->rsc_discover_mode;
	new_node->weight = this_node->weight;
	new_node->fixed = this_node->fixed; // @COMPAT deprecated and unused
	new_node->count = this_node->count;
	new_node->details = this_node->details;

	return new_node;
	}

	/*!
	* \internal
	* \brief Create a node hash table from a node list
	*
	* \param[in] list Node list
	*
	* \return Hash table equivalent of node list
	*/
	GHashTable *
	pe__node_list2table(const GList *list)
	{
	GHashTable *result = NULL;

	result = pcmk__strkey_table(NULL, free);
	for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *new_node = NULL;

	new_node = pe__copy_node((const pcmk_node_t *) gIter->data);
	g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Compare two nodes by name, with numeric portions sorted numerically
	*
	* Sort two node names case-insensitively like strcasecmp(), but with any
	* numeric portions of the name sorted numerically. For example, "node10" will
	* sort higher than "node9" but lower than "remotenode9".
	*
	* \param[in] a First node to compare (can be \c NULL)
	* \param[in] b Second node to compare (can be \c NULL)
	*
	* \retval -1 \c a comes before \c b (or \c a is \c NULL and \c b is not)
	* \retval 0 \c a and \c b are equal (or both are \c NULL)
	* \retval 1 \c a comes after \c b (or \c b is \c NULL and \c a is not)
	*/
	gint
	pe__cmp_node_name(gconstpointer a, gconstpointer b)
	{
	const pcmk_node_t node1 = (const pcmk_node_t ) a;
	const pcmk_node_t node2 = (const pcmk_node_t ) b;

	if ((node1 == NULL) && (node2 == NULL)) {
	return 0;
	}

	if (node1 == NULL) {
	return -1;
	}

	if (node2 == NULL) {
	return 1;
	}

	return pcmk__numeric_strcasecmp(node1->details->uname,
	node2->details->uname);
	}

	/*!
	* \internal
	* \brief Output node weights to stdout
	*
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	pe__output_node_weights(const pcmk_resource_t rsc, const char comment,
	GHashTable nodes, pcmk_scheduler_t scheduler)
	{
	pcmk__output_t *out = scheduler->priv;

	// Sort the nodes so the output is consistent for regression tests
	GList *list = g_list_sort(g_hash_table_get_values(nodes),
	pe__cmp_node_name);

	for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	const pcmk_node_t node = (const pcmk_node_t ) gIter->data;

	out->message(out, "node-weight", rsc, comment, node->details->uname,
	pcmk_readable_score(node->weight));
	}
	g_list_free(list);
	}

	/*!
	* \internal
	* \brief Log node weights at trace level
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] rsc If not NULL, include this resource's ID in logs
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes Nodes whose scores should be logged
	*/
	static void
	pe__log_node_weights(const char file, const char function, int line,
	const pcmk_resource_t rsc, const char comment,
	GHashTable *nodes)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	// Don't waste time if we're not tracing at this point
	pcmk__if_tracing({}, return);

	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (rsc) {
	qb_log_from_external_source(function, file,
	"%s: %s allocation score on %s: %s",
	LOG_TRACE, line, 0,
	comment, rsc->id,
	pe__node_name(node),
	pcmk_readable_score(node->weight));
	} else {
	qb_log_from_external_source(function, file, "%s: %s = %s",
	LOG_TRACE, line, 0,
	comment, pe__node_name(node),
	pcmk_readable_score(node->weight));
	}
	}
	}

	/*!
	* \internal
	* \brief Log or output node weights
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] to_log Log if true, otherwise output
	* \param[in] rsc If not NULL, use this resource's ID in logs,
	* and show scores recursively for any children
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes Nodes whose scores should be shown
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pe__show_node_scores_as(const char file, const char function, int line,
	bool to_log, const pcmk_resource_t *rsc,
	const char comment, GHashTable nodes,
	pcmk_scheduler_t *scheduler)
	{
	if ((rsc != NULL) && pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	// Don't show allocation scores for orphans
	return;
	}
	if (nodes == NULL) {
	// Nothing to show
	return;
	}

	if (to_log) {
	pe__log_node_weights(file, function, line, rsc, comment, nodes);
	} else {
	pe__output_node_weights(rsc, comment, nodes, scheduler);
	}

	// If this resource has children, repeat recursively for each
	if (rsc && rsc->children) {
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	pe__show_node_scores_as(file, function, line, to_log, child,
	comment, child->allowed_nodes, scheduler);
	}
	}
	}

	/*!
	* \internal
	* \brief Compare two resources by priority
	*
	* \param[in] a First resource to compare (can be \c NULL)
	* \param[in] b Second resource to compare (can be \c NULL)
	*
	* \retval -1 \c a->priority > \c b->priority (or \c b is \c NULL and \c a is
	* not)
	* \retval 0 \c a->priority == \c b->priority (or both \c a and \c b are
	* \c NULL)
	* \retval 1 \c a->priority < \c b->priority (or \c a is \c NULL and \c b is
	* not)
	*/
	gint
	pe__cmp_rsc_priority(gconstpointer a, gconstpointer b)
	{
	const pcmk_resource_t resource1 = (const pcmk_resource_t )a;
	const pcmk_resource_t resource2 = (const pcmk_resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->priority > resource2->priority) {
	return -1;
	}

	if (resource1->priority < resource2->priority) {
	return 1;
	}

	return 0;
	}

	static void
	resource_node_score(pcmk_resource_t rsc, const pcmk_node_t node, int score,
	const char *tag)
	{
	pcmk_node_t *match = NULL;

	if ((rsc->exclusive_discover
	\|\| (node->rsc_discover_mode == pcmk_probe_never))
	&& pcmk__str_eq(tag, "symmetric_default", pcmk__str_casei)) {
	/* This string comparision may be fragile, but exclusive resources and
	* exclusive nodes should not have the symmetric_default constraint
	* applied to them.
	*/
	return;

	} else if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	resource_node_score(child_rsc, node, score, tag);
	}
	}

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match == NULL) {
	match = pe__copy_node(node);
	g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
	}
	match->weight = pcmk__add_scores(match->weight, score);
	pcmk__rsc_trace(rsc,
	"Enabling %s preference (%s) for %s on %s (now %s)",
	tag, pcmk_readable_score(score), rsc->id,
	pe__node_name(node), pcmk_readable_score(match->weight));
	}

	void
	resource_location(pcmk_resource_t rsc, const pcmk_node_t node, int score,
	const char tag, pcmk_scheduler_t scheduler)
	{
	if (node != NULL) {
	resource_node_score(rsc, node, score, tag);

	} else if (scheduler != NULL) {
	GList *gIter = scheduler->nodes;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node_iter = (pcmk_node_t ) gIter->data;

	resource_node_score(rsc, node_iter, score, tag);
	}

	} else {
	GHashTableIter iter;
	pcmk_node_t *node_iter = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
	resource_node_score(rsc, node_iter, score, tag);
	}
	}

	if (node == NULL && score == -INFINITY) {
	if (rsc->allocated_to) {
	crm_info("Deallocating %s from %s",
	rsc->id, pe__node_name(rsc->allocated_to));
	free(rsc->allocated_to);
	rsc->allocated_to = NULL;
	}
	}
	}

	time_t
	get_effective_time(pcmk_scheduler_t *scheduler)
	{
	if(scheduler) {
	if (scheduler->now == NULL) {
	crm_trace("Recording a new 'now'");
	scheduler->now = crm_time_new(NULL);
	}
	return crm_time_get_seconds_since_epoch(scheduler->now);
	}

	crm_trace("Defaulting to 'now'");
	return time(NULL);
	}

	gboolean
	get_target_role(const pcmk_resource_t rsc, enum rsc_role_e role)
	{
	enum rsc_role_e local_role = pcmk_role_unknown;
	const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);

	CRM_CHECK(role != NULL, return FALSE);

	if (pcmk__str_eq(value, "started", pcmk__str_null_matches \| pcmk__str_casei)
	\|\| pcmk__str_eq("default", value, pcmk__str_casei)) {
	return FALSE;
	}

	local_role = text2role(value);
	if (local_role == pcmk_role_unknown) {
	pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
	"because '%s' is not valid", rsc->id, value);
	return FALSE;

	} else if (local_role > pcmk_role_started) {
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)) {
	if (local_role > pcmk_role_unpromoted) {
	/* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
	return FALSE;
	}

	} else {
	pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
	"because '%s' only makes sense for promotable "
	"clones", rsc->id, value);
	return FALSE;
	}
	}

	*role = local_role;
	return TRUE;
	}

	gboolean
	order_actions(pcmk_action_t lh_action, pcmk_action_t rh_action,
	uint32_t flags)
	{
	GList *gIter = NULL;
	pcmk__related_action_t *wrapper = NULL;
	GList *list = NULL;

	if (flags == pcmk__ar_none) {
	return FALSE;
	}

	if (lh_action == NULL \|\| rh_action == NULL) {
	return FALSE;
	}

	crm_trace("Creating action wrappers for ordering: %s then %s",
	lh_action->uuid, rh_action->uuid);

	/* Ensure we never create a dependency on ourselves... it's happened */
	CRM_ASSERT(lh_action != rh_action);

	/* Filter dups, otherwise update_action_states() has too much work to do */
	gIter = lh_action->actions_after;
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk__related_action_t *after = gIter->data;

	if (after->action == rh_action && (after->type & flags)) {
	return FALSE;
	}
	}

	wrapper = calloc(1, sizeof(pcmk__related_action_t));
	wrapper->action = rh_action;
	wrapper->type = flags;
	list = lh_action->actions_after;
	list = g_list_prepend(list, wrapper);
	lh_action->actions_after = list;

	wrapper = calloc(1, sizeof(pcmk__related_action_t));
	wrapper->action = lh_action;
	wrapper->type = flags;
	list = rh_action->actions_before;
	list = g_list_prepend(list, wrapper);
	rh_action->actions_before = list;
	return TRUE;
	}

	void
	destroy_ticket(gpointer data)
	{
	pcmk_ticket_t *ticket = data;

	if (ticket->state) {
	g_hash_table_destroy(ticket->state);
	}
	free(ticket->id);
	free(ticket);
	}

	pcmk_ticket_t *
	ticket_new(const char ticket_id, pcmk_scheduler_t scheduler)
	{
	pcmk_ticket_t *ticket = NULL;

	if (pcmk__str_empty(ticket_id)) {
	return NULL;
	}

	if (scheduler->tickets == NULL) {
	scheduler->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {

	ticket = calloc(1, sizeof(pcmk_ticket_t));
	if (ticket == NULL) {
	- crm_err("Cannot allocate ticket '%s'", ticket_id);
	+ pcmk__sched_err("Cannot allocate ticket '%s'", ticket_id);
	return NULL;
	}

	crm_trace("Creaing ticket entry for %s", ticket_id);

	ticket->id = strdup(ticket_id);
	ticket->granted = FALSE;
	ticket->last_granted = -1;
	ticket->standby = FALSE;
	ticket->state = pcmk__strkey_table(free, free);

	g_hash_table_insert(scheduler->tickets, strdup(ticket->id), ticket);
	}

	return ticket;
	}

	const char *
	rsc_printable_id(const pcmk_resource_t *rsc)
	{
	return pcmk_is_set(rsc->flags, pcmk_rsc_unique)? rsc->id : ID(rsc->xml);
	}

	void
	pe__clear_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags)
	{
	pe__clear_resource_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__clear_resource_flags_recursive((pcmk_resource_t *) gIter->data,
	flags);
	}
	}

	void
	pe__clear_resource_flags_on_all(pcmk_scheduler_t *scheduler, uint64_t flag)
	{
	for (GList *lpc = scheduler->resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t r = (pcmk_resource_t ) lpc->data;
	pe__clear_resource_flags_recursive(r, flag);
	}
	}

	void
	pe__set_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags)
	{
	pe__set_resource_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__set_resource_flags_recursive((pcmk_resource_t *) gIter->data,
	flags);
	}
	}

	void
	trigger_unfencing(pcmk_resource_t rsc, pcmk_node_t node, const char *reason,
	pcmk_action_t dependency, pcmk_scheduler_t scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_enable_unfencing)) {
	/* No resources require it */
	return;

	} else if ((rsc != NULL)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	/* Wasn't a stonith device */
	return;

	} else if(node
	&& node->details->online
	&& node->details->unclean == FALSE
	&& node->details->shutdown == FALSE) {
	pcmk_action_t *unfence = pe_fence_op(node, PCMK_ACTION_ON, FALSE,
	reason, FALSE, scheduler);

	if(dependency) {
	order_actions(unfence, dependency, pcmk__ar_ordered);
	}

	} else if(rsc) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
	trigger_unfencing(rsc, node, reason, dependency, scheduler);
	}
	}
	}
	}

	gboolean
	add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref)
	{
	pcmk_tag_t *tag = NULL;
	GList *gIter = NULL;
	gboolean is_existing = FALSE;

	CRM_CHECK(tags && tag_name && obj_ref, return FALSE);

	tag = g_hash_table_lookup(tags, tag_name);
	if (tag == NULL) {
	tag = calloc(1, sizeof(pcmk_tag_t));
	if (tag == NULL) {
	+ pcmk__sched_err("Could not allocate memory for tag %s", tag_name);
	return FALSE;
	}
	tag->id = strdup(tag_name);
	tag->refs = NULL;
	g_hash_table_insert(tags, strdup(tag_name), tag);
	}

	for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
	const char existing_ref = (const char ) gIter->data;

	if (pcmk__str_eq(existing_ref, obj_ref, pcmk__str_none)){
	is_existing = TRUE;
	break;
	}
	}

	if (is_existing == FALSE) {
	tag->refs = g_list_append(tag->refs, strdup(obj_ref));
	crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Check whether shutdown has been requested for a node
	*
	* \param[in] node Node to check
	*
	* \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
	* \note This differs from simply using node->details->shutdown in that it can
	* be used before that has been determined (and in fact to determine it),
	* and it can also be used to distinguish requested shutdown from implicit
	* shutdown of remote nodes by virtue of their connection stopping.
	*/
	bool
	pe__shutdown_requested(const pcmk_node_t *node)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);

	return !pcmk__str_eq(shutdown, "0", pcmk__str_null_matches);
	}

	/*!
	* \internal
	* \brief Update "recheck by" time in scheduler data
	*
	* \param[in] recheck Epoch time when recheck should happen
	* \param[in,out] scheduler Scheduler data
	* \param[in] reason What time is being updated for (for logs)
	*/
	void
	pe__update_recheck_time(time_t recheck, pcmk_scheduler_t *scheduler,
	const char *reason)
	{
	if ((recheck > get_effective_time(scheduler))
	&& ((scheduler->recheck_by == 0)
	\|\| (scheduler->recheck_by > recheck))) {
	scheduler->recheck_by = recheck;
	crm_debug("Updated next scheduler recheck to %s for %s",
	pcmk__trim(ctime(&recheck)), reason);
	}
	}

	/*!
	* \internal
	* \brief Extract nvpair blocks contained by a CIB XML element into a hash table
	*
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name If not NULL, only use blocks of this element
	* \param[in] rule_data Matching parameters to use when unpacking
	* \param[out] hash Where to store extracted name/value pairs
	* \param[in] always_first If not NULL, process block with this ID first
	* \param[in] overwrite Whether to replace existing values with same name
	* \param[in,out] scheduler Scheduler data containing \p xml_obj
	*/
	void
	pe__unpack_dataset_nvpairs(const xmlNode xml_obj, const char set_name,
	const pe_rule_eval_data_t *rule_data,
	GHashTable hash, const char always_first,
	gboolean overwrite, pcmk_scheduler_t *scheduler)
	{
	crm_time_t *next_change = crm_time_new_undefined();

	pe_eval_nvpairs(scheduler->input, xml_obj, set_name, rule_data, hash,
	always_first, overwrite, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, scheduler, "rule evaluation");
	}
	crm_time_free(next_change);
	}

	bool
	pe__resource_is_disabled(const pcmk_resource_t *rsc)
	{
	const char *target_role = NULL;

	CRM_CHECK(rsc != NULL, return false);
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	if (target_role) {
	// If invalid, we've already logged an error when unpacking
	enum rsc_role_e target_role_e = text2role(target_role);

	if ((target_role_e == pcmk_role_stopped)
	\|\| ((target_role_e == pcmk_role_unpromoted)
	&& pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable))) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a resource is running only on given node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p rsc is running only on \p node, otherwise false
	*/
	bool
	pe__rsc_running_on_only(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	return (rsc != NULL) && pcmk__list_of_1(rsc->running_on)
	&& pe__same_node((const pcmk_node_t *) rsc->running_on->data, node);
	}

	bool
	pe__rsc_running_on_any(pcmk_resource_t rsc, GList node_list)
	{
	for (GList *ele = rsc->running_on; ele; ele = ele->next) {
	pcmk_node_t node = (pcmk_node_t ) ele->data;
	if (pcmk__str_in_list(node->details->uname, node_list,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	return true;
	}
	}

	return false;
	}

	bool
	pcmk__rsc_filtered_by_node(pcmk_resource_t rsc, GList only_node)
	{
	return (rsc->fns->active(rsc, FALSE) && !pe__rsc_running_on_any(rsc, only_node));
	}

	GList *
	pe__filter_rsc_list(GList rscs, GList filter)
	{
	GList *retval = NULL;

	for (GList *gIter = rscs; gIter; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	/* I think the second condition is safe here for all callers of this
	* function. If not, it needs to move into pe__node_text.
	*/
	if (pcmk__str_in_list(rsc_printable_id(rsc), filter, pcmk__str_star_matches) \|\|
	(rsc->parent && pcmk__str_in_list(rsc_printable_id(rsc->parent), filter, pcmk__str_star_matches))) {
	retval = g_list_prepend(retval, rsc);
	}
	}

	return retval;
	}

	GList *
	pe__build_node_name_list(pcmk_scheduler_t scheduler, const char s)
	{
	GList *nodes = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	/* Nothing was given so return a list of all node names. Or, '*' was
	* given. This would normally fall into the pe__unames_with_tag branch
	* where it will return an empty list. Catch it here instead.
	*/
	nodes = g_list_prepend(nodes, strdup("*"));
	} else {
	pcmk_node_t *node = pe_find_node(scheduler->nodes, s);

	if (node) {
	/* The given string was a valid uname for a node. Return a
	* singleton list containing just that uname.
	*/
	nodes = g_list_prepend(nodes, strdup(s));
	} else {
	/* The given string was not a valid uname. It's either a tag or
	* it's a typo or something. In the first case, we'll return a
	* list of all the unames of the nodes with the given tag. In the
	* second case, we'll return a NULL pointer and nothing will
	* get displayed.
	*/
	nodes = pe__unames_with_tag(scheduler, s);
	}
	}

	return nodes;
	}

	GList *
	pe__build_rsc_list(pcmk_scheduler_t scheduler, const char s)
	{
	GList *resources = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	resources = g_list_prepend(resources, strdup("*"));
	} else {
	const uint32_t flags = pcmk_rsc_match_history\|pcmk_rsc_match_basename;
	pcmk_resource_t *rsc = pe_find_resource_with_flags(scheduler->resources,
	s, flags);

	if (rsc) {
	/* A colon in the name we were given means we're being asked to filter
	* on a specific instance of a cloned resource. Put that exact string
	* into the filter list. Otherwise, use the printable ID of whatever
	* resource was found that matches what was asked for.
	*/
	if (strstr(s, ":") != NULL) {
	resources = g_list_prepend(resources, strdup(rsc->id));
	} else {
	resources = g_list_prepend(resources, strdup(rsc_printable_id(rsc)));
	}
	} else {
	/* The given string was not a valid resource name. It's a tag or a
	* typo or something. See pe__build_node_name_list() for more
	* detail.
	*/
	resources = pe__rscs_with_tag(scheduler, s);
	}
	}

	return resources;
	}

	xmlNode *
	pe__failed_probe_for_rsc(const pcmk_resource_t rsc, const char name)
	{
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
	const char *rsc_id = rsc->id;

	if (parent->variant == pcmk_rsc_variant_clone) {
	rsc_id = pe__clone_child_id(parent);
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(rsc->cluster->failed); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	const char *value = NULL;
	char *op_id = NULL;

	/* This resource operation is not a failed probe. */
	if (!pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	/* This resource operation was not run on the given node. Note that if name is
	* NULL, this will always succeed.
	*/
	value = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);
	if (value == NULL \|\| !pcmk__str_eq(value, name, pcmk__str_casei\|pcmk__str_null_matches)) {
	continue;
	}

	if (!parse_op_key(pe__xe_history_key(xml_op), &op_id, NULL, NULL)) {
	continue; // This history entry is missing an operation key
	}

	/* This resource operation's ID does not match the rsc_id we are looking for. */
	if (!pcmk__str_eq(op_id, rsc_id, pcmk__str_none)) {
	free(op_id);
	continue;
	}

	free(op_id);
	return xml_op;
	}

	return NULL;
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 8:05 PM (6 h, 29 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1655371
Default Alt Text: (655 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions