No OneTemporary
Actions

Size

1 MB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/daemons/fenced/fenced_scheduler.c b/daemons/fenced/fenced_scheduler.c
	index 03b96cc18c..e878aded6f 100644
	--- a/daemons/fenced/fenced_scheduler.c
	+++ b/daemons/fenced/fenced_scheduler.c
	@@ -1,232 +1,232 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <errno.h>
	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>

	#include <pacemaker-internal.h>
	#include <pacemaker-fenced.h>

	static pcmk_scheduler_t *scheduler = NULL;

	/*!
	* \internal
	* \brief Initialize scheduler data for fencer purposes
	*
	* \return Standard Pacemaker return code
	*/
	int
	fenced_scheduler_init(void)
	{
	pcmk__output_t *logger = NULL;
	int rc = pcmk__log_output_new(&logger);

	if (rc != pcmk_rc_ok) {
	return rc;
	}

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	pcmk__output_free(logger);
	return ENOMEM;
	}

	pe__register_messages(logger);
	pcmk__register_lib_messages(logger);
	pcmk__output_set_log_level(logger, LOG_TRACE);
	scheduler->priv = logger;

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Free all scheduler-related resources
	*/
	void
	fenced_scheduler_cleanup(void)
	{
	if (scheduler != NULL) {
	pcmk__output_t *logger = scheduler->priv;

	if (logger != NULL) {
	logger->finish(logger, CRM_EX_OK, true, NULL);
	pcmk__output_free(logger);
	scheduler->priv = NULL;
	}
	pe_free_working_set(scheduler);
	scheduler = NULL;
	}
	}

	/*!
	* \internal
	* \brief Check whether the local node is in a resource's allowed node list
	*
	* \param[in] rsc Resource to check
	*
	* \return Pointer to node if found, otherwise NULL
	*/
	static pcmk_node_t *
	local_node_allowed_for(const pcmk_resource_t *rsc)
	{
	if ((rsc != NULL) && (stonith_our_uname != NULL)) {
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (pcmk__str_eq(node->private->name, stonith_our_uname,
	pcmk__str_casei)) {
	return node;
	}
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief If a given resource or any of its children are fencing devices,
	* register the devices
	*
	* \param[in,out] data Resource to check
	* \param[in,out] user_data Ignored
	*/
	static void
	register_if_fencing_device(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const char *rsc_id = pcmk__s(rsc->private->history_id, rsc->id);

	xmlNode *xml = NULL;
	GHashTableIter hash_iter;
	pcmk_node_t *node = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *rclass = NULL;
	const char *agent = NULL;
	const char *rsc_provides = NULL;
	stonith_key_value_t *params = NULL;

	// If this is a collective resource, check children instead
	if (rsc->private->children != NULL) {

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	register_if_fencing_device(iter->data, NULL);
	if (pcmk__is_clone(rsc)) {
	return; // Only one instance needs to be checked for clones
	}
	}
	return;
	}

	rclass = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);
	if (!pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	return; // Not a fencing device
	}

	if (pe__resource_is_disabled(rsc)) {
	crm_info("Ignoring fencing device %s because it is disabled", rsc->id);
	return;
	}

	if ((stonith_watchdog_timeout_ms <= 0) &&
	pcmk__str_eq(rsc->id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
	crm_info("Ignoring fencing device %s "
	"because watchdog fencing is disabled", rsc->id);
	return;
	}

	// Check whether local node is allowed to run resource
	node = local_node_allowed_for(rsc);
	if (node == NULL) {
	crm_info("Ignoring fencing device %s "
	"because local node is not allowed to run it", rsc->id);
	return;
	}
	- if (node->weight < 0) {
	+ if (node->assign->score < 0) {
	crm_info("Ignoring fencing device %s "
	"because local node has preference %s for it",
	- rsc->id, pcmk_readable_score(node->weight));
	+ rsc->id, pcmk_readable_score(node->assign->score));
	return;
	}

	// If device is in a group, check whether local node is allowed for group
	if (pcmk__is_group(rsc->private->parent)) {
	pcmk_node_t *group_node = local_node_allowed_for(rsc->private->parent);

	- if ((group_node != NULL) && (group_node->weight < 0)) {
	+ if ((group_node != NULL) && (group_node->assign->score < 0)) {
	crm_info("Ignoring fencing device %s "
	"because local node has preference %s for its group",
	- rsc->id, pcmk_readable_score(group_node->weight));
	+ rsc->id, pcmk_readable_score(group_node->assign->score));
	return;
	}
	}

	crm_debug("Reloading configuration of fencing device %s", rsc->id);

	agent = crm_element_value(rsc->private->xml, PCMK_XA_TYPE);

	/* @COMPAT Support for node attribute expressions in rules for resource
	* meta-attributes is deprecated. When we can break behavioral backward
	* compatibility, replace node with NULL here.
	*/
	get_meta_attributes(rsc->private->meta, rsc, node, scheduler);
	rsc_provides = g_hash_table_lookup(rsc->private->meta,
	PCMK_STONITH_PROVIDES);

	g_hash_table_iter_init(&hash_iter, pe_rsc_params(rsc, node, scheduler));
	while (g_hash_table_iter_next(&hash_iter, (gpointer *) &name,
	(gpointer *) &value)) {
	if ((name == NULL) \|\| (value == NULL)) {
	continue;
	}
	params = stonith_key_value_add(params, name, value);
	}

	xml = create_device_registration_xml(rsc_id, st_namespace_any, agent,
	params, rsc_provides);
	stonith_key_value_freeall(params, 1, 1);
	CRM_ASSERT(stonith_device_register(xml, TRUE) == pcmk_ok);
	pcmk__xml_free(xml);
	}

	/*!
	* \internal
	* \brief Run the scheduler for fencer purposes
	*
	* \param[in] cib Cluster's current CIB
	*/
	void
	fenced_scheduler_run(xmlNode *cib)
	{
	CRM_CHECK((cib != NULL) && (scheduler != NULL), return);

	if (scheduler->now != NULL) {
	crm_time_free(scheduler->now);
	scheduler->now = NULL;
	}
	scheduler->localhost = stonith_our_uname;
	pcmk__schedule_actions(cib, pcmk_sched_location_only
	\|pcmk_sched_no_compat
	\|pcmk_sched_no_counts, scheduler);
	g_list_foreach(scheduler->resources, register_if_fencing_device, NULL);

	scheduler->input = NULL; // Wasn't a copy, so don't let API free it
	pe_reset_working_set(scheduler);
	}
	diff --git a/include/crm/common/location_internal.h b/include/crm/common/location_internal.h
	index a00691e5e3..5991feea98 100644
	--- a/include/crm/common/location_internal.h
	+++ b/include/crm/common/location_internal.h
	@@ -1,36 +1,36 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_LOCATION_INTERNAL__H
	#define PCMK__CRM_COMMON_LOCATION_INTERNAL__H

	#include <glib.h> // GList

	-#include <crm/common/nodes.h> // enum pe_discover_e
	+#include <crm/common/nodes_internal.h> // enum pcmk__probe_mode
	#include <crm/common/resources.h> // enum rsc_role_e
	#include <crm/common/scheduler_types.h> // pcmk_resource_t

	#ifdef __cplusplus
	extern "C" {
	#endif

	//! Location constraint object
	typedef struct {
	char *id; // XML ID of location constraint
	pcmk_resource_t *rsc; // Resource with location preference
	enum rsc_role_e role_filter; // Limit to instances with this role
	- enum pe_discover_e discover_mode; // How to probe resource on node
	+ enum pcmk__probe_mode probe_mode; // How to probe resource on node
	GList *nodes; // Affected nodes, with preference score
	} pcmk__location_t;

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_LOCATION_INTERNAL__H
	diff --git a/include/crm/common/nodes.h b/include/crm/common/nodes.h
	index fb55cca045..4d9a8d93e5 100644
	--- a/include/crm/common/nodes.h
	+++ b/include/crm/common/nodes.h
	@@ -1,124 +1,102 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_NODES__H
	#define PCMK__CRM_COMMON_NODES__H

	#include <stdbool.h> // bool
	#include <glib.h> // gboolean, GList, GHashTable

	#include <crm/common/scheduler_types.h> // pcmk_resource_t, pcmk_scheduler_t

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*!
	* \file
	* \brief Scheduler API for nodes
	* \ingroup core
	*/

	// Special node attributes

	#define PCMK_NODE_ATTR_MAINTENANCE "maintenance"
	#define PCMK_NODE_ATTR_STANDBY "standby"
	#define PCMK_NODE_ATTR_TERMINATE "terminate"


	-// When to probe a resource on a node (as specified in location constraints)
	-// @COMPAT Make this internal when we can break API backward compatibility
	-//!@{
	-//! \deprecated Do not use (public access will be removed in a future release)
	-enum pe_discover_e {
	- pcmk_probe_always = 0, // Always probe resource on node
	- pcmk_probe_never = 1, // Never probe resource on node
	- pcmk_probe_exclusive = 2, // Probe only on designated nodes
	-
	-#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	- pe_discover_always = pcmk_probe_always,
	- pe_discover_never = pcmk_probe_never,
	- pe_discover_exclusive = pcmk_probe_exclusive,
	-#endif
	-};
	-//!@}
	-
	//! \internal Do not use
	typedef struct pcmk__node_private pcmk__node_private_t;

	// Basic node information (all node objects for the same node share this)
	// @COMPAT Drop this struct once all members are moved to pcmk__node_private_t
	//!@{
	//! \deprecated Do not use (public access will be removed in a future release)
	struct pe_node_shared_s {
	/* @COMPAT Convert these gbooleans into new enum pcmk__node_flags values
	* when we no longer support versions of sbd that use them
	*/

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Call pcmk_node_is_online() instead
	gboolean online; // Whether online

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Call pcmk_node_is_pending() instead
	gboolean pending; // Whether controller membership is pending

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Call !pcmk_node_is_clean() instead
	gboolean unclean; // Whether node requires fencing

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Call pcmk_node_is_shutting_down() instead
	gboolean shutdown; // Whether shutting down

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Call pcmk_node_is_in_maintenance() instead
	gboolean maintenance; // Whether in maintenance mode

	// NOTE: sbd (as of at least 1.5.2) uses this
	// \deprecated Call pcmk_foreach_active_resource() instead
	GList *running_rsc; // List of resources active on node
	};
	//!@}

	// Implementation of pcmk_node_t
	// @COMPAT Make contents internal when we can break API backward compatibility
	//!@{
	//! \deprecated Do not use (public access will be removed in a future release)
	-struct pe_node_s {
	- int weight; // Node score for a given resource
	- gboolean fixed; // \deprecated Do not use
	- int count; // Counter reused by assignment and promotion code
	+struct pcmk__scored_node {
	+ struct pcmk__node_assignment *assign;

	// NOTE: sbd (as of at least 1.5.2) uses this
	struct pe_node_shared_s *details; // Basic node information

	- // @COMPAT This should be enum pe_discover_e
	- int rsc_discover_mode; // Probe mode (enum pe_discover_e)
	-
	//! \internal Do not use
	pcmk__node_private_t *private;
	};
	//!@}

	bool pcmk_node_is_online(const pcmk_node_t *node);
	bool pcmk_node_is_pending(const pcmk_node_t *node);
	bool pcmk_node_is_clean(const pcmk_node_t *node);
	bool pcmk_node_is_shutting_down(const pcmk_node_t *node);
	bool pcmk_node_is_in_maintenance(const pcmk_node_t *node);

	bool pcmk_foreach_active_resource(pcmk_node_t *node,
	bool (fn)(pcmk_resource_t , void *),
	void *user_data);

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_NODES__H
	diff --git a/include/crm/common/nodes_internal.h b/include/crm/common/nodes_internal.h
	index f3684203c5..ada54b9b07 100644
	--- a/include/crm/common/nodes_internal.h
	+++ b/include/crm/common/nodes_internal.h
	@@ -1,176 +1,194 @@
	/*
	* Copyright 2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_NODES_INTERNAL__H
	#define PCMK__CRM_COMMON_NODES_INTERNAL__H

	#include <stdio.h> // NULL
	#include <stdbool.h> // bool
	#include <stdint.h> // uint32_t, UINT32_C()

	#include <glib.h>
	#include <crm/common/nodes.h>

	/*
	* Special node attributes
	*/

	#define PCMK__NODE_ATTR_SHUTDOWN "shutdown"

	/* @COMPAT Deprecated since 2.1.8. Use a location constraint with
	* PCMK_XA_RSC_PATTERN=".*" and PCMK_XA_RESOURCE_DISCOVERY="never" instead of
	* PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED="false".
	*/
	#define PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED "resource-discovery-enabled"

	enum pcmk__node_variant { // Possible node types
	pcmk__node_variant_ping = 0, // deprecated
	pcmk__node_variant_cluster = 1, // Cluster layer node
	pcmk__node_variant_remote = 2, // Pacemaker Remote node
	};

	enum pcmk__node_flags {
	pcmk__node_none = UINT32_C(0),

	// Whether node is in standby mode
	pcmk__node_standby = (UINT32_C(1) << 0),

	// Whether node is in standby mode due to PCMK_META_ON_FAIL
	pcmk__node_fail_standby = (UINT32_C(1) << 1),

	// Whether node has ever joined cluster (and thus has node state in CIB)
	pcmk__node_seen = (UINT32_C(1) << 2),

	// Whether expected join state is member
	pcmk__node_expected_up = (UINT32_C(1) << 3),

	// Whether probes are allowed on node
	pcmk__node_probes_allowed = (UINT32_C(1) << 4),

	/* Whether this either is a guest node whose guest resource must be
	* recovered or a remote node that must be fenced
	*/
	pcmk__node_remote_reset = (UINT32_C(1) << 5),

	/* Whether this is a Pacemaker Remote node that was fenced since it was last
	* connected by the cluster
	*/
	pcmk__node_remote_fenced = (UINT32_C(1) << 6),

	/*
	* Whether this is a Pacemaker Remote node previously marked in its
	* node state as being in maintenance mode
	*/
	pcmk__node_remote_maint = (UINT32_C(1) << 7),

	// Whether node history has been unpacked
	pcmk__node_unpacked = (UINT32_C(1) << 8),
	};

	+// When to probe a resource on a node (as specified in location constraints)
	+enum pcmk__probe_mode {
	+ pcmk__probe_always = 0, // Always probe resource on node
	+ pcmk__probe_never = 1, // Never probe resource on node
	+ pcmk__probe_exclusive = 2, // Probe only on designated nodes
	+};
	+
	+/* Per-node data used in resource assignment
	+ *
	+ * @COMPAT When we can make the pcmk_node_t implementation internal, move these
	+ * there and drop this struct.
	+ */
	+struct pcmk__node_assignment {
	+ int score; // Node's score for relevant resource
	+ int count; // Counter reused by assignment and promotion code
	+ enum pcmk__probe_mode probe_mode; // When to probe resource on this node
	+};
	+
	/* Implementation of pcmk__node_private_t (pcmk_node_t objects are shallow
	* copies, so all pcmk_node_t objects for the same node will share the same
	* private data)
	*/
	typedef struct pcmk__node_private {
	/* Node's XML ID in the CIB (the cluster layer ID for cluster nodes,
	* the node name for Pacemaker Remote nodes)
	*/
	const char *id;

	/*
	* Sum of priorities of all resources active on node and on any guest nodes
	* connected to this node, with +1 for promoted instances (used to compare
	* nodes for PCMK_OPT_PRIORITY_FENCING_DELAY)
	*/
	int priority;

	const char *name; // Node name in cluster
	enum pcmk__node_variant variant; // Node variant
	uint32_t flags; // Group of enum pcmk__node_flags
	GHashTable *attrs; // Node attributes
	GHashTable *utilization; // Node utilization attributes
	int num_resources; // Number of active resources on node
	GList *assigned_resources; // List of resources assigned to node
	GHashTable *digest_cache; // Cache of calculated resource digests
	pcmk_resource_t *remote; // Pacemaker Remote connection (if any)
	pcmk_scheduler_t *scheduler; // Scheduler data that node is part of
	} pcmk__node_private_t;

	pcmk_node_t pcmk__find_node_in_list(const GList nodes, const char *node_name);

	/*!
	* \internal
	* \brief Set node flags
	*
	* \param[in,out] node Node to set flags for
	* \param[in] flags_to_set Group of enum pcmk_node_flags to set
	*/
	#define pcmk__set_node_flags(node, flags_to_set) do { \
	(node)->private->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Node", pcmk__node_name(node), \
	(node)->private->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	/*!
	* \internal
	* \brief Clear node flags
	*
	* \param[in,out] node Node to clear flags for
	* \param[in] flags_to_clear Group of enum pcmk_node_flags to clear
	*/
	#define pcmk__clear_node_flags(node, flags_to_clear) do { \
	(node)->private->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Node", pcmk__node_name(node), \
	(node)->private->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	/*!
	* \internal
	* \brief Return a string suitable for logging as a node name
	*
	* \param[in] node Node to return a node name string for
	*
	* \return Node name if available, otherwise node ID if available,
	* otherwise "unspecified node" if node is NULL or "unidentified node"
	* if node has neither a name nor ID.
	*/
	static inline const char *
	pcmk__node_name(const pcmk_node_t *node)
	{
	if (node == NULL) {
	return "unspecified node";

	} else if (node->private->name != NULL) {
	return node->private->name;

	} else if (node->private->id != NULL) {
	return node->private->id;

	} else {
	return "unidentified node";
	}
	}

	/*!
	* \internal
	* \brief Check whether two node objects refer to the same node
	*
	* \param[in] node1 First node object to compare
	* \param[in] node2 Second node object to compare
	*
	* \return true if \p node1 and \p node2 refer to the same node
	*/
	static inline bool
	pcmk__same_node(const pcmk_node_t node1, const pcmk_node_t node2)
	{
	return (node1 != NULL) && (node2 != NULL)
	&& (node1->private == node2->private);
	}

	#endif // PCMK__CRM_COMMON_NODES_INTERNAL__H
	diff --git a/include/crm/common/scheduler_types.h b/include/crm/common/scheduler_types.h
	index 970d3e497c..a2a3250e5e 100644
	--- a/include/crm/common/scheduler_types.h
	+++ b/include/crm/common/scheduler_types.h
	@@ -1,39 +1,39 @@
	/*
	* Copyright 2023-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_SCHEDULER_TYPES__H
	#define PCMK__CRM_COMMON_SCHEDULER_TYPES__H

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*!
	* \file
	* \brief Type aliases needed to define scheduler objects
	* \ingroup core
	*/

	//! Node object (including information that may vary depending on resource)
	-typedef struct pe_node_s pcmk_node_t;
	+typedef struct pcmk__scored_node pcmk_node_t;

	//! Resource object
	typedef struct pe_resource_s pcmk_resource_t;

	//! Action object
	typedef struct pe_action_s pcmk_action_t;

	//! Scheduler object
	typedef struct pe_working_set_s pcmk_scheduler_t;

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_SCHEDULER_TYPES__H
	diff --git a/include/crm/pengine/pe_types_compat.h b/include/crm/pengine/pe_types_compat.h
	index 95e45ae703..8218e2910f 100644
	--- a/include/crm/pengine/pe_types_compat.h
	+++ b/include/crm/pengine/pe_types_compat.h
	@@ -1,56 +1,56 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
	# define PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H

	#include <crm/common/scheduler.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Deprecated Pacemaker scheduler API
	* \ingroup pengine
	* \deprecated Do not include this header directly. The scheduler APIs in this
	* header, and the header itself, will be removed in a future
	* release.
	*/

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Do not use
	#define pe_rsc_managed (1ULL << 1)

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Do not use
	#define pe_flag_have_quorum pcmk_sched_quorate

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Do not use
	#define pe_flag_have_stonith_resource pcmk_sched_have_fencing

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Use pcmk_node_t instead
	-typedef struct pe_node_s node_t;
	+typedef struct pcmk__scored_node node_t;

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated use pcmk_resource_t instead
	typedef struct pe_resource_s resource_t;

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Use pcmk_scheduler_t instead
	typedef struct pe_working_set_s pe_working_set_t;

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
	diff --git a/lib/common/tests/scheduler/pcmk_get_dc_test.c b/lib/common/tests/scheduler/pcmk_get_dc_test.c
	index 5d9d459f62..83f2b76cfd 100644
	--- a/lib/common/tests/scheduler/pcmk_get_dc_test.c
	+++ b/lib/common/tests/scheduler/pcmk_get_dc_test.c
	@@ -1,47 +1,50 @@
	/*
	* Copyright 2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/scheduler.h>
	#include <crm/common/unittest_internal.h>

	static void
	null_scheduler(void **state)
	{
	assert_null(pcmk_get_dc(NULL));
	}

	static void
	null_dc(void **state)
	{
	pcmk_scheduler_t scheduler = {
	.dc_node = NULL,
	};

	assert_null(pcmk_get_dc(&scheduler));
	}

	static void
	valid_dc(void **state)
	{
	+ struct pcmk__node_assignment assign = {
	+ .score = 1,
	+ };
	pcmk_node_t dc = {
	- .weight = 1,
	+ .assign = &assign,
	};
	pcmk_scheduler_t scheduler = {
	.dc_node = &dc,
	};

	assert_ptr_equal(&dc, pcmk_get_dc(&scheduler));
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(null_scheduler),
	cmocka_unit_test(null_dc),
	cmocka_unit_test(valid_dc))
	diff --git a/lib/pacemaker/pcmk_output.c b/lib/pacemaker/pcmk_output.c
	index 23ca913fb3..2315b62f77 100644
	--- a/lib/pacemaker/pcmk_output.c
	+++ b/lib/pacemaker/pcmk_output.c
	@@ -1,2712 +1,2712 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/common/xml.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h> // stonith__*
	#include <crm/pengine/internal.h>
	#include <libxml/tree.h>
	#include <pacemaker-internal.h>

	#include <inttypes.h>
	#include <stdint.h>

	static char *
	colocations_header(pcmk_resource_t rsc, pcmk__colocation_t cons,
	bool dependents) {
	char *retval = NULL;

	if (cons->primary_role > pcmk_role_started) {
	retval = crm_strdup_printf("%s (score=%s, %s role=%s, id=%s)",
	rsc->id, pcmk_readable_score(cons->score),
	(dependents? "needs" : "with"),
	pcmk_role_text(cons->primary_role),
	cons->id);
	} else {
	retval = crm_strdup_printf("%s (score=%s, id=%s)",
	rsc->id, pcmk_readable_score(cons->score),
	cons->id);
	}
	return retval;
	}

	static void
	colocations_xml_node(pcmk__output_t out, pcmk_resource_t rsc,
	pcmk__colocation_t *cons) {
	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, PCMK_XE_RSC_COLOCATION,
	PCMK_XA_ID, cons->id,
	PCMK_XA_RSC, cons->dependent->id,
	PCMK_XA_WITH_RSC, cons->primary->id,
	PCMK_XA_SCORE,
	pcmk_readable_score(cons->score),
	NULL);

	if (cons->node_attribute) {
	xmlSetProp(node, (pcmkXmlStr) PCMK_XA_NODE_ATTRIBUTE,
	(pcmkXmlStr) cons->node_attribute);
	}

	if (cons->dependent_role != pcmk_role_unknown) {
	xmlSetProp(node, (pcmkXmlStr) PCMK_XA_RSC_ROLE,
	(pcmkXmlStr) pcmk_role_text(cons->dependent_role));
	}

	if (cons->primary_role != pcmk_role_unknown) {
	xmlSetProp(node, (pcmkXmlStr) PCMK_XA_WITH_RSC_ROLE,
	(pcmkXmlStr) pcmk_role_text(cons->primary_role));
	}
	}

	static int
	do_locations_list_xml(pcmk__output_t out, pcmk_resource_t rsc,
	bool add_header)
	{
	GList *lpc = NULL;
	int rc = pcmk_rc_no_output;

	for (lpc = rsc->private->location_constraints;
	lpc != NULL; lpc = lpc->next) {
	pcmk__location_t *cons = lpc->data;

	GList *lpc2 = NULL;

	for (lpc2 = cons->nodes; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_node_t node = (pcmk_node_t ) lpc2->data;

	if (add_header) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "locations");
	}

	pcmk__output_create_xml_node(out, PCMK_XE_RSC_LOCATION,
	PCMK_XA_NODE, node->private->name,
	PCMK_XA_RSC, rsc->id,
	PCMK_XA_ID, cons->id,
	PCMK_XA_SCORE,
	- pcmk_readable_score(node->weight),
	+ pcmk_readable_score(node->assign->score),
	NULL);
	}
	}

	if (add_header) {
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("rsc-action-item", "const char ", "pcmk_resource_t ",
	"pcmk_node_t ", "pcmk_node_t ", "pcmk_action_t *",
	"pcmk_action_t *")
	static int
	rsc_action_item(pcmk__output_t *out, va_list args)
	{
	const char change = va_arg(args, const char );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t origin = va_arg(args, pcmk_node_t );
	pcmk_node_t destination = va_arg(args, pcmk_node_t );
	pcmk_action_t action = va_arg(args, pcmk_action_t );
	pcmk_action_t source = va_arg(args, pcmk_action_t );

	int len = 0;
	char *reason = NULL;
	char *details = NULL;
	bool same_host = false;
	bool same_role = false;
	bool need_role = false;

	static int rsc_width = 5;
	static int detail_width = 5;

	CRM_ASSERT(action);
	CRM_ASSERT(destination != NULL \|\| origin != NULL);

	if (source == NULL) {
	source = action;
	}

	len = strlen(rsc->id);
	if (len > rsc_width) {
	rsc_width = len + 2;
	}

	if ((rsc->private->orig_role > pcmk_role_started)
	\|\| (rsc->private->next_role > pcmk_role_unpromoted)) {
	need_role = true;
	}

	if (pcmk__same_node(origin, destination)) {
	same_host = true;
	}

	if (rsc->private->orig_role == rsc->private->next_role) {
	same_role = true;
	}

	if (need_role && (origin == NULL)) {
	/* Starting and promoting a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s",
	pcmk_role_text(rsc->private->orig_role),
	pcmk_role_text(rsc->private->next_role),
	pcmk__node_name(destination));

	} else if (origin == NULL) {
	/* Starting a resource */
	details = crm_strdup_printf("%s", pcmk__node_name(destination));

	} else if (need_role && (destination == NULL)) {
	/* Stopping a promotable clone instance */
	details = crm_strdup_printf("%s %s",
	pcmk_role_text(rsc->private->orig_role),
	pcmk__node_name(origin));

	} else if (destination == NULL) {
	/* Stopping a resource */
	details = crm_strdup_printf("%s", pcmk__node_name(origin));

	} else if (need_role && same_role && same_host) {
	/* Recovering, restarting or re-promoting a promotable clone instance */
	details = crm_strdup_printf("%s %s",
	pcmk_role_text(rsc->private->orig_role),
	pcmk__node_name(origin));

	} else if (same_role && same_host) {
	/* Recovering or Restarting a normal resource */
	details = crm_strdup_printf("%s", pcmk__node_name(origin));

	} else if (need_role && same_role) {
	/* Moving a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", pcmk__node_name(origin),
	pcmk__node_name(destination),
	pcmk_role_text(rsc->private->orig_role));

	} else if (same_role) {
	/* Moving a normal resource */
	details = crm_strdup_printf("%s -> %s", pcmk__node_name(origin),
	pcmk__node_name(destination));

	} else if (same_host) {
	/* Promoting or demoting a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s",
	pcmk_role_text(rsc->private->orig_role),
	pcmk_role_text(rsc->private->next_role),
	pcmk__node_name(origin));

	} else {
	/* Moving and promoting/demoting */
	details = crm_strdup_printf("%s %s -> %s %s",
	pcmk_role_text(rsc->private->orig_role),
	pcmk__node_name(origin),
	pcmk_role_text(rsc->private->next_role),
	pcmk__node_name(destination));
	}

	len = strlen(details);
	if (len > detail_width) {
	detail_width = len;
	}

	if ((source->reason != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_runnable)) {
	reason = crm_strdup_printf("due to %s (blocked)", source->reason);

	} else if (source->reason) {
	reason = crm_strdup_printf("due to %s", source->reason);

	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	reason = strdup("blocked");

	}

	out->list_item(out, NULL, "%-8s %-s ( %s )%s%s",
	change, rsc_width, rsc->id, detail_width, details,
	((reason == NULL)? "" : " "), pcmk__s(reason, ""));

	free(details);
	free(reason);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("rsc-action-item", "const char ", "pcmk_resource_t ",
	"pcmk_node_t ", "pcmk_node_t ", "pcmk_action_t *",
	"pcmk_action_t *")
	static int
	rsc_action_item_xml(pcmk__output_t *out, va_list args)
	{
	const char change = va_arg(args, const char );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t origin = va_arg(args, pcmk_node_t );
	pcmk_node_t destination = va_arg(args, pcmk_node_t );
	pcmk_action_t action = va_arg(args, pcmk_action_t );
	pcmk_action_t source = va_arg(args, pcmk_action_t );

	char *change_str = NULL;

	bool same_host = false;
	bool same_role = false;
	bool need_role = false;
	xmlNode *xml = NULL;

	CRM_ASSERT(action);
	CRM_ASSERT(destination != NULL \|\| origin != NULL);

	if (source == NULL) {
	source = action;
	}

	if ((rsc->private->orig_role > pcmk_role_started)
	\|\| (rsc->private->next_role > pcmk_role_unpromoted)) {
	need_role = true;
	}

	if (pcmk__same_node(origin, destination)) {
	same_host = true;
	}

	if (rsc->private->orig_role == rsc->private->next_role) {
	same_role = true;
	}

	change_str = g_ascii_strdown(change, -1);
	xml = pcmk__output_create_xml_node(out, PCMK_XE_RSC_ACTION,
	PCMK_XA_ACTION, change_str,
	PCMK_XA_RESOURCE, rsc->id,
	NULL);
	g_free(change_str);

	if (need_role && (origin == NULL)) {
	/* Starting and promoting a promotable clone instance */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE,
	pcmk_role_text(rsc->private->orig_role),
	PCMK_XA_NEXT_ROLE,
	pcmk_role_text(rsc->private->next_role),
	PCMK_XA_DEST, destination->private->name,
	NULL);

	} else if (origin == NULL) {
	/* Starting a resource */
	crm_xml_add(xml, PCMK_XA_NODE, destination->private->name);

	} else if (need_role && (destination == NULL)) {
	/* Stopping a promotable clone instance */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE,
	pcmk_role_text(rsc->private->orig_role),
	PCMK_XA_NODE, origin->private->name,
	NULL);

	} else if (destination == NULL) {
	/* Stopping a resource */
	crm_xml_add(xml, PCMK_XA_NODE, origin->private->name);

	} else if (need_role && same_role && same_host) {
	/* Recovering, restarting or re-promoting a promotable clone instance */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE,
	pcmk_role_text(rsc->private->orig_role),
	PCMK_XA_SOURCE, origin->private->name,
	NULL);

	} else if (same_role && same_host) {
	/* Recovering or Restarting a normal resource */
	crm_xml_add(xml, PCMK_XA_SOURCE, origin->private->name);

	} else if (need_role && same_role) {
	/* Moving a promotable clone instance */
	pcmk__xe_set_props(xml,
	PCMK_XA_SOURCE, origin->private->name,
	PCMK_XA_DEST, destination->private->name,
	PCMK_XA_ROLE,
	pcmk_role_text(rsc->private->orig_role),
	NULL);

	} else if (same_role) {
	/* Moving a normal resource */
	pcmk__xe_set_props(xml,
	PCMK_XA_SOURCE, origin->private->name,
	PCMK_XA_DEST, destination->private->name,
	NULL);

	} else if (same_host) {
	/* Promoting or demoting a promotable clone instance */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE,
	pcmk_role_text(rsc->private->orig_role),
	PCMK_XA_NEXT_ROLE,
	pcmk_role_text(rsc->private->next_role),
	PCMK_XA_SOURCE, origin->private->name,
	NULL);

	} else {
	/* Moving and promoting/demoting */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE,
	pcmk_role_text(rsc->private->orig_role),
	PCMK_XA_SOURCE, origin->private->name,
	PCMK_XA_NEXT_ROLE,
	pcmk_role_text(rsc->private->next_role),
	PCMK_XA_DEST, destination->private->name,
	NULL);
	}

	if ((source->reason != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_runnable)) {
	pcmk__xe_set_props(xml,
	PCMK_XA_REASON, source->reason,
	PCMK_XA_BLOCKED, PCMK_VALUE_TRUE,
	NULL);

	} else if (source->reason != NULL) {
	crm_xml_add(xml, PCMK_XA_REASON, source->reason);

	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	pcmk__xe_set_bool_attr(xml, PCMK_XA_BLOCKED, true);

	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("rsc-is-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rsc_is_colocated_with_list(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk__rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use
	* rsc->private->this_with_colocations directly rather than call
	* rsc->private->cmds->this_with_colocations().
	*/
	pcmk__set_rsc_flags(rsc, pcmk__rsc_detect_loop);
	for (GList *lpc = rsc->private->this_with_colocations;
	lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;
	char *hdr = NULL;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc,
	"Resources %s is colocated with", rsc->id);

	if (pcmk_is_set(cons->primary->flags, pcmk__rsc_detect_loop)) {
	out->list_item(out, NULL, "%s (id=%s - loop)",
	cons->primary->id, cons->id);
	continue;
	}

	hdr = colocations_header(cons->primary, cons, false);
	out->list_item(out, NULL, "%s", hdr);
	free(hdr);

	// Empty list header for indentation of information about this resource
	out->begin_list(out, NULL, NULL, NULL);

	out->message(out, "locations-list", cons->primary);
	if (recursive) {
	out->message(out, "rsc-is-colocated-with-list",
	cons->primary, recursive);
	}

	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("rsc-is-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rsc_is_colocated_with_list_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk__rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use
	* rsc->private->this_with_colocations directly rather than call
	* rsc->private->cmds->this_with_colocations().
	*/
	pcmk__set_rsc_flags(rsc, pcmk__rsc_detect_loop);
	for (GList *lpc = rsc->private->this_with_colocations;
	lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	if (pcmk_is_set(cons->primary->flags, pcmk__rsc_detect_loop)) {
	colocations_xml_node(out, cons->primary, cons);
	continue;
	}

	colocations_xml_node(out, cons->primary, cons);
	do_locations_list_xml(out, cons->primary, false);

	if (recursive) {
	out->message(out, "rsc-is-colocated-with-list",
	cons->primary, recursive);
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("rscs-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rscs_colocated_with_list(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk__rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use
	* rsc->private->with_this_colocations directly rather than
	* rsc->private->cmds->with_this_colocations().
	*/
	pcmk__set_rsc_flags(rsc, pcmk__rsc_detect_loop);
	for (GList *lpc = rsc->private->with_this_colocations;
	lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;
	char *hdr = NULL;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources colocated with %s",
	rsc->id);

	if (pcmk_is_set(cons->dependent->flags, pcmk__rsc_detect_loop)) {
	out->list_item(out, NULL, "%s (id=%s - loop)",
	cons->dependent->id, cons->id);
	continue;
	}

	hdr = colocations_header(cons->dependent, cons, true);
	out->list_item(out, NULL, "%s", hdr);
	free(hdr);

	// Empty list header for indentation of information about this resource
	out->begin_list(out, NULL, NULL, NULL);

	out->message(out, "locations-list", cons->dependent);
	if (recursive) {
	out->message(out, "rscs-colocated-with-list",
	cons->dependent, recursive);
	}

	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("rscs-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rscs_colocated_with_list_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk__rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use
	* rsc->private->with_this_colocations directly rather than
	* rsc->private->cmds->with_this_colocations().
	*/
	pcmk__set_rsc_flags(rsc, pcmk__rsc_detect_loop);
	for (GList *lpc = rsc->private->with_this_colocations;
	lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	if (pcmk_is_set(cons->dependent->flags, pcmk__rsc_detect_loop)) {
	colocations_xml_node(out, cons->dependent, cons);
	continue;
	}

	colocations_xml_node(out, cons->dependent, cons);
	do_locations_list_xml(out, cons->dependent, false);

	if (recursive) {
	out->message(out, "rscs-colocated-with-list",
	cons->dependent, recursive);
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("locations-list", "pcmk_resource_t *")
	static int
	locations_list(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );

	GList *lpc = NULL;
	int rc = pcmk_rc_no_output;

	for (lpc = rsc->private->location_constraints;
	lpc != NULL; lpc = lpc->next) {
	pcmk__location_t *cons = lpc->data;

	GList *lpc2 = NULL;

	for (lpc2 = cons->nodes; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_node_t node = (pcmk_node_t ) lpc2->data;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Locations");
	out->list_item(out, NULL, "Node %s (score=%s, id=%s, rsc=%s)",
	pcmk__node_name(node),
	- pcmk_readable_score(node->weight), cons->id,
	+ pcmk_readable_score(node->assign->score), cons->id,
	rsc->id);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("locations-list", "pcmk_resource_t *")
	static int
	locations_list_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	return do_locations_list_xml(out, rsc, true);
	}

	PCMK__OUTPUT_ARGS("locations-and-colocations", "pcmk_resource_t *",
	"bool", "bool")
	static int
	locations_and_colocations(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);
	bool force = va_arg(args, int);

	pcmk__unpack_constraints(rsc->private->scheduler);

	// Constraints apply to group/clone, not member/instance
	if (!force) {
	rsc = uber_parent(rsc);
	}

	out->message(out, "locations-list", rsc);

	pe__clear_resource_flags_on_all(rsc->private->scheduler,
	pcmk__rsc_detect_loop);
	out->message(out, "rscs-colocated-with-list", rsc, recursive);

	pe__clear_resource_flags_on_all(rsc->private->scheduler,
	pcmk__rsc_detect_loop);
	out->message(out, "rsc-is-colocated-with-list", rsc, recursive);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("locations-and-colocations", "pcmk_resource_t *",
	"bool", "bool")
	static int
	locations_and_colocations_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);
	bool force = va_arg(args, int);

	pcmk__unpack_constraints(rsc->private->scheduler);

	// Constraints apply to group/clone, not member/instance
	if (!force) {
	rsc = uber_parent(rsc);
	}

	pcmk__output_xml_create_parent(out, PCMK_XE_CONSTRAINTS, NULL);
	do_locations_list_xml(out, rsc, false);

	pe__clear_resource_flags_on_all(rsc->private->scheduler,
	pcmk__rsc_detect_loop);
	out->message(out, "rscs-colocated-with-list", rsc, recursive);

	pe__clear_resource_flags_on_all(rsc->private->scheduler,
	pcmk__rsc_detect_loop);
	out->message(out, "rsc-is-colocated-with-list", rsc, recursive);

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("health", "const char ", "const char ", "const char *",
	"const char *")
	static int
	health(pcmk__output_t *out, va_list args)
	{
	const char sys_from G_GNUC_UNUSED = va_arg(args, const char );
	const char host_from = va_arg(args, const char );
	const char fsa_state = va_arg(args, const char );
	const char result = va_arg(args, const char );

	return out->info(out, "Controller on %s in state %s: %s",
	pcmk__s(host_from, "unknown node"),
	pcmk__s(fsa_state, "unknown"),
	pcmk__s(result, "unknown result"));
	}

	PCMK__OUTPUT_ARGS("health", "const char ", "const char ", "const char *",
	"const char *")
	static int
	health_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return health(out, args);
	} else {
	const char sys_from G_GNUC_UNUSED = va_arg(args, const char );
	const char host_from G_GNUC_UNUSED = va_arg(args, const char );
	const char fsa_state = va_arg(args, const char );
	const char result G_GNUC_UNUSED = va_arg(args, const char );

	if (fsa_state != NULL) {
	pcmk__formatted_printf(out, "%s\n", fsa_state);
	return pcmk_rc_ok;
	}
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("health", "const char ", "const char ", "const char *",
	"const char *")
	static int
	health_xml(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	const char host_from = va_arg(args, const char );
	const char fsa_state = va_arg(args, const char );
	const char result = va_arg(args, const char );

	pcmk__output_create_xml_node(out, pcmk__s(sys_from, ""),
	PCMK_XA_NODE_NAME, pcmk__s(host_from, ""),
	PCMK_XA_STATE, pcmk__s(fsa_state, ""),
	PCMK_XA_RESULT, pcmk__s(result, ""),
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated = va_arg(args, time_t);

	char *last_updated_s = NULL;
	int rc = pcmk_rc_ok;

	if (sys_from == NULL) {
	if (state == pcmk_pacemakerd_state_remote) {
	sys_from = "pacemaker-remoted";
	} else {
	sys_from = CRM_SYSTEM_MCP;
	}
	}

	if (state_s == NULL) {
	state_s = pcmk__pcmkd_state_enum2friendly(state);
	}

	if (last_updated != 0) {
	last_updated_s = pcmk__epoch2str(&last_updated,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);
	}

	rc = out->info(out, "Status of %s: '%s' (last updated %s)",
	sys_from, state_s,
	pcmk__s(last_updated_s, "at unknown time"));

	free(last_updated_s);
	return rc;
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health_html(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated = va_arg(args, time_t);

	char *last_updated_s = NULL;
	char *msg = NULL;

	if (sys_from == NULL) {
	if (state == pcmk_pacemakerd_state_remote) {
	sys_from = "pacemaker-remoted";
	} else {
	sys_from = CRM_SYSTEM_MCP;
	}
	}

	if (state_s == NULL) {
	state_s = pcmk__pcmkd_state_enum2friendly(state);
	}

	if (last_updated != 0) {
	last_updated_s = pcmk__epoch2str(&last_updated,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);
	}

	msg = crm_strdup_printf("Status of %s: '%s' (last updated %s)",
	sys_from, state_s,
	pcmk__s(last_updated_s, "at unknown time"));
	pcmk__output_create_html_node(out, "li", NULL, NULL, msg);

	free(msg);
	free(last_updated_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return pacemakerd_health(out, args);
	} else {
	const char sys_from G_GNUC_UNUSED = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated G_GNUC_UNUSED = va_arg(args, time_t);

	if (state_s == NULL) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
	}
	pcmk__formatted_printf(out, "%s\n", state_s);
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health_xml(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated = va_arg(args, time_t);

	char *last_updated_s = NULL;

	if (sys_from == NULL) {
	if (state == pcmk_pacemakerd_state_remote) {
	sys_from = "pacemaker-remoted";
	} else {
	sys_from = CRM_SYSTEM_MCP;
	}
	}

	if (state_s == NULL) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
	}

	if (last_updated != 0) {
	last_updated_s = pcmk__epoch2str(&last_updated,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);
	}

	pcmk__output_create_xml_node(out, PCMK_XE_PACEMAKERD,
	PCMK_XA_SYS_FROM, sys_from,
	PCMK_XA_STATE, state_s,
	PCMK_XA_LAST_UPDATED, last_updated_s,
	NULL);
	free(last_updated_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("profile", "const char *", "clock_t", "clock_t")
	static int
	profile_default(pcmk__output_t *out, va_list args) {
	const char xml_file = va_arg(args, const char );
	clock_t start = va_arg(args, clock_t);
	clock_t end = va_arg(args, clock_t);

	out->list_item(out, NULL, "Testing %s ... %.2f secs", xml_file,
	(end - start) / (float) CLOCKS_PER_SEC);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("profile", "const char *", "clock_t", "clock_t")
	static int
	profile_xml(pcmk__output_t *out, va_list args) {
	const char xml_file = va_arg(args, const char );
	clock_t start = va_arg(args, clock_t);
	clock_t end = va_arg(args, clock_t);

	char *duration = pcmk__ftoa((end - start) / (float) CLOCKS_PER_SEC);

	pcmk__output_create_xml_node(out, PCMK_XE_TIMING,
	PCMK_XA_FILE, xml_file,
	PCMK_XA_DURATION, duration,
	NULL);

	free(duration);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("dc", "const char *")
	static int
	dc(pcmk__output_t *out, va_list args)
	{
	const char dc = va_arg(args, const char );

	return out->info(out, "Designated Controller is: %s",
	pcmk__s(dc, "not yet elected"));
	}

	PCMK__OUTPUT_ARGS("dc", "const char *")
	static int
	dc_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return dc(out, args);
	} else {
	const char dc = va_arg(args, const char );

	if (dc != NULL) {
	pcmk__formatted_printf(out, "%s\n", pcmk__s(dc, ""));
	return pcmk_rc_ok;
	}
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("dc", "const char *")
	static int
	dc_xml(pcmk__output_t *out, va_list args)
	{
	const char dc = va_arg(args, const char );

	pcmk__output_create_xml_node(out, PCMK_XE_DC,
	PCMK_XA_NODE_NAME, pcmk__s(dc, ""),
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("crmadmin-node", "const char ", "const char ",
	"const char *", "bool")
	static int
	crmadmin_node(pcmk__output_t *out, va_list args)
	{
	const char type = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char id = va_arg(args, const char );
	bool bash_export = va_arg(args, int);

	if (bash_export) {
	return out->info(out, "export %s=%s",
	pcmk__s(name, "<null>"), pcmk__s(id, ""));
	} else {
	return out->info(out, "%s node: %s (%s)", type ? type : "cluster",
	pcmk__s(name, "<null>"), pcmk__s(id, "<null>"));
	}
	}

	PCMK__OUTPUT_ARGS("crmadmin-node", "const char ", "const char ",
	"const char *", "bool")
	static int
	crmadmin_node_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return crmadmin_node(out, args);
	} else {
	const char type G_GNUC_UNUSED = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char id G_GNUC_UNUSED = va_arg(args, const char );
	bool bash_export G_GNUC_UNUSED = va_arg(args, int);

	pcmk__formatted_printf(out, "%s\n", pcmk__s(name, "<null>"));
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("crmadmin-node", "const char ", "const char ",
	"const char *", "bool")
	static int
	crmadmin_node_xml(pcmk__output_t *out, va_list args)
	{
	const char type = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char id = va_arg(args, const char );
	bool bash_export G_GNUC_UNUSED = va_arg(args, int);

	pcmk__output_create_xml_node(out, PCMK_XE_NODE,
	PCMK_XA_TYPE, pcmk__s(type, "cluster"),
	PCMK_XA_NAME, pcmk__s(name, ""),
	PCMK_XA_ID, pcmk__s(id, ""),
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("digests", "const pcmk_resource_t ", "const pcmk_node_t ",
	"const char ", "guint", "const pcmk__op_digest_t ")
	static int
	digests_text(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const pcmk_node_t node = va_arg(args, const pcmk_node_t );
	const char task = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);
	const pcmk__op_digest_t digests = va_arg(args, const pcmk__op_digest_t );

	char *action_desc = NULL;
	const char *rsc_desc = "unknown resource";
	const char *node_desc = "unknown node";

	if (interval_ms != 0) {
	action_desc = crm_strdup_printf("%ums-interval %s action", interval_ms,
	((task == NULL)? "unknown" : task));
	} else if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	action_desc = strdup("probe action");
	} else {
	action_desc = crm_strdup_printf("%s action",
	((task == NULL)? "unknown" : task));
	}
	if ((rsc != NULL) && (rsc->id != NULL)) {
	rsc_desc = rsc->id;
	}
	if ((node != NULL) && (node->private->name != NULL)) {
	node_desc = node->private->name;
	}
	out->begin_list(out, NULL, NULL, "Digests for %s %s on %s",
	rsc_desc, action_desc, node_desc);
	free(action_desc);

	if (digests == NULL) {
	out->list_item(out, NULL, "none");
	out->end_list(out);
	return pcmk_rc_ok;
	}
	if (digests->digest_all_calc != NULL) {
	out->list_item(out, NULL, "%s (all parameters)",
	digests->digest_all_calc);
	}
	if (digests->digest_secure_calc != NULL) {
	out->list_item(out, NULL, "%s (non-private parameters)",
	digests->digest_secure_calc);
	}
	if (digests->digest_restart_calc != NULL) {
	out->list_item(out, NULL, "%s (non-reloadable parameters)",
	digests->digest_restart_calc);
	}
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static void
	add_digest_xml(xmlNode parent, const char type, const char *digest,
	xmlNode *digest_source)
	{
	if (digest != NULL) {
	xmlNodePtr digest_xml = pcmk__xe_create(parent, PCMK_XE_DIGEST);

	crm_xml_add(digest_xml, PCMK_XA_TYPE, pcmk__s(type, "unspecified"));
	crm_xml_add(digest_xml, PCMK_XA_HASH, digest);
	pcmk__xml_copy(digest_xml, digest_source);
	}
	}

	PCMK__OUTPUT_ARGS("digests", "const pcmk_resource_t ", "const pcmk_node_t ",
	"const char ", "guint", "const pcmk__op_digest_t ")
	static int
	digests_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const pcmk_node_t node = va_arg(args, const pcmk_node_t );
	const char task = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);
	const pcmk__op_digest_t digests = va_arg(args, const pcmk__op_digest_t );

	char *interval_s = crm_strdup_printf("%ums", interval_ms);
	xmlNode *xml = NULL;

	xml = pcmk__output_create_xml_node(out, PCMK_XE_DIGESTS,
	PCMK_XA_RESOURCE, pcmk__s(rsc->id, ""),
	PCMK_XA_NODE,
	pcmk__s(node->private->name, ""),
	PCMK_XA_TASK, pcmk__s(task, ""),
	PCMK_XA_INTERVAL, interval_s,
	NULL);
	free(interval_s);
	if (digests != NULL) {
	add_digest_xml(xml, "all", digests->digest_all_calc,
	digests->params_all);
	add_digest_xml(xml, "nonprivate", digests->digest_secure_calc,
	digests->params_secure);
	add_digest_xml(xml, "nonreloadable", digests->digest_restart_calc,
	digests->params_restart);
	}
	return pcmk_rc_ok;
	}

	#define STOP_SANITY_ASSERT(lineno) do { \
	if ((current != NULL) && current->details->unclean) { \
	/* It will be a pseudo op */ \
	} else if (stop == NULL) { \
	crm_err("%s:%d: No stop action exists for %s", \
	__func__, lineno, rsc->id); \
	CRM_ASSERT(stop != NULL); \
	} else if (pcmk_is_set(stop->flags, pcmk_action_optional)) { \
	crm_err("%s:%d: Action %s is still optional", \
	__func__, lineno, stop->uuid); \
	CRM_ASSERT(!pcmk_is_set(stop->flags, pcmk_action_optional));\
	} \
	} while (0)

	PCMK__OUTPUT_ARGS("rsc-action", "pcmk_resource_t ", "pcmk_node_t ",
	"pcmk_node_t *")
	static int
	rsc_action_default(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t current = va_arg(args, pcmk_node_t );
	pcmk_node_t next = va_arg(args, pcmk_node_t );

	GList *possible_matches = NULL;
	char *key = NULL;
	int rc = pcmk_rc_no_output;
	bool moving = false;

	pcmk_node_t *start_node = NULL;
	pcmk_action_t *start = NULL;
	pcmk_action_t *stop = NULL;
	pcmk_action_t *promote = NULL;
	pcmk_action_t *demote = NULL;
	pcmk_action_t *reason_op = NULL;

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)
	\|\| (current == NULL && next == NULL)) {
	const bool managed = pcmk_is_set(rsc->flags, pcmk__rsc_managed);

	pcmk__rsc_info(rsc, "Leave %s\t(%s%s)",
	rsc->id, pcmk_role_text(rsc->private->orig_role),
	(managed? "" : " unmanaged"));
	return rc;
	}

	moving = (current != NULL) && (next != NULL)
	&& !pcmk__same_node(current, next);

	possible_matches = pe__resource_actions(rsc, next, PCMK_ACTION_START,
	false);
	if (possible_matches) {
	start = possible_matches->data;
	g_list_free(possible_matches);
	}

	if ((start == NULL)
	\|\| !pcmk_is_set(start->flags, pcmk_action_runnable)) {
	start_node = NULL;
	} else {
	start_node = current;
	}
	possible_matches = pe__resource_actions(rsc, start_node, PCMK_ACTION_STOP,
	false);
	if (possible_matches) {
	stop = possible_matches->data;
	g_list_free(possible_matches);
	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_stop_unexpected)) {
	/* The resource is multiply active with PCMK_META_MULTIPLE_ACTIVE set to
	* PCMK_VALUE_STOP_UNEXPECTED, and not stopping on its current node, but
	* it should be stopping elsewhere.
	*/
	possible_matches = pe__resource_actions(rsc, NULL, PCMK_ACTION_STOP,
	false);
	if (possible_matches != NULL) {
	stop = possible_matches->data;
	g_list_free(possible_matches);
	}
	}

	possible_matches = pe__resource_actions(rsc, next, PCMK_ACTION_PROMOTE,
	false);
	if (possible_matches) {
	promote = possible_matches->data;
	g_list_free(possible_matches);
	}

	possible_matches = pe__resource_actions(rsc, next, PCMK_ACTION_DEMOTE,
	false);
	if (possible_matches) {
	demote = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (rsc->private->orig_role == rsc->private->next_role) {
	pcmk_action_t *migrate_op = NULL;

	CRM_CHECK(next != NULL, return rc);

	possible_matches = pe__resource_actions(rsc, next,
	PCMK_ACTION_MIGRATE_FROM,
	false);
	if (possible_matches) {
	migrate_op = possible_matches->data;
	}

	if ((migrate_op != NULL) && (current != NULL)
	&& pcmk_is_set(migrate_op->flags, pcmk_action_runnable)) {
	rc = out->message(out, "rsc-action-item", "Migrate", rsc, current,
	next, start, NULL);

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_reload)) {
	rc = out->message(out, "rsc-action-item", "Reload", rsc, current,
	next, start, NULL);

	} else if ((start == NULL)
	\|\| pcmk_is_set(start->flags, pcmk_action_optional)) {
	if ((demote != NULL) && (promote != NULL)
	&& !pcmk_is_set(demote->flags, pcmk_action_optional)
	&& !pcmk_is_set(promote->flags, pcmk_action_optional)) {
	rc = out->message(out, "rsc-action-item", "Re-promote", rsc,
	current, next, promote, demote);
	} else {
	pcmk__rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id,
	pcmk_role_text(rsc->private->orig_role),
	pcmk__node_name(next));
	}

	} else if (!pcmk_is_set(start->flags, pcmk_action_runnable)) {
	if ((stop == NULL) \|\| (stop->reason == NULL)) {
	reason_op = start;
	} else {
	reason_op = stop;
	}
	rc = out->message(out, "rsc-action-item", "Stop", rsc, current,
	NULL, stop, reason_op);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (moving && current) {
	const bool failed = pcmk_is_set(rsc->flags, pcmk__rsc_failed);

	rc = out->message(out, "rsc-action-item",
	(failed? "Recover" : "Move"), rsc, current, next,
	stop, NULL);

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
	rc = out->message(out, "rsc-action-item", "Recover", rsc, current,
	NULL, stop, NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else {
	rc = out->message(out, "rsc-action-item", "Restart", rsc, current,
	next, start, NULL);
	#if 0
	/* @TODO This can be reached in situations that should really be
	* "Start" (see for example the migrate-fail-7 regression test)
	*/
	STOP_SANITY_ASSERT(__LINE__);
	#endif
	}

	g_list_free(possible_matches);
	return rc;
	}

	if ((stop != NULL)
	&& ((rsc->private->next_role == pcmk_role_stopped)
	\|\| ((start != NULL)
	&& !pcmk_is_set(start->flags, pcmk_action_runnable)))) {

	key = stop_key(rsc);
	for (GList *iter = rsc->private->active_nodes;
	iter != NULL; iter = iter->next) {

	pcmk_node_t *node = iter->data;
	pcmk_action_t *stop_op = NULL;

	reason_op = start;
	possible_matches = find_actions(rsc->private->actions, key, node);
	if (possible_matches) {
	stop_op = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (stop_op != NULL) {
	if (pcmk_is_set(stop_op->flags, pcmk_action_runnable)) {
	STOP_SANITY_ASSERT(__LINE__);
	}
	if (stop_op->reason != NULL) {
	reason_op = stop_op;
	}
	}

	if (out->message(out, "rsc-action-item", "Stop", rsc, node, NULL,
	stop_op, reason_op) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	free(key);

	} else if ((stop != NULL)
	&& pcmk_all_flags_set(rsc->flags,
	pcmk__rsc_failed
	\|pcmk__rsc_stop_if_failed)) {
	/* 'stop' may be NULL if the failure was ignored */
	rc = out->message(out, "rsc-action-item", "Recover", rsc, current,
	next, stop, start);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (moving) {
	rc = out->message(out, "rsc-action-item", "Move", rsc, current, next,
	stop, NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_reload)) {
	rc = out->message(out, "rsc-action-item", "Reload", rsc, current, next,
	start, NULL);

	} else if ((stop != NULL)
	&& !pcmk_is_set(stop->flags, pcmk_action_optional)) {
	rc = out->message(out, "rsc-action-item", "Restart", rsc, current,
	next, start, NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (rsc->private->orig_role == pcmk_role_promoted) {
	CRM_LOG_ASSERT(current != NULL);
	rc = out->message(out, "rsc-action-item", "Demote", rsc, current,
	next, demote, NULL);

	} else if (rsc->private->next_role == pcmk_role_promoted) {
	CRM_LOG_ASSERT(next);
	rc = out->message(out, "rsc-action-item", "Promote", rsc, current,
	next, promote, NULL);

	} else if ((rsc->private->orig_role == pcmk_role_stopped)
	&& (rsc->private->next_role > pcmk_role_stopped)) {
	rc = out->message(out, "rsc-action-item", "Start", rsc, current, next,
	start, NULL);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("node-action", "const char ", "const char ", "const char *")
	static int
	node_action(pcmk__output_t *out, va_list args)
	{
	const char task = va_arg(args, const char );
	const char node_name = va_arg(args, const char );
	const char reason = va_arg(args, const char );

	if (task == NULL) {
	return pcmk_rc_no_output;
	} else if (reason) {
	out->list_item(out, NULL, "%s %s '%s'", task, node_name, reason);
	} else {
	crm_notice(" * %s %s", task, node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-action", "const char ", "const char ", "const char *")
	static int
	node_action_xml(pcmk__output_t *out, va_list args)
	{
	const char task = va_arg(args, const char );
	const char node_name = va_arg(args, const char );
	const char reason = va_arg(args, const char );

	if (task == NULL) {
	return pcmk_rc_no_output;
	} else if (reason) {
	pcmk__output_create_xml_node(out, PCMK_XE_NODE_ACTION,
	PCMK_XA_TASK, task,
	PCMK_XA_NODE, node_name,
	PCMK_XA_REASON, reason,
	NULL);
	} else {
	crm_notice(" * %s %s", task, node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-info", "uint32_t", "const char ", "const char ",
	"const char *", "bool", "bool")
	static int
	node_info_default(pcmk__output_t *out, va_list args)
	{
	uint32_t node_id = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );
	const char uuid = va_arg(args, const char );
	const char state = va_arg(args, const char );
	bool have_quorum = (bool) va_arg(args, int);
	bool is_remote = (bool) va_arg(args, int);

	return out->info(out,
	"Node %" PRIu32 ": %s "
	"(uuid=%s, state=%s, have_quorum=%s, is_remote=%s)",
	node_id, pcmk__s(node_name, "unknown"),
	pcmk__s(uuid, "unknown"), pcmk__s(state, "unknown"),
	pcmk__btoa(have_quorum), pcmk__btoa(is_remote));
	}

	PCMK__OUTPUT_ARGS("node-info", "uint32_t", "const char ", "const char ",
	"const char *", "bool", "bool")
	static int
	node_info_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t node_id = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );
	const char uuid = va_arg(args, const char );
	const char state = va_arg(args, const char );
	bool have_quorum = (bool) va_arg(args, int);
	bool is_remote = (bool) va_arg(args, int);

	char *id_s = crm_strdup_printf("%" PRIu32, node_id);

	pcmk__output_create_xml_node(out, PCMK_XE_NODE_INFO,
	PCMK_XA_NODEID, id_s,
	PCMK_XA_UNAME, node_name,
	PCMK_XA_ID, uuid,
	PCMK_XA_CRMD, state,
	PCMK_XA_HAVE_QUORUM, pcmk__btoa(have_quorum),
	PCMK_XA_REMOTE_NODE, pcmk__btoa(is_remote),
	NULL);
	free(id_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-cluster-action", "const char ", "const char ",
	"xmlNode *")
	static int
	inject_cluster_action(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );
	xmlNodePtr rsc = va_arg(args, xmlNodePtr);

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (rsc != NULL) {
	out->list_item(out, NULL, "Cluster action: %s for %s on %s",
	task, pcmk__xe_id(rsc), node);
	} else {
	out->list_item(out, NULL, "Cluster action: %s on %s", task, node);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-cluster-action", "const char ", "const char ",
	"xmlNode *")
	static int
	inject_cluster_action_xml(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );
	xmlNodePtr rsc = va_arg(args, xmlNodePtr);

	xmlNodePtr xml_node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	xml_node = pcmk__output_create_xml_node(out, PCMK_XE_CLUSTER_ACTION,
	PCMK_XA_TASK, task,
	PCMK_XA_NODE, node,
	NULL);

	if (rsc) {
	crm_xml_add(xml_node, PCMK_XA_ID, pcmk__xe_id(rsc));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-fencing-action", "const char ", "const char ")
	static int
	inject_fencing_action(pcmk__output_t *out, va_list args)
	{
	const char target = va_arg(args, const char );
	const char op = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->list_item(out, NULL, "Fencing %s (%s)", target, op);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-fencing-action", "const char ", "const char ")
	static int
	inject_fencing_action_xml(pcmk__output_t *out, va_list args)
	{
	const char target = va_arg(args, const char );
	const char op = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, PCMK_XE_FENCING_ACTION,
	PCMK_XA_TARGET, target,
	PCMK_XA_OP, op,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-attr", "const char ", "const char ", "xmlNode *")
	static int
	inject_attr(pcmk__output_t *out, va_list args)
	{
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	xmlNodePtr cib_node = va_arg(args, xmlNodePtr);

	xmlChar *node_path = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	node_path = xmlGetNodePath(cib_node);

	out->list_item(out, NULL, "Injecting attribute %s=%s into %s '%s'",
	name, value, node_path, pcmk__xe_id(cib_node));

	free(node_path);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-attr", "const char ", "const char ", "xmlNode *")
	static int
	inject_attr_xml(pcmk__output_t *out, va_list args)
	{
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	xmlNodePtr cib_node = va_arg(args, xmlNodePtr);

	xmlChar *node_path = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	node_path = xmlGetNodePath(cib_node);

	pcmk__output_create_xml_node(out, PCMK_XE_INJECT_ATTR,
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	PCMK_XA_NODE_PATH, node_path,
	PCMK_XA_CIB_NODE, pcmk__xe_id(cib_node),
	NULL);
	free(node_path);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-spec", "const char *")
	static int
	inject_spec(pcmk__output_t *out, va_list args)
	{
	const char spec = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->list_item(out, NULL, "Injecting %s into the configuration", spec);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-spec", "const char *")
	static int
	inject_spec_xml(pcmk__output_t *out, va_list args)
	{
	const char spec = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, PCMK_XE_INJECT_SPEC,
	PCMK_XA_SPEC, spec,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-config", "const char ", "const char ")
	static int
	inject_modify_config(pcmk__output_t *out, va_list args)
	{
	const char quorum = va_arg(args, const char );
	const char watchdog = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Performing Requested Modifications");

	if (quorum) {
	out->list_item(out, NULL, "Setting quorum: %s", quorum);
	}

	if (watchdog) {
	out->list_item(out, NULL, "Setting watchdog: %s", watchdog);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-config", "const char ", "const char ")
	static int
	inject_modify_config_xml(pcmk__output_t *out, va_list args)
	{
	const char quorum = va_arg(args, const char );
	const char watchdog = va_arg(args, const char );

	xmlNodePtr node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	node = pcmk__output_xml_create_parent(out, PCMK_XE_MODIFICATIONS, NULL);

	if (quorum) {
	crm_xml_add(node, PCMK_XA_QUORUM, quorum);
	}

	if (watchdog) {
	crm_xml_add(node, PCMK_XA_WATCHDOG, watchdog);
	}

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-node", "const char ", "const char ")
	static int
	inject_modify_node(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char node = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (pcmk__str_eq(action, "Online", pcmk__str_none)) {
	out->list_item(out, NULL, "Bringing node %s online", node);
	return pcmk_rc_ok;
	} else if (pcmk__str_eq(action, "Offline", pcmk__str_none)) {
	out->list_item(out, NULL, "Taking node %s offline", node);
	return pcmk_rc_ok;
	} else if (pcmk__str_eq(action, "Failing", pcmk__str_none)) {
	out->list_item(out, NULL, "Failing node %s", node);
	return pcmk_rc_ok;
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("inject-modify-node", "const char ", "const char ")
	static int
	inject_modify_node_xml(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char node = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, PCMK_XE_MODIFY_NODE,
	PCMK_XA_ACTION, action,
	PCMK_XA_NODE, node,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-ticket", "const char ", "const char ")
	static int
	inject_modify_ticket(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char ticket = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (pcmk__str_eq(action, "Standby", pcmk__str_none)) {
	out->list_item(out, NULL, "Making ticket %s standby", ticket);
	} else {
	out->list_item(out, NULL, "%s ticket %s", action, ticket);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-ticket", "const char ", "const char ")
	static int
	inject_modify_ticket_xml(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char ticket = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, PCMK_XE_MODIFY_TICKET,
	PCMK_XA_ACTION, action,
	PCMK_XA_TICKET, ticket,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-pseudo-action", "const char ", "const char ")
	static int
	inject_pseudo_action(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->list_item(out, NULL, "Pseudo action: %s%s%s",
	task, ((node == NULL)? "" : " on "), pcmk__s(node, ""));
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-pseudo-action", "const char ", "const char ")
	static int
	inject_pseudo_action_xml(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );

	xmlNodePtr xml_node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	xml_node = pcmk__output_create_xml_node(out, PCMK_XE_PSEUDO_ACTION,
	PCMK_XA_TASK, task,
	NULL);
	if (node) {
	crm_xml_add(xml_node, PCMK_XA_NODE, node);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-rsc-action", "const char ", "const char ",
	"const char *", "guint")
	static int
	inject_rsc_action(pcmk__output_t *out, va_list args)
	{
	const char rsc = va_arg(args, const char );
	const char operation = va_arg(args, const char );
	const char node = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (interval_ms) {
	out->list_item(out, NULL, "Resource action: %-15s %s=%u on %s",
	rsc, operation, interval_ms, node);
	} else {
	out->list_item(out, NULL, "Resource action: %-15s %s on %s",
	rsc, operation, node);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-rsc-action", "const char ", "const char ",
	"const char *", "guint")
	static int
	inject_rsc_action_xml(pcmk__output_t *out, va_list args)
	{
	const char rsc = va_arg(args, const char );
	const char operation = va_arg(args, const char );
	const char node = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);

	xmlNodePtr xml_node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	xml_node = pcmk__output_create_xml_node(out, PCMK_XE_RSC_ACTION,
	PCMK_XA_RESOURCE, rsc,
	PCMK_XA_OP, operation,
	PCMK_XA_NODE, node,
	NULL);

	if (interval_ms) {
	char *interval_s = pcmk__itoa(interval_ms);

	crm_xml_add(xml_node, PCMK_XA_INTERVAL, interval_s);
	free(interval_s);
	}

	return pcmk_rc_ok;
	}

	#define CHECK_RC(retcode, retval) \
	if (retval == pcmk_rc_ok) { \
	retcode = pcmk_rc_ok; \
	}

	PCMK__OUTPUT_ARGS("cluster-status", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "crm_exit_t",
	"stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
	"uint32_t", "const char ", "GList ", "GList *")
	int
	pcmk__cluster_status_text(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t stonith_history = va_arg(args, stonith_history_t );
	enum pcmk__fence_history fence_history = va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	const char prefix = va_arg(args, const char );
	GList unames = va_arg(args, GList );
	GList resources = va_arg(args, GList );

	int rc = pcmk_rc_no_output;
	bool already_printed_failure = false;

	CHECK_RC(rc, out->message(out, "cluster-summary", scheduler, pcmkd_state,
	section_opts, show_opts));

	if (pcmk_is_set(section_opts, pcmk_section_nodes) && unames) {
	CHECK_RC(rc, out->message(out, "node-list", scheduler->nodes, unames,
	resources, show_opts, rc == pcmk_rc_ok));
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(section_opts, pcmk_section_resources)) {
	CHECK_RC(rc, out->message(out, "resource-list", scheduler, show_opts,
	true, unames, resources, rc == pcmk_rc_ok));
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(section_opts, pcmk_section_attributes)) {
	CHECK_RC(rc, out->message(out, "node-attribute-list", scheduler,
	show_opts, (rc == pcmk_rc_ok), unames,
	resources));
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_any_flags_set(section_opts,
	pcmk_section_operations\|pcmk_section_failcounts)) {
	CHECK_RC(rc, out->message(out, "node-summary", scheduler, unames,
	resources, section_opts, show_opts,
	(rc == pcmk_rc_ok)));
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(section_opts, pcmk_section_failures)
	&& (scheduler->failed != NULL)
	&& (scheduler->failed->children != NULL)) {

	CHECK_RC(rc, out->message(out, "failed-action-list", scheduler, unames,
	resources, show_opts, rc == pcmk_rc_ok));
	}

	/* Print failed stonith actions */
	if (pcmk_is_set(section_opts, pcmk_section_fence_failed) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc == 0) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_eq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	CHECK_RC(rc, out->message(out, "failed-fencing-list",
	stonith_history, unames, section_opts,
	show_opts, rc == pcmk_rc_ok));
	}
	} else {
	PCMK__OUTPUT_SPACER_IF(out, rc == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);

	already_printed_failure = true;
	}
	}

	/* Print tickets if requested */
	if (pcmk_is_set(section_opts, pcmk_section_tickets)) {
	CHECK_RC(rc, out->message(out, "ticket-list", scheduler->tickets,
	(rc == pcmk_rc_ok), false, false));
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(section_opts, pcmk_section_bans)) {
	CHECK_RC(rc, out->message(out, "ban-list", scheduler, prefix, resources,
	show_opts, rc == pcmk_rc_ok));
	}

	/* Print stonith history */
	if (pcmk_any_flags_set(section_opts, pcmk_section_fencing_all) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc != 0) {
	if (!already_printed_failure) {
	PCMK__OUTPUT_SPACER_IF(out, rc == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_worked)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_neq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	CHECK_RC(rc, out->message(out, "fencing-list", hp, unames,
	section_opts, show_opts,
	rc == pcmk_rc_ok));
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_pending)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_pending,
	NULL);
	if (hp) {
	CHECK_RC(rc, out->message(out, "pending-fencing-list", hp,
	unames, section_opts, show_opts,
	rc == pcmk_rc_ok));
	}
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-status", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "crm_exit_t",
	"stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
	"uint32_t", "const char ", "GList ", "GList *")
	static int
	cluster_status_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t stonith_history = va_arg(args, stonith_history_t );
	enum pcmk__fence_history fence_history = va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	const char prefix = va_arg(args, const char );
	GList unames = va_arg(args, GList );
	GList resources = va_arg(args, GList );

	out->message(out, "cluster-summary", scheduler, pcmkd_state, section_opts,
	show_opts);

	/* NODES */
	if (pcmk_is_set(section_opts, pcmk_section_nodes)) {
	out->message(out, "node-list", scheduler->nodes, unames, resources,
	show_opts, false);
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(section_opts, pcmk_section_resources)) {
	/* XML output always displays full details. */
	uint32_t full_show_opts = show_opts & ~pcmk_show_brief;

	out->message(out, "resource-list", scheduler, full_show_opts,
	false, unames, resources, false);
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(section_opts, pcmk_section_attributes)) {
	out->message(out, "node-attribute-list", scheduler, show_opts, false,
	unames, resources);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_any_flags_set(section_opts,
	pcmk_section_operations\|pcmk_section_failcounts)) {
	out->message(out, "node-summary", scheduler, unames,
	resources, section_opts, show_opts, false);
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(section_opts, pcmk_section_failures)
	&& (scheduler->failed != NULL)
	&& (scheduler->failed->children != NULL)) {

	out->message(out, "failed-action-list", scheduler, unames, resources,
	show_opts, false);
	}

	/* Print stonith history */
	if (pcmk_is_set(section_opts, pcmk_section_fencing_all) &&
	fence_history != pcmk__fence_history_none) {
	out->message(out, "full-fencing-list", history_rc, stonith_history,
	unames, section_opts, show_opts, false);
	}

	/* Print tickets if requested */
	if (pcmk_is_set(section_opts, pcmk_section_tickets)) {
	out->message(out, "ticket-list", scheduler->tickets, false, false, false);
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(section_opts, pcmk_section_bans)) {
	out->message(out, "ban-list", scheduler, prefix, resources, show_opts,
	false);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-status", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "crm_exit_t",
	"stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
	"uint32_t", "const char ", "GList ", "GList *")
	static int
	cluster_status_html(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t stonith_history = va_arg(args, stonith_history_t );
	enum pcmk__fence_history fence_history = va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	const char prefix = va_arg(args, const char );
	GList unames = va_arg(args, GList );
	GList resources = va_arg(args, GList );
	bool already_printed_failure = false;

	out->message(out, "cluster-summary", scheduler, pcmkd_state, section_opts,
	show_opts);

	/* NODE LIST */
	if (pcmk_is_set(section_opts, pcmk_section_nodes) && unames) {
	out->message(out, "node-list", scheduler->nodes, unames, resources,
	show_opts, false);
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(section_opts, pcmk_section_resources)) {
	out->message(out, "resource-list", scheduler, show_opts, true, unames,
	resources, false);
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(section_opts, pcmk_section_attributes)) {
	out->message(out, "node-attribute-list", scheduler, show_opts, false,
	unames, resources);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_any_flags_set(section_opts,
	pcmk_section_operations\|pcmk_section_failcounts)) {
	out->message(out, "node-summary", scheduler, unames,
	resources, section_opts, show_opts, false);
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(section_opts, pcmk_section_failures)
	&& (scheduler->failed != NULL)
	&& (scheduler->failed->children != NULL)) {

	out->message(out, "failed-action-list", scheduler, unames, resources,
	show_opts, false);
	}

	/* Print failed stonith actions */
	if (pcmk_is_set(section_opts, pcmk_section_fence_failed) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc == 0) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_eq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	out->message(out, "failed-fencing-list", stonith_history,
	unames, section_opts, show_opts, false);
	}
	} else {
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);
	}
	}

	/* Print stonith history */
	if (pcmk_any_flags_set(section_opts, pcmk_section_fencing_all) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc != 0) {
	if (!already_printed_failure) {
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_worked)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_neq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	out->message(out, "fencing-list", hp, unames, section_opts,
	show_opts, false);
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_pending)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_pending,
	NULL);
	if (hp) {
	out->message(out, "pending-fencing-list", hp, unames,
	section_opts, show_opts, false);
	}
	}
	}

	/* Print tickets if requested */
	if (pcmk_is_set(section_opts, pcmk_section_tickets)) {
	out->message(out, "ticket-list", scheduler->tickets, false, false, false);
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(section_opts, pcmk_section_bans)) {
	out->message(out, "ban-list", scheduler, prefix, resources, show_opts,
	false);
	}

	return pcmk_rc_ok;
	}

	#define KV_PAIR(k, v) do { \
	if (legacy) { \
	pcmk__g_strcat(s, k "=", pcmk__s(v, ""), " ", NULL); \
	} else { \
	pcmk__g_strcat(s, k "=\"", pcmk__s(v, ""), "\" ", NULL); \
	} \
	} while (0)

	PCMK__OUTPUT_ARGS("attribute", "const char ", "const char ", "const char *",
	"const char ", "const char ", "bool", "bool")
	static int
	attribute_default(pcmk__output_t *out, va_list args)
	{
	const char scope = va_arg(args, const char );
	const char instance = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	const char host = va_arg(args, const char );
	bool quiet = va_arg(args, int);
	bool legacy = va_arg(args, int);

	gchar *value_esc = NULL;
	GString *s = NULL;

	if (quiet) {
	if (value != NULL) {
	/* Quiet needs to be turned off for ->info() to do anything */
	bool was_quiet = out->is_quiet(out);

	if (was_quiet) {
	out->quiet = false;
	}

	out->info(out, "%s", value);

	out->quiet = was_quiet;
	}

	return pcmk_rc_ok;
	}

	s = g_string_sized_new(50);

	if (pcmk__xml_needs_escape(value, pcmk__xml_escape_attr_pretty)) {
	value_esc = pcmk__xml_escape(value, pcmk__xml_escape_attr_pretty);
	value = value_esc;
	}

	if (!pcmk__str_empty(scope)) {
	KV_PAIR(PCMK_XA_SCOPE, scope);
	}

	if (!pcmk__str_empty(instance)) {
	KV_PAIR(PCMK_XA_ID, instance);
	}

	KV_PAIR(PCMK_XA_NAME, name);

	if (!pcmk__str_empty(host)) {
	KV_PAIR(PCMK_XA_HOST, host);
	}

	if (legacy) {
	pcmk__g_strcat(s, PCMK_XA_VALUE "=", pcmk__s(value, "(null)"), NULL);
	} else {
	pcmk__g_strcat(s, PCMK_XA_VALUE "=\"", pcmk__s(value, ""), "\"", NULL);
	}

	out->info(out, "%s", s->str);

	g_free(value_esc);
	g_string_free(s, TRUE);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute", "const char ", "const char ", "const char *",
	"const char ", "const char ", "bool", "bool")
	static int
	attribute_xml(pcmk__output_t *out, va_list args)
	{
	const char scope = va_arg(args, const char );
	const char instance = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	const char host = va_arg(args, const char );
	bool quiet G_GNUC_UNUSED = va_arg(args, int);
	bool legacy G_GNUC_UNUSED = va_arg(args, int);

	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, PCMK_XE_ATTRIBUTE,
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, pcmk__s(value, ""),
	NULL);

	if (!pcmk__str_empty(scope)) {
	crm_xml_add(node, PCMK_XA_SCOPE, scope);
	}

	if (!pcmk__str_empty(instance)) {
	crm_xml_add(node, PCMK_XA_ID, instance);
	}

	if (!pcmk__str_empty(host)) {
	crm_xml_add(node, PCMK_XA_HOST, host);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("rule-check", "const char ", "int", "const char ")
	static int
	rule_check_default(pcmk__output_t *out, va_list args)
	{
	const char rule_id = va_arg(args, const char );
	int result = va_arg(args, int);
	const char error = va_arg(args, const char );

	switch (result) {
	case pcmk_rc_within_range:
	return out->info(out, "Rule %s is still in effect", rule_id);
	case pcmk_rc_ok:
	return out->info(out, "Rule %s satisfies conditions", rule_id);
	case pcmk_rc_after_range:
	return out->info(out, "Rule %s is expired", rule_id);
	case pcmk_rc_before_range:
	return out->info(out, "Rule %s has not yet taken effect", rule_id);
	case pcmk_rc_op_unsatisfied:
	return out->info(out, "Rule %s does not satisfy conditions",
	rule_id);
	default:
	out->err(out,
	"Could not determine whether rule %s is in effect: %s",
	rule_id, ((error != NULL)? error : "unexpected error"));
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("rule-check", "const char ", "int", "const char ")
	static int
	rule_check_xml(pcmk__output_t *out, va_list args)
	{
	const char rule_id = va_arg(args, const char );
	int result = va_arg(args, int);
	const char error = va_arg(args, const char );

	char *rc_str = pcmk__itoa(pcmk_rc2exitc(result));

	pcmk__output_create_xml_node(out, PCMK_XE_RULE_CHECK,
	PCMK_XA_RULE_ID, rule_id,
	PCMK_XA_RC, rc_str,
	NULL);
	free(rc_str);

	switch (result) {
	case pcmk_rc_within_range:
	case pcmk_rc_ok:
	case pcmk_rc_after_range:
	case pcmk_rc_before_range:
	case pcmk_rc_op_unsatisfied:
	return pcmk_rc_ok;
	default:
	out->err(out,
	"Could not determine whether rule %s is in effect: %s",
	rule_id, ((error != NULL)? error : "unexpected error"));
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("result-code", "int", "const char ", "const char ")
	static int
	result_code_none(pcmk__output_t *out, va_list args)
	{
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("result-code", "int", "const char ", "const char ")
	static int
	result_code_text(pcmk__output_t *out, va_list args)
	{
	int code = va_arg(args, int);
	const char name = va_arg(args, const char );
	const char desc = va_arg(args, const char );

	static int code_width = 0;

	if (out->is_quiet(out)) {
	/* If out->is_quiet(), don't print the code. Print name and/or desc in a
	* compact format for text output, or print nothing at all for none-type
	* output.
	*/
	if ((name != NULL) && (desc != NULL)) {
	pcmk__formatted_printf(out, "%s - %s\n", name, desc);

	} else if ((name != NULL) \|\| (desc != NULL)) {
	pcmk__formatted_printf(out, "%s\n", ((name != NULL)? name : desc));
	}
	return pcmk_rc_ok;
	}

	/* Get length of longest (most negative) standard Pacemaker return code
	* This should be longer than all the values of any other type of return
	* code.
	*/
	if (code_width == 0) {
	long long most_negative = pcmk_rc_error - (long long) pcmk__n_rc + 1;
	code_width = (int) snprintf(NULL, 0, "%lld", most_negative);
	}

	if ((name != NULL) && (desc != NULL)) {
	static int name_width = 0;

	if (name_width == 0) {
	// Get length of longest standard Pacemaker return code name
	for (int lpc = 0; lpc < pcmk__n_rc; lpc++) {
	int len = (int) strlen(pcmk_rc_name(pcmk_rc_error - lpc));
	name_width = QB_MAX(name_width, len);
	}
	}
	return out->info(out, "% d: %-s %s", code_width, code, name_width,
	name, desc);
	}

	if ((name != NULL) \|\| (desc != NULL)) {
	return out->info(out, "% *d: %s", code_width, code,
	((name != NULL)? name : desc));
	}

	return out->info(out, "% *d", code_width, code);
	}

	PCMK__OUTPUT_ARGS("result-code", "int", "const char ", "const char ")
	static int
	result_code_xml(pcmk__output_t *out, va_list args)
	{
	int code = va_arg(args, int);
	const char name = va_arg(args, const char );
	const char desc = va_arg(args, const char );

	char *code_str = pcmk__itoa(code);

	pcmk__output_create_xml_node(out, PCMK_XE_RESULT_CODE,
	PCMK_XA_CODE, code_str,
	PCMK_XA_NAME, name,
	PCMK_XA_DESCRIPTION, desc,
	NULL);
	free(code_str);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-attribute", "const char ", "const char ", "const char *")
	static int
	ticket_attribute_default(pcmk__output_t *out, va_list args)
	{
	const char ticket_id G_GNUC_UNUSED = va_arg(args, const char );
	const char name G_GNUC_UNUSED = va_arg(args, const char );
	const char value = va_arg(args, const char );

	out->info(out, "%s", value);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-attribute", "const char ", "const char ", "const char *")
	static int
	ticket_attribute_xml(pcmk__output_t *out, va_list args)
	{
	const char ticket_id = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );

	/* Create:
	* <tickets>
	* <ticket id="">
	* <attribute name="" value="" />
	* </ticket>
	* </tickets>
	*/
	pcmk__output_xml_create_parent(out, PCMK_XE_TICKETS, NULL);
	pcmk__output_xml_create_parent(out, PCMK_XE_TICKET,
	PCMK_XA_ID, ticket_id, NULL);
	pcmk__output_create_xml_node(out, PCMK_XA_ATTRIBUTE,
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);
	pcmk__output_xml_pop_parent(out);
	pcmk__output_xml_pop_parent(out);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-constraints", "xmlNode *")
	static int
	ticket_constraints_default(pcmk__output_t *out, va_list args)
	{
	xmlNode constraint_xml = va_arg(args, xmlNode );

	/* constraint_xml can take two forms:
	*
	* <rsc_ticket id="rsc1-req-ticketA" rsc="rsc1" ticket="ticketA" ... />
	*
	* for when there's only one ticket in the CIB, or when the user asked
	* for a specific ticket (crm_ticket -c -t for instance)
	*
	* <xpath-query>
	* <rsc_ticket id="rsc1-req-ticketA" rsc="rsc1" ticket="ticketA" ... />
	* <rsc_ticket id="rsc1-req-ticketB" rsc="rsc2" ticket="ticketB" ... />
	* </xpath-query>
	*
	* for when there's multiple tickets in the and the user did not ask for
	* a specific one.
	*
	* In both cases, we simply output a <rsc_ticket> element for each ticket
	* in the results.
	*/
	out->info(out, "Constraints XML:\n");

	if (pcmk__xe_is(constraint_xml, PCMK__XE_XPATH_QUERY)) {
	xmlNode *child = pcmk__xe_first_child(constraint_xml, NULL, NULL, NULL);

	do {
	GString *buf = g_string_sized_new(1024);

	pcmk__xml_string(child, pcmk__xml_fmt_pretty, buf, 0);
	out->output_xml(out, PCMK_XE_CONSTRAINT, buf->str);
	g_string_free(buf, TRUE);

	child = pcmk__xe_next(child);
	} while (child != NULL);
	} else {
	GString *buf = g_string_sized_new(1024);

	pcmk__xml_string(constraint_xml, pcmk__xml_fmt_pretty, buf, 0);
	out->output_xml(out, PCMK_XE_CONSTRAINT, buf->str);
	g_string_free(buf, TRUE);
	}

	return pcmk_rc_ok;
	}

	static int
	add_ticket_element_with_constraints(xmlNode node, void userdata)
	{
	pcmk__output_t out = (pcmk__output_t ) userdata;
	const char *ticket_id = crm_element_value(node, PCMK_XA_TICKET);

	pcmk__output_xml_create_parent(out, PCMK_XE_TICKET,
	PCMK_XA_ID, ticket_id, NULL);
	pcmk__output_xml_create_parent(out, PCMK_XE_CONSTRAINTS, NULL);
	pcmk__output_xml_add_node_copy(out, node);

	/* Pop two parents so now we are back under the <tickets> element */
	pcmk__output_xml_pop_parent(out);
	pcmk__output_xml_pop_parent(out);

	return pcmk_rc_ok;
	}

	static int
	add_resource_element(xmlNode node, void userdata)
	{
	pcmk__output_t out = (pcmk__output_t ) userdata;
	const char *rsc = crm_element_value(node, PCMK_XA_RSC);

	pcmk__output_create_xml_node(out, PCMK_XE_RESOURCE,
	PCMK_XA_ID, rsc, NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-constraints", "xmlNode *")
	static int
	ticket_constraints_xml(pcmk__output_t *out, va_list args)
	{
	xmlNode constraint_xml = va_arg(args, xmlNode );

	/* Create:
	* <tickets>
	* <ticket id="">
	* <constraints>
	* <rsc_ticket />
	* </constraints>
	* </ticket>
	* ...
	* </tickets>
	*/
	pcmk__output_xml_create_parent(out, PCMK_XE_TICKETS, NULL);

	if (pcmk__xe_is(constraint_xml, PCMK__XE_XPATH_QUERY)) {
	/* Iterate through the list of children once to create all the
	* ticket/constraint elements.
	*/
	pcmk__xe_foreach_child(constraint_xml, NULL, add_ticket_element_with_constraints, out);

	/* Put us back at the same level as where <tickets> was created. */
	pcmk__output_xml_pop_parent(out);

	/* Constraints can reference a resource ID that is defined in the XML
	* schema as an IDREF. This requires some other element to be present
	* with an id= attribute that matches.
	*
	* Iterate through the list of children a second time to create the
	* following:
	*
	* <resources>
	* <resource id="" />
	* ...
	* </resources>
	*/
	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCES, NULL);
	pcmk__xe_foreach_child(constraint_xml, NULL, add_resource_element, out);
	pcmk__output_xml_pop_parent(out);

	} else {
	/* Creating the output for a single constraint is much easier. All the
	* comments in the above block apply here.
	*/
	add_ticket_element_with_constraints(constraint_xml, out);
	pcmk__output_xml_pop_parent(out);

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCES, NULL);
	add_resource_element(constraint_xml, out);
	pcmk__output_xml_pop_parent(out);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-state", "xmlNode *")
	static int
	ticket_state_default(pcmk__output_t *out, va_list args)
	{
	xmlNode state_xml = va_arg(args, xmlNode );

	GString *buf = g_string_sized_new(1024);

	out->info(out, "State XML:\n");
	pcmk__xml_string(state_xml, pcmk__xml_fmt_pretty, buf, 0);
	out->output_xml(out, PCMK__XE_TICKET_STATE, buf->str);

	g_string_free(buf, TRUE);
	return pcmk_rc_ok;
	}

	static int
	add_ticket_element(xmlNode node, void userdata)
	{
	pcmk__output_t out = (pcmk__output_t ) userdata;
	xmlNode *ticket_node = NULL;

	ticket_node = pcmk__output_create_xml_node(out, PCMK_XE_TICKET, NULL);
	pcmk__xe_copy_attrs(ticket_node, node, pcmk__xaf_none);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-state", "xmlNode *")
	static int
	ticket_state_xml(pcmk__output_t *out, va_list args)
	{
	xmlNode state_xml = va_arg(args, xmlNode );

	/* Create:
	* <tickets>
	* <ticket />
	* ...
	* </tickets>
	*/
	pcmk__output_xml_create_parent(out, PCMK_XE_TICKETS, NULL);

	if (state_xml->children != NULL) {
	/* Iterate through the list of children once to create all the
	* ticket elements.
	*/
	pcmk__xe_foreach_child(state_xml, PCMK__XE_TICKET_STATE, add_ticket_element, out);

	} else {
	add_ticket_element(state_xml, out);
	}

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "attribute", "default", attribute_default },
	{ "attribute", "xml", attribute_xml },
	{ "cluster-status", "default", pcmk__cluster_status_text },
	{ "cluster-status", "html", cluster_status_html },
	{ "cluster-status", "xml", cluster_status_xml },
	{ "crmadmin-node", "default", crmadmin_node },
	{ "crmadmin-node", "text", crmadmin_node_text },
	{ "crmadmin-node", "xml", crmadmin_node_xml },
	{ "dc", "default", dc },
	{ "dc", "text", dc_text },
	{ "dc", "xml", dc_xml },
	{ "digests", "default", digests_text },
	{ "digests", "xml", digests_xml },
	{ "health", "default", health },
	{ "health", "text", health_text },
	{ "health", "xml", health_xml },
	{ "inject-attr", "default", inject_attr },
	{ "inject-attr", "xml", inject_attr_xml },
	{ "inject-cluster-action", "default", inject_cluster_action },
	{ "inject-cluster-action", "xml", inject_cluster_action_xml },
	{ "inject-fencing-action", "default", inject_fencing_action },
	{ "inject-fencing-action", "xml", inject_fencing_action_xml },
	{ "inject-modify-config", "default", inject_modify_config },
	{ "inject-modify-config", "xml", inject_modify_config_xml },
	{ "inject-modify-node", "default", inject_modify_node },
	{ "inject-modify-node", "xml", inject_modify_node_xml },
	{ "inject-modify-ticket", "default", inject_modify_ticket },
	{ "inject-modify-ticket", "xml", inject_modify_ticket_xml },
	{ "inject-pseudo-action", "default", inject_pseudo_action },
	{ "inject-pseudo-action", "xml", inject_pseudo_action_xml },
	{ "inject-rsc-action", "default", inject_rsc_action },
	{ "inject-rsc-action", "xml", inject_rsc_action_xml },
	{ "inject-spec", "default", inject_spec },
	{ "inject-spec", "xml", inject_spec_xml },
	{ "locations-and-colocations", "default", locations_and_colocations },
	{ "locations-and-colocations", "xml", locations_and_colocations_xml },
	{ "locations-list", "default", locations_list },
	{ "locations-list", "xml", locations_list_xml },
	{ "node-action", "default", node_action },
	{ "node-action", "xml", node_action_xml },
	{ "node-info", "default", node_info_default },
	{ "node-info", "xml", node_info_xml },
	{ "pacemakerd-health", "default", pacemakerd_health },
	{ "pacemakerd-health", "html", pacemakerd_health_html },
	{ "pacemakerd-health", "text", pacemakerd_health_text },
	{ "pacemakerd-health", "xml", pacemakerd_health_xml },
	{ "profile", "default", profile_default, },
	{ "profile", "xml", profile_xml },
	{ "result-code", PCMK_VALUE_NONE, result_code_none },
	{ "result-code", "text", result_code_text },
	{ "result-code", "xml", result_code_xml },
	{ "rsc-action", "default", rsc_action_default },
	{ "rsc-action-item", "default", rsc_action_item },
	{ "rsc-action-item", "xml", rsc_action_item_xml },
	{ "rsc-is-colocated-with-list", "default", rsc_is_colocated_with_list },
	{ "rsc-is-colocated-with-list", "xml", rsc_is_colocated_with_list_xml },
	{ "rscs-colocated-with-list", "default", rscs_colocated_with_list },
	{ "rscs-colocated-with-list", "xml", rscs_colocated_with_list_xml },
	{ "rule-check", "default", rule_check_default },
	{ "rule-check", "xml", rule_check_xml },
	{ "ticket-attribute", "default", ticket_attribute_default },
	{ "ticket-attribute", "xml", ticket_attribute_xml },
	{ "ticket-constraints", "default", ticket_constraints_default },
	{ "ticket-constraints", "xml", ticket_constraints_xml },
	{ "ticket-state", "default", ticket_state_default },
	{ "ticket-state", "xml", ticket_state_xml },

	{ NULL, NULL, NULL }
	};

	void
	pcmk__register_lib_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/lib/pacemaker/pcmk_sched_bundle.c b/lib/pacemaker/pcmk_sched_bundle.c
	index d1872a5732..adba71e447 100644
	--- a/lib/pacemaker/pcmk_sched_bundle.c
	+++ b/lib/pacemaker/pcmk_sched_bundle.c
	@@ -1,1066 +1,1066 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/common/xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	struct assign_data {
	const pcmk_node_t *prefer;
	bool stop_if_fail;
	};

	/*!
	* \internal
	* \brief Assign a single bundle replica's resources (other than container)
	*
	* \param[in,out] replica Replica to assign
	* \param[in] user_data Preferred node, if any
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	assign_replica(pcmk__bundle_replica_t replica, void user_data)
	{
	pcmk_node_t *container_host = NULL;

	struct assign_data *assign_data = user_data;
	const pcmk_node_t *prefer = assign_data->prefer;
	bool stop_if_fail = assign_data->stop_if_fail;

	const pcmk_resource_t *bundle = pe__const_top_resource(replica->container,
	true);

	if (replica->ip != NULL) {
	pcmk__rsc_trace(bundle, "Assigning bundle %s IP %s",
	bundle->id, replica->ip->id);
	replica->ip->private->cmds->assign(replica->ip, prefer, stop_if_fail);
	}

	container_host = replica->container->private->assigned_node;
	if (replica->remote != NULL) {
	if (pcmk__is_pacemaker_remote_node(container_host)) {
	/* REMOTE_CONTAINER_HACK: "Nested" connection resources must be on
	* the same host because Pacemaker Remote only supports a single
	* active connection.
	*/
	pcmk__new_colocation("#replica-remote-with-host-remote", NULL,
	PCMK_SCORE_INFINITY, replica->remote,
	container_host->private->remote, NULL,
	NULL, pcmk__coloc_influence);
	}
	pcmk__rsc_trace(bundle, "Assigning bundle %s connection %s",
	bundle->id, replica->remote->id);
	replica->remote->private->cmds->assign(replica->remote, prefer,
	stop_if_fail);
	}

	if (replica->child != NULL) {
	pcmk_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, replica->child->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (!pcmk__same_node(node, replica->node)) {
	- node->weight = -PCMK_SCORE_INFINITY;
	+ node->assign->score = -PCMK_SCORE_INFINITY;
	} else if (!pcmk__threshold_reached(replica->child, node, NULL)) {
	- node->weight = PCMK_SCORE_INFINITY;
	+ node->assign->score = PCMK_SCORE_INFINITY;
	}
	}

	pcmk__set_rsc_flags(replica->child->private->parent,
	pcmk__rsc_assigning);
	pcmk__rsc_trace(bundle, "Assigning bundle %s replica child %s",
	bundle->id, replica->child->id);
	replica->child->private->cmds->assign(replica->child, replica->node,
	stop_if_fail);
	pcmk__clear_rsc_flags(replica->child->private->parent,
	pcmk__rsc_assigning);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Assign a bundle resource to a node
	*
	* \param[in,out] rsc Resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and a primitive descendant of \p rsc
	* can't be assigned to a node, set the
	* descendant's next role to stopped and update
	* existing actions
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pcmk_node_t *
	pcmk__bundle_assign(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *containers = NULL;
	pcmk_resource_t *bundled_resource = NULL;
	struct assign_data assign_data = { prefer, stop_if_fail };

	CRM_ASSERT(pcmk__is_bundle(rsc));

	pcmk__rsc_trace(rsc, "Assigning bundle %s", rsc->id);
	pcmk__set_rsc_flags(rsc, pcmk__rsc_assigning);

	pe__show_node_scores(!pcmk_is_set(rsc->private->scheduler->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->private->allowed_nodes,
	rsc->private->scheduler);

	// Assign all containers first, so we know what nodes the bundle will be on
	containers = g_list_sort(pe__bundle_containers(rsc), pcmk__cmp_instance);
	pcmk__assign_instances(rsc, containers, pe__bundle_max(rsc),
	rsc->private->fns->max_per_node(rsc));
	g_list_free(containers);

	// Then assign remaining replica resources
	pe__foreach_bundle_replica(rsc, assign_replica, (void *) &assign_data);

	// Finally, assign the bundled resources to each bundle node
	bundled_resource = pe__bundled_resource(rsc);
	if (bundled_resource != NULL) {
	pcmk_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, bundled_resource->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
	if (pe__node_is_bundle_instance(rsc, node)) {
	- node->weight = 0;
	+ node->assign->score = 0;
	} else {
	- node->weight = -PCMK_SCORE_INFINITY;
	+ node->assign->score = -PCMK_SCORE_INFINITY;
	}
	}
	bundled_resource->private->cmds->assign(bundled_resource, prefer,
	stop_if_fail);
	}

	pcmk__clear_rsc_flags(rsc, pcmk__rsc_assigning\|pcmk__rsc_unassigned);
	return NULL;
	}

	/*!
	* \internal
	* \brief Create actions for a bundle replica's resources (other than child)
	*
	* \param[in,out] replica Replica to create actions for
	* \param[in] user_data Unused
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	create_replica_actions(pcmk__bundle_replica_t replica, void user_data)
	{
	if (replica->ip != NULL) {
	replica->ip->private->cmds->create_actions(replica->ip);
	}
	if (replica->container != NULL) {
	replica->container->private->cmds->create_actions(replica->container);
	}
	if (replica->remote != NULL) {
	replica->remote->private->cmds->create_actions(replica->remote);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given bundle resource
	*
	* \param[in,out] rsc Bundle resource to create actions for
	*/
	void
	pcmk__bundle_create_actions(pcmk_resource_t *rsc)
	{
	pcmk_action_t *action = NULL;
	GList *containers = NULL;
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT(pcmk__is_bundle(rsc));

	pe__foreach_bundle_replica(rsc, create_replica_actions, NULL);

	containers = pe__bundle_containers(rsc);
	pcmk__create_instance_actions(rsc, containers);
	g_list_free(containers);

	bundled_resource = pe__bundled_resource(rsc);
	if (bundled_resource != NULL) {
	bundled_resource->private->cmds->create_actions(bundled_resource);

	if (pcmk_is_set(bundled_resource->flags, pcmk__rsc_promotable)) {
	pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_PROMOTE, true, true);
	action = pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_PROMOTED,
	true, true);
	action->priority = PCMK_SCORE_INFINITY;

	pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_DEMOTE, true, true);
	action = pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_DEMOTED,
	true, true);
	action->priority = PCMK_SCORE_INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Create internal constraints for a bundle replica's resources
	*
	* \param[in,out] replica Replica to create internal constraints for
	* \param[in,out] user_data Replica's parent bundle
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	replica_internal_constraints(pcmk__bundle_replica_t replica, void user_data)
	{
	pcmk_resource_t *bundle = user_data;

	replica->container->private->cmds->internal_constraints(replica->container);

	// Start bundle -> start replica container
	pcmk__order_starts(bundle, replica->container,
	pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_then_implies_first_graphed);

	// Stop bundle -> stop replica child and container
	if (replica->child != NULL) {
	pcmk__order_stops(bundle, replica->child,
	pcmk__ar_then_implies_first_graphed);
	}
	pcmk__order_stops(bundle, replica->container,
	pcmk__ar_then_implies_first_graphed);

	// Start replica container -> bundle is started
	pcmk__order_resource_actions(replica->container, PCMK_ACTION_START, bundle,
	PCMK_ACTION_RUNNING,
	pcmk__ar_first_implies_then_graphed);

	// Stop replica container -> bundle is stopped
	pcmk__order_resource_actions(replica->container, PCMK_ACTION_STOP, bundle,
	PCMK_ACTION_STOPPED,
	pcmk__ar_first_implies_then_graphed);

	if (replica->ip != NULL) {
	replica->ip->private->cmds->internal_constraints(replica->ip);

	// Replica IP address -> replica container (symmetric)
	pcmk__order_starts(replica->ip, replica->container,
	pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_guest_allowed);
	pcmk__order_stops(replica->container, replica->ip,
	pcmk__ar_then_implies_first\|pcmk__ar_guest_allowed);

	pcmk__new_colocation("#ip-with-container", NULL, PCMK_SCORE_INFINITY,
	replica->ip, replica->container, NULL, NULL,
	pcmk__coloc_influence);
	}

	if (replica->remote != NULL) {
	/* This handles ordering and colocating remote relative to container
	* (via "#resource-with-container"). Since IP is also ordered and
	* colocated relative to the container, we don't need to do anything
	* explicit here with IP.
	*/
	replica->remote->private->cmds->internal_constraints(replica->remote);
	}

	if (replica->child != NULL) {
	CRM_ASSERT(replica->remote != NULL);
	// "Start remote then child" is implicit in scheduler's remote logic
	}
	return true;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a bundle resource
	*
	* \param[in,out] rsc Bundle resource to create implicit constraints for
	*/
	void
	pcmk__bundle_internal_constraints(pcmk_resource_t *rsc)
	{
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT(pcmk__is_bundle(rsc));

	pe__foreach_bundle_replica(rsc, replica_internal_constraints, rsc);

	bundled_resource = pe__bundled_resource(rsc);
	if (bundled_resource == NULL) {
	return;
	}

	// Start bundle -> start bundled clone
	pcmk__order_resource_actions(rsc, PCMK_ACTION_START, bundled_resource,
	PCMK_ACTION_START,
	pcmk__ar_then_implies_first_graphed);

	// Bundled clone is started -> bundle is started
	pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_RUNNING,
	pcmk__ar_first_implies_then_graphed);

	// Stop bundle -> stop bundled clone
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP, bundled_resource,
	PCMK_ACTION_STOP,
	pcmk__ar_then_implies_first_graphed);

	// Bundled clone is stopped -> bundle is stopped
	pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_STOPPED,
	pcmk__ar_first_implies_then_graphed);

	bundled_resource->private->cmds->internal_constraints(bundled_resource);

	if (!pcmk_is_set(bundled_resource->flags, pcmk__rsc_promotable)) {
	return;
	}
	pcmk__promotable_restart_ordering(rsc);

	// Demote bundle -> demote bundled clone
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE, bundled_resource,
	PCMK_ACTION_DEMOTE,
	pcmk__ar_then_implies_first_graphed);

	// Bundled clone is demoted -> bundle is demoted
	pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_DEMOTED,
	pcmk__ar_first_implies_then_graphed);

	// Promote bundle -> promote bundled clone
	pcmk__order_resource_actions(rsc, PCMK_ACTION_PROMOTE,
	bundled_resource, PCMK_ACTION_PROMOTE,
	pcmk__ar_then_implies_first_graphed);

	// Bundled clone is promoted -> bundle is promoted
	pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_PROMOTED,
	rsc, PCMK_ACTION_PROMOTED,
	pcmk__ar_first_implies_then_graphed);
	}

	struct match_data {
	const pcmk_node_t *node; // Node to compare against replica
	pcmk_resource_t *container; // Replica container corresponding to node
	};

	/*!
	* \internal
	* \brief Check whether a replica container is assigned to a given node
	*
	* \param[in] replica Replica to check
	* \param[in,out] user_data struct match_data with node to compare against
	*
	* \return true if the replica does not match (to indicate further replicas
	* should be processed), otherwise false
	*/
	static bool
	match_replica_container(const pcmk__bundle_replica_t replica, void user_data)
	{
	struct match_data *match_data = user_data;

	if (pcmk__instance_matches(replica->container, match_data->node,
	pcmk_role_unknown, false)) {
	match_data->container = replica->container;
	return false; // Match found, don't bother searching further replicas
	}
	return true; // No match, keep searching
	}

	/*!
	* \internal
	* \brief Get the host to which a bundle node is assigned
	*
	* \param[in] node Possible bundle node to check
	*
	* \return Node to which the container for \p node is assigned if \p node is a
	* bundle node, otherwise \p node itself
	*/
	static const pcmk_node_t *
	get_bundle_node_host(const pcmk_node_t *node)
	{
	if (pcmk__is_bundle_node(node)) {
	const pcmk_resource_t *container = NULL;

	container = node->private->remote->private->launcher;
	return container->private->fns->location(container, NULL, 0);
	}
	return node;
	}

	/*!
	* \internal
	* \brief Find a bundle container compatible with a dependent resource
	*
	* \param[in] dependent Dependent resource in colocation with bundle
	* \param[in] bundle Bundle that \p dependent is colocated with
	*
	* \return A container from \p bundle assigned to the same node as \p dependent
	* if assigned, otherwise assigned to any of dependent's allowed nodes,
	* otherwise NULL.
	*/
	static pcmk_resource_t *
	compatible_container(const pcmk_resource_t *dependent,
	const pcmk_resource_t *bundle)
	{
	GList *scratch = NULL;
	struct match_data match_data = { NULL, NULL };

	// If dependent is assigned, only check there
	match_data.node = dependent->private->fns->location(dependent, NULL, 0);
	match_data.node = get_bundle_node_host(match_data.node);
	if (match_data.node != NULL) {
	pe__foreach_const_bundle_replica(bundle, match_replica_container,
	&match_data);
	return match_data.container;
	}

	// Otherwise, check for any of the dependent's allowed nodes
	scratch = g_hash_table_get_values(dependent->private->allowed_nodes);
	scratch = pcmk__sort_nodes(scratch, NULL);
	for (const GList *iter = scratch; iter != NULL; iter = iter->next) {
	match_data.node = iter->data;
	match_data.node = get_bundle_node_host(match_data.node);
	if (match_data.node == NULL) {
	continue;
	}

	pe__foreach_const_bundle_replica(bundle, match_replica_container,
	&match_data);
	if (match_data.container != NULL) {
	break;
	}
	}
	g_list_free(scratch);
	return match_data.container;
	}

	struct coloc_data {
	const pcmk__colocation_t *colocation;
	pcmk_resource_t *dependent;
	GList *container_hosts;
	};

	/*!
	* \internal
	* \brief Apply a colocation score to replica node scores or resource priority
	*
	* \param[in] replica Replica of primary bundle resource in colocation
	* \param[in,out] user_data struct coloc_data for colocation being applied
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	replica_apply_coloc_score(const pcmk__bundle_replica_t *replica,
	void *user_data)
	{
	struct coloc_data *coloc_data = user_data;
	pcmk_node_t *chosen = NULL;
	pcmk_resource_t *container = replica->container;

	if (coloc_data->colocation->score < PCMK_SCORE_INFINITY) {
	container->private->cmds->apply_coloc_score(coloc_data->dependent,
	container,
	coloc_data->colocation,
	false);
	return true;
	}

	chosen = container->private->fns->location(container, NULL, 0);
	if ((chosen == NULL)
	\|\| is_set_recursive(container, pcmk__rsc_blocked, true)) {
	return true;
	}

	if ((coloc_data->colocation->primary_role >= pcmk_role_promoted)
	&& ((replica->child == NULL)
	\|\| (replica->child->private->next_role < pcmk_role_promoted))) {
	return true;
	}

	pcmk__rsc_trace(pe__const_top_resource(container, true),
	"Allowing mandatory colocation %s using %s @%d",
	coloc_data->colocation->id, pcmk__node_name(chosen),
	- chosen->weight);
	+ chosen->assign->score);
	coloc_data->container_hosts = g_list_prepend(coloc_data->container_hosts,
	chosen);
	return true;
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__bundle_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	struct coloc_data coloc_data = { colocation, dependent, NULL };

	/* This should never be called for the bundle itself as a dependent.
	* Instead, we add its colocation constraints to its containers and bundled
	* primitive and call the apply_coloc_score() method for them as dependents.
	*/
	CRM_ASSERT(pcmk__is_bundle(primary) && pcmk__is_primitive(dependent)
	&& (colocation != NULL) && !for_dependent);

	if (pcmk_is_set(primary->flags, pcmk__rsc_unassigned)) {
	pcmk__rsc_trace(primary,
	"Skipping applying colocation %s "
	"because %s is still provisional",
	colocation->id, primary->id);
	return;
	}
	pcmk__rsc_trace(primary, "Applying colocation %s (%s with %s at %s)",
	colocation->id, dependent->id, primary->id,
	pcmk_readable_score(colocation->score));

	/* If the constraint dependent is a clone or bundle, "dependent" here is one
	* of its instances. Look for a compatible instance of this bundle.
	*/
	if (colocation->dependent->private->variant > pcmk__rsc_variant_group) {
	const pcmk_resource_t *primary_container = NULL;

	primary_container = compatible_container(dependent, primary);
	if (primary_container != NULL) { // Success, we found one
	pcmk__rsc_debug(primary, "Pairing %s with %s",
	dependent->id, primary_container->id);
	dependent->private->cmds->apply_coloc_score(dependent,
	primary_container,
	colocation, true);

	} else if (colocation->score >= PCMK_SCORE_INFINITY) {
	// Failure, and it's fatal
	crm_notice("%s cannot run because there is no compatible "
	"instance of %s to colocate with",
	dependent->id, primary->id);
	pcmk__assign_resource(dependent, NULL, true, true);

	} else { // Failure, but we can ignore it
	pcmk__rsc_debug(primary,
	"%s cannot be colocated with any instance of %s",
	dependent->id, primary->id);
	}
	return;
	}

	pe__foreach_const_bundle_replica(primary, replica_apply_coloc_score,
	&coloc_data);

	if (colocation->score >= PCMK_SCORE_INFINITY) {
	pcmk__colocation_intersect_nodes(dependent, primary, colocation,
	coloc_data.container_hosts, false);
	}
	g_list_free(coloc_data.container_hosts);
	}

	// Bundle implementation of pcmk__assignment_methods_t:with_this_colocations()
	void
	pcmk__with_bundle_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	const pcmk_resource_t *bundled_rsc = NULL;

	CRM_ASSERT(pcmk__is_bundle(rsc) && (orig_rsc != NULL) && (list != NULL));

	// The bundle itself and its containers always get its colocations
	if ((orig_rsc == rsc)
	\|\| pcmk_is_set(orig_rsc->flags, pcmk__rsc_replica_container)) {

	pcmk__add_with_this_list(list, rsc->private->with_this_colocations,
	orig_rsc);
	return;
	}

	/* The bundled resource gets the colocations if it's promotable and we've
	* begun choosing roles
	*/
	bundled_rsc = pe__bundled_resource(rsc);
	if ((bundled_rsc == NULL)
	\|\| !pcmk_is_set(bundled_rsc->flags, pcmk__rsc_promotable)
	\|\| (pe__const_top_resource(orig_rsc, false) != bundled_rsc)) {
	return;
	}

	if (orig_rsc == bundled_rsc) {
	if (pe__clone_flag_is_set(orig_rsc,
	pcmk__clone_promotion_constrained)) {
	/* orig_rsc is the clone and we're setting roles (or have already
	* done so)
	*/
	pcmk__add_with_this_list(list, rsc->private->with_this_colocations,
	orig_rsc);
	}

	} else if (!pcmk_is_set(orig_rsc->flags, pcmk__rsc_unassigned)) {
	/* orig_rsc is an instance and is already assigned. If something
	* requests colocations for orig_rsc now, it's for setting roles.
	*/
	pcmk__add_with_this_list(list, rsc->private->with_this_colocations,
	orig_rsc);
	}
	}

	// Bundle implementation of pcmk__assignment_methods_t:this_with_colocations()
	void
	pcmk__bundle_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	const pcmk_resource_t *bundled_rsc = NULL;

	CRM_ASSERT(pcmk__is_bundle(rsc) && (orig_rsc != NULL) && (list != NULL));

	// The bundle itself and its containers always get its colocations
	if ((orig_rsc == rsc)
	\|\| pcmk_is_set(orig_rsc->flags, pcmk__rsc_replica_container)) {

	pcmk__add_this_with_list(list, rsc->private->this_with_colocations,
	orig_rsc);
	return;
	}

	/* The bundled resource gets the colocations if it's promotable and we've
	* begun choosing roles
	*/
	bundled_rsc = pe__bundled_resource(rsc);
	if ((bundled_rsc == NULL)
	\|\| !pcmk_is_set(bundled_rsc->flags, pcmk__rsc_promotable)
	\|\| (pe__const_top_resource(orig_rsc, false) != bundled_rsc)) {
	return;
	}

	if (orig_rsc == bundled_rsc) {
	if (pe__clone_flag_is_set(orig_rsc,
	pcmk__clone_promotion_constrained)) {
	/* orig_rsc is the clone and we're setting roles (or have already
	* done so)
	*/
	pcmk__add_this_with_list(list, rsc->private->this_with_colocations,
	orig_rsc);
	}

	} else if (!pcmk_is_set(orig_rsc->flags, pcmk__rsc_unassigned)) {
	/* orig_rsc is an instance and is already assigned. If something
	* requests colocations for orig_rsc now, it's for setting roles.
	*/
	pcmk__add_this_with_list(list, rsc->private->this_with_colocations,
	orig_rsc);
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given bundle resource action
	*
	* \param[in,out] action Bundle resource action to get flags for
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__bundle_action_flags(pcmk_action_t action, const pcmk_node_t node)
	{
	GList *containers = NULL;
	uint32_t flags = 0;
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT((action != NULL) && pcmk__is_bundle(action->rsc));

	bundled_resource = pe__bundled_resource(action->rsc);
	if (bundled_resource != NULL) {
	GList *children = bundled_resource->private->children;

	// Clone actions are done on the bundled clone resource, not container
	switch (get_complex_task(bundled_resource, action->task)) {
	case pcmk_action_unspecified:
	case pcmk_action_notify:
	case pcmk_action_notified:
	case pcmk_action_promote:
	case pcmk_action_promoted:
	case pcmk_action_demote:
	case pcmk_action_demoted:
	return pcmk__collective_action_flags(action, children, node);
	default:
	break;
	}
	}

	containers = pe__bundle_containers(action->rsc);
	flags = pcmk__collective_action_flags(action, containers, node);
	g_list_free(containers);
	return flags;
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a bundle replica
	*
	* \param[in,out] replica Replica to apply constraint to
	* \param[in,out] user_data Location constraint to apply
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	apply_location_to_replica(pcmk__bundle_replica_t replica, void user_data)
	{
	pcmk__location_t *location = user_data;

	replica->container->private->cmds->apply_location(replica->container,
	location);
	if (replica->ip != NULL) {
	replica->ip->private->cmds->apply_location(replica->ip, location);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a bundle resource's allowed node scores
	*
	* \param[in,out] rsc Bundle resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*/
	void
	pcmk__bundle_apply_location(pcmk_resource_t rsc, pcmk__location_t location)
	{
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT((location != NULL) && pcmk__is_bundle(rsc));

	pcmk__apply_location(rsc, location);
	pe__foreach_bundle_replica(rsc, apply_location_to_replica, location);

	bundled_resource = pe__bundled_resource(rsc);
	if ((bundled_resource != NULL)
	&& ((location->role_filter == pcmk_role_unpromoted)
	\|\| (location->role_filter == pcmk_role_promoted))) {

	bundled_resource->private->cmds->apply_location(bundled_resource,
	location);
	bundled_resource->private->location_constraints =
	g_list_prepend(bundled_resource->private->location_constraints,
	location);
	}
	}

	#define XPATH_REMOTE "//nvpair[@name='" PCMK_REMOTE_RA_ADDR "']"

	/*!
	* \internal
	* \brief Add a bundle replica's actions to transition graph
	*
	* \param[in,out] replica Replica to add to graph
	* \param[in] user_data Bundle that replica belongs to (for logging only)
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	add_replica_actions_to_graph(pcmk__bundle_replica_t replica, void user_data)
	{
	if ((replica->remote != NULL)
	&& pe__bundle_needs_remote_name(replica->remote)) {

	/* REMOTE_CONTAINER_HACK: Allow remote nodes to run containers that
	* run pacemaker-remoted inside, without needing a separate IP for
	* the container. This is done by configuring the inner remote's
	* connection host as the magic string "#uname", then
	* replacing it with the underlying host when needed.
	*/
	xmlNode *nvpair = get_xpath_object(XPATH_REMOTE,
	replica->remote->private->xml,
	LOG_ERR);
	const char *calculated_addr = NULL;

	// Replace the value in replica->remote->xml (if appropriate)
	calculated_addr = pe__add_bundle_remote_name(replica->remote, nvpair,
	PCMK_XA_VALUE);
	if (calculated_addr != NULL) {
	/* Since this is for the bundle as a resource, and not any
	* particular action, replace the value in the default
	* parameters (not evaluated for node). create_graph_action()
	* will grab it from there to replace it in node-evaluated
	* parameters.
	*/
	GHashTable *params = NULL;

	params = pe_rsc_params(replica->remote, NULL,
	replica->remote->private->scheduler);
	pcmk__insert_dup(params, PCMK_REMOTE_RA_ADDR, calculated_addr);
	} else {
	pcmk_resource_t *bundle = user_data;

	/* The only way to get here is if the remote connection is
	* neither currently running nor scheduled to run. That means we
	* won't be doing any operations that require addr (only start
	* requires it; we additionally use it to compare digests when
	* unpacking status, promote, and migrate_from history, but
	* that's already happened by this point).
	*/
	pcmk__rsc_info(bundle,
	"Unable to determine address for bundle %s "
	"remote connection", bundle->id);
	}
	}
	if (replica->ip != NULL) {
	replica->ip->private->cmds->add_actions_to_graph(replica->ip);
	}
	replica->container->private->cmds->add_actions_to_graph(replica->container);
	if (replica->remote != NULL) {
	replica->remote->private->cmds->add_actions_to_graph(replica->remote);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Add a bundle resource's actions to the transition graph
	*
	* \param[in,out] rsc Bundle resource whose actions should be added
	*/
	void
	pcmk__bundle_add_actions_to_graph(pcmk_resource_t *rsc)
	{
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT(pcmk__is_bundle(rsc));

	bundled_resource = pe__bundled_resource(rsc);
	if (bundled_resource != NULL) {
	bundled_resource->private->cmds->add_actions_to_graph(bundled_resource);
	}
	pe__foreach_bundle_replica(rsc, add_replica_actions_to_graph, rsc);
	}

	struct probe_data {
	pcmk_resource_t *bundle; // Bundle being probed
	pcmk_node_t *node; // Node to create probes on
	bool any_created; // Whether any probes have been created
	};

	/*!
	* \internal
	* \brief Order a bundle replica's start after another replica's probe
	*
	* \param[in,out] replica Replica to order start for
	* \param[in,out] user_data Replica with probe to order after
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	order_replica_start_after(pcmk__bundle_replica_t replica, void user_data)
	{
	pcmk__bundle_replica_t *probed_replica = user_data;

	if ((replica == probed_replica) \|\| (replica->container == NULL)) {
	return true;
	}
	pcmk__new_ordering(probed_replica->container,
	pcmk__op_key(probed_replica->container->id,
	PCMK_ACTION_MONITOR, 0),
	NULL, replica->container,
	pcmk__op_key(replica->container->id, PCMK_ACTION_START,
	0),
	NULL, pcmk__ar_ordered\|pcmk__ar_if_on_same_node,
	replica->container->private->scheduler);
	return true;
	}

	/*!
	* \internal
	* \brief Create probes for a bundle replica's resources
	*
	* \param[in,out] replica Replica to create probes for
	* \param[in,out] user_data struct probe_data
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	create_replica_probes(pcmk__bundle_replica_t replica, void user_data)
	{
	struct probe_data *probe_data = user_data;
	pcmk_resource_t *bundle = probe_data->bundle;

	if ((replica->ip != NULL)
	&& replica->ip->private->cmds->create_probe(replica->ip,
	probe_data->node)) {
	probe_data->any_created = true;
	}
	if ((replica->child != NULL)
	&& pcmk__same_node(probe_data->node, replica->node)
	&& replica->child->private->cmds->create_probe(replica->child,
	probe_data->node)) {
	probe_data->any_created = true;
	}
	if (replica->container->private->cmds->create_probe(replica->container,
	probe_data->node)) {
	probe_data->any_created = true;

	/* If we're limited to one replica per host (due to
	* the lack of an IP range probably), then we don't
	* want any of our peer containers starting until
	* we've established that no other copies are already
	* running.
	*
	* Partly this is to ensure that the maximum replicas per host is
	* observed, but also to ensure that the containers
	* don't fail to start because the necessary port
	* mappings (which won't include an IP for uniqueness)
	* are already taken
	*/
	if (bundle->private->fns->max_per_node(bundle) == 1) {
	pe__foreach_bundle_replica(bundle, order_replica_start_after,
	replica);
	}
	}
	if ((replica->remote != NULL)
	&& replica->remote->private->cmds->create_probe(replica->remote,
	probe_data->node)) {
	/* Do not probe the remote resource until we know where the container is
	* running. This is required for REMOTE_CONTAINER_HACK to correctly
	* probe remote resources.
	*/
	char *probe_uuid = pcmk__op_key(replica->remote->id,
	PCMK_ACTION_MONITOR, 0);
	pcmk_action_t *probe = NULL;

	probe = find_first_action(replica->remote->private->actions, probe_uuid,
	NULL, probe_data->node);
	free(probe_uuid);
	if (probe != NULL) {
	probe_data->any_created = true;
	pcmk__rsc_trace(bundle, "Ordering %s probe on %s",
	replica->remote->id,
	pcmk__node_name(probe_data->node));
	pcmk__new_ordering(replica->container,
	pcmk__op_key(replica->container->id,
	PCMK_ACTION_START, 0),
	NULL, replica->remote, NULL, probe,
	pcmk__ar_nested_remote_probe,
	bundle->private->scheduler);
	}
	}
	return true;
	}

	/*!
	* \internal
	*
	* \brief Schedule any probes needed for a bundle resource on a node
	*
	* \param[in,out] rsc Bundle resource to create probes for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__bundle_create_probe(pcmk_resource_t rsc, pcmk_node_t node)
	{
	struct probe_data probe_data = { rsc, node, false };

	CRM_ASSERT(pcmk__is_bundle(rsc));
	pe__foreach_bundle_replica(rsc, create_replica_probes, &probe_data);
	return probe_data.any_created;
	}

	/*!
	* \internal
	* \brief Output actions for one bundle replica
	*
	* \param[in,out] replica Replica to output actions for
	* \param[in] user_data Unused
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	output_replica_actions(pcmk__bundle_replica_t replica, void user_data)
	{
	if (replica->ip != NULL) {
	replica->ip->private->cmds->output_actions(replica->ip);
	}
	replica->container->private->cmds->output_actions(replica->container);
	if (replica->remote != NULL) {
	replica->remote->private->cmds->output_actions(replica->remote);
	}
	if (replica->child != NULL) {
	replica->child->private->cmds->output_actions(replica->child);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Output a summary of scheduled actions for a bundle resource
	*
	* \param[in,out] rsc Bundle resource to output actions for
	*/
	void
	pcmk__output_bundle_actions(pcmk_resource_t *rsc)
	{
	CRM_ASSERT(pcmk__is_bundle(rsc));
	pe__foreach_bundle_replica(rsc, output_replica_actions, NULL);
	}

	// Bundle implementation of pcmk__assignment_methods_t:add_utilization()
	void
	pcmk__bundle_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization)
	{
	pcmk_resource_t *container = NULL;

	CRM_ASSERT(pcmk__is_bundle(rsc));

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
	return;
	}

	/* All bundle replicas are identical, so using the utilization of the first
	* is sufficient for any. Only the implicit container resource can have
	* utilization values.
	*/
	container = pe__first_container(rsc);
	if (container != NULL) {
	container->private->cmds->add_utilization(container, orig_rsc, all_rscs,
	utilization);
	}
	}

	// Bundle implementation of pcmk__assignment_methods_t:shutdown_lock()
	void
	pcmk__bundle_shutdown_lock(pcmk_resource_t *rsc)
	{
	CRM_ASSERT(pcmk__is_bundle(rsc));
	// Bundles currently don't support shutdown locks
	}
	diff --git a/lib/pacemaker/pcmk_sched_clone.c b/lib/pacemaker/pcmk_sched_clone.c
	index 74fedaa7eb..f8ba3abeac 100644
	--- a/lib/pacemaker/pcmk_sched_clone.c
	+++ b/lib/pacemaker/pcmk_sched_clone.c
	@@ -1,739 +1,739 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Assign a clone resource's instances to nodes
	*
	* \param[in,out] rsc Clone resource to assign
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and a primitive descendant of \p rsc
	* can't be assigned to a node, set the
	* descendant's next role to stopped and update
	* existing actions
	*
	* \return NULL (clones are not assigned to a single node)
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pcmk_node_t *
	pcmk__clone_assign(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *colocations = NULL;

	CRM_ASSERT(pcmk__is_clone(rsc));

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
	return NULL; // Assignment has already been done
	}

	// Detect assignment loops
	if (pcmk_is_set(rsc->flags, pcmk__rsc_assigning)) {
	pcmk__rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
	return NULL;
	}
	pcmk__set_rsc_flags(rsc, pcmk__rsc_assigning);

	// If this clone is promotable, consider nodes' promotion scores
	if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
	pcmk__add_promotion_scores(rsc);
	}

	// If this clone is colocated with any other resources, assign those first
	colocations = pcmk__this_with_colocations(rsc);
	for (GList *iter = colocations; iter != NULL; iter = iter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) iter->data;
	pcmk_resource_t *primary = constraint->primary;

	pcmk__rsc_trace(rsc, "%s: Assigning colocation %s primary %s first",
	rsc->id, constraint->id, primary->id);
	primary->private->cmds->assign(primary, prefer, stop_if_fail);
	}
	g_list_free(colocations);

	// If any resources are colocated with this one, consider their preferences
	colocations = pcmk__with_this_colocations(rsc);
	g_list_foreach(colocations, pcmk__add_dependent_scores, rsc);
	g_list_free(colocations);

	pe__show_node_scores(!pcmk_is_set(rsc->private->scheduler->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->private->allowed_nodes,
	rsc->private->scheduler);

	rsc->private->children = g_list_sort(rsc->private->children,
	pcmk__cmp_instance);
	pcmk__assign_instances(rsc, rsc->private->children, pe__clone_max(rsc),
	pe__clone_node_max(rsc));

	if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
	pcmk__set_instance_roles(rsc);
	}

	pcmk__clear_rsc_flags(rsc, pcmk__rsc_unassigned\|pcmk__rsc_assigning);
	pcmk__rsc_trace(rsc, "Assigned clone %s", rsc->id);
	return NULL;
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given clone resource
	*
	* \param[in,out] rsc Clone resource to create actions for
	*/
	void
	pcmk__clone_create_actions(pcmk_resource_t *rsc)
	{
	CRM_ASSERT(pcmk__is_clone(rsc));

	pcmk__rsc_trace(rsc, "Creating actions for clone %s", rsc->id);
	pcmk__create_instance_actions(rsc, rsc->private->children);
	if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
	pcmk__create_promotable_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a clone resource
	*
	* \param[in,out] rsc Clone resource to create implicit constraints for
	*/
	void
	pcmk__clone_internal_constraints(pcmk_resource_t *rsc)
	{
	bool ordered = false;

	CRM_ASSERT(pcmk__is_clone(rsc));

	pcmk__rsc_trace(rsc, "Creating internal constraints for clone %s", rsc->id);

	// Restart ordering: Stop -> stopped -> start -> started
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_START,
	rsc, PCMK_ACTION_RUNNING,
	pcmk__ar_unrunnable_first_blocks);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_STOPPED,
	pcmk__ar_unrunnable_first_blocks);

	// Demoted -> stop and started -> promote
	if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_unrunnable_first_blocks);
	}

	ordered = pe__clone_is_ordered(rsc);
	if (ordered) {
	/* Ordered clone instances must start and stop by instance number. The
	* instances might have been previously shuffled for assignment or
	* promotion purposes, so re-sort them.
	*/
	rsc->private->children = g_list_sort(rsc->private->children,
	pcmk__cmp_instance_number);
	}
	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->private->cmds->internal_constraints(instance);

	// Start clone -> start instance -> clone started
	pcmk__order_starts(rsc, instance, pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_then_implies_first_graphed);
	pcmk__order_resource_actions(instance, PCMK_ACTION_START,
	rsc, PCMK_ACTION_RUNNING,
	pcmk__ar_first_implies_then_graphed);

	// Stop clone -> stop instance -> clone stopped
	pcmk__order_stops(rsc, instance, pcmk__ar_then_implies_first_graphed);
	pcmk__order_resource_actions(instance, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_STOPPED,
	pcmk__ar_first_implies_then_graphed);

	/* Instances of ordered clones must be started and stopped by instance
	* number. Since only some instances may be starting or stopping, order
	* each instance relative to every later instance.
	*/
	if (ordered) {
	for (GList *later = iter->next;
	later != NULL; later = later->next) {
	pcmk__order_starts(instance, (pcmk_resource_t *) later->data,
	pcmk__ar_ordered);
	pcmk__order_stops((pcmk_resource_t *) later->data, instance,
	pcmk__ar_ordered);
	}
	}
	}
	if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
	pcmk__order_promotable_instances(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether colocated resources can be interleaved
	*
	* \param[in] colocation Colocation constraint with clone as primary
	*
	* \return true if colocated resources can be interleaved, otherwise false
	*/
	static bool
	can_interleave(const pcmk__colocation_t *colocation)
	{
	const pcmk_resource_t *primary = colocation->primary;
	const pcmk_resource_t *dependent = colocation->dependent;

	// Only colocations between clone or bundle resources use interleaving
	if (dependent->private->variant <= pcmk__rsc_variant_group) {
	return false;
	}

	// Only the dependent needs to be marked for interleaving
	if (!crm_is_true(g_hash_table_lookup(dependent->private->meta,
	PCMK_META_INTERLEAVE))) {
	return false;
	}

	/* @TODO Do we actually care about multiple primary instances sharing a
	* dependent instance?
	*/
	if (dependent->private->fns->max_per_node(dependent)
	!= primary->private->fns->max_per_node(primary)) {
	pcmk__config_err("Cannot interleave %s and %s because they do not "
	"support the same number of instances per node",
	dependent->id, primary->id);
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__clone_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	const GList *iter = NULL;

	/* This should never be called for the clone itself as a dependent. Instead,
	* we add its colocation constraints to its instances and call the
	* apply_coloc_score() method for the instances as dependents.
	*/
	CRM_ASSERT(!for_dependent);

	CRM_ASSERT((colocation != NULL) && pcmk__is_clone(primary)
	&& pcmk__is_primitive(dependent));

	if (pcmk_is_set(primary->flags, pcmk__rsc_unassigned)) {
	pcmk__rsc_trace(primary,
	"Delaying processing colocation %s "
	"because cloned primary %s is still provisional",
	colocation->id, primary->id);
	return;
	}

	pcmk__rsc_trace(primary, "Processing colocation %s (%s with clone %s @%s)",
	colocation->id, dependent->id, primary->id,
	pcmk_readable_score(colocation->score));

	// Apply role-specific colocations
	if (pcmk_is_set(primary->flags, pcmk__rsc_promotable)
	&& (colocation->primary_role != pcmk_role_unknown)) {

	if (pcmk_is_set(dependent->flags, pcmk__rsc_unassigned)) {
	// We're assigning the dependent to a node
	pcmk__update_dependent_with_promotable(primary, dependent,
	colocation);
	return;
	}

	if (colocation->dependent_role == pcmk_role_promoted) {
	// We're choosing a role for the dependent
	pcmk__update_promotable_dependent_priority(primary, dependent,
	colocation);
	return;
	}
	}

	// Apply interleaved colocations
	if (can_interleave(colocation)) {
	const pcmk_resource_t *primary_instance = NULL;

	primary_instance = pcmk__find_compatible_instance(dependent, primary,
	pcmk_role_unknown,
	false);
	if (primary_instance != NULL) {
	pcmk__rsc_debug(primary, "Interleaving %s with %s",
	dependent->id, primary_instance->id);
	dependent->private->cmds->apply_coloc_score(dependent,
	primary_instance,
	colocation, true);

	} else if (colocation->score >= PCMK_SCORE_INFINITY) {
	crm_notice("%s cannot run because it cannot interleave with "
	"any instance of %s", dependent->id, primary->id);
	pcmk__assign_resource(dependent, NULL, true, true);

	} else {
	pcmk__rsc_debug(primary,
	"%s will not colocate with %s "
	"because no instance can interleave with it",
	dependent->id, primary->id);
	}

	return;
	}

	// Apply mandatory colocations
	if (colocation->score >= PCMK_SCORE_INFINITY) {
	GList *primary_nodes = NULL;

	// Dependent can run only where primary will have unblocked instances
	for (iter = primary->private->children;
	iter != NULL; iter = iter->next) {

	const pcmk_resource_t *instance = iter->data;
	pcmk_node_t *chosen = NULL;

	chosen = instance->private->fns->location(instance, NULL, 0);
	if ((chosen != NULL)
	&& !is_set_recursive(instance, pcmk__rsc_blocked, TRUE)) {
	pcmk__rsc_trace(primary, "Allowing %s: %s %d",
	colocation->id, pcmk__node_name(chosen),
	- chosen->weight);
	+ chosen->assign->score);
	primary_nodes = g_list_prepend(primary_nodes, chosen);
	}
	}
	pcmk__colocation_intersect_nodes(dependent, primary, colocation,
	primary_nodes, false);
	g_list_free(primary_nodes);
	return;
	}

	// Apply optional colocations
	for (iter = primary->private->children; iter != NULL; iter = iter->next) {
	const pcmk_resource_t *instance = iter->data;

	instance->private->cmds->apply_coloc_score(dependent, instance,
	colocation, false);
	}
	}

	// Clone implementation of pcmk__assignment_methods_t:with_this_colocations()
	void
	pcmk__with_clone_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	const pcmk_resource_t *parent = NULL;

	CRM_CHECK((rsc != NULL) && (orig_rsc != NULL) && (list != NULL), return);
	parent = rsc->private->parent;

	pcmk__add_with_this_list(list, rsc->private->with_this_colocations,
	orig_rsc);

	if (parent != NULL) {
	parent->private->cmds->with_this_colocations(parent, orig_rsc, list);
	}
	}

	// Clone implementation of pcmk__assignment_methods_t:this_with_colocations()
	void
	pcmk__clone_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	const pcmk_resource_t *parent = NULL;

	CRM_CHECK((rsc != NULL) && (orig_rsc != NULL) && (list != NULL), return);
	parent = rsc->private->parent;

	pcmk__add_this_with_list(list, rsc->private->this_with_colocations,
	orig_rsc);

	if (parent != NULL) {
	parent->private->cmds->this_with_colocations(parent, orig_rsc, list);
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given clone resource action
	*
	* \param[in,out] action Action to get flags for
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__clone_action_flags(pcmk_action_t action, const pcmk_node_t node)
	{
	CRM_ASSERT((action != NULL) && pcmk__is_clone(action->rsc));

	return pcmk__collective_action_flags(action, action->rsc->private->children,
	node);
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a clone resource's allowed node scores
	*
	* \param[in,out] rsc Clone resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*/
	void
	pcmk__clone_apply_location(pcmk_resource_t rsc, pcmk__location_t location)
	{
	CRM_CHECK((location != NULL) && pcmk__is_clone(rsc), return);

	pcmk__apply_location(rsc, location);

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->private->cmds->apply_location(instance, location);
	}
	}

	// GFunc wrapper for calling the action_flags() resource method
	static void
	call_action_flags(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = user_data;

	rsc->private->cmds->action_flags((pcmk_action_t *) data, NULL);
	}

	/*!
	* \internal
	* \brief Add a clone resource's actions to the transition graph
	*
	* \param[in,out] rsc Resource whose actions should be added
	*/
	void
	pcmk__clone_add_actions_to_graph(pcmk_resource_t *rsc)
	{
	CRM_ASSERT(pcmk__is_clone(rsc));

	g_list_foreach(rsc->private->actions, call_action_flags, rsc);
	pe__create_clone_notifications(rsc);

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t child_rsc = (pcmk_resource_t ) iter->data;

	child_rsc->private->cmds->add_actions_to_graph(child_rsc);
	}

	pcmk__add_rsc_actions_to_graph(rsc);
	pe__free_clone_notification_data(rsc);
	}

	/*!
	* \internal
	* \brief Check whether a resource or any children have been probed on a node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is in the probed_nodes table of \p rsc or any of its
	* children, otherwise false
	*/
	static bool
	rsc_probed_on(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	if (rsc->private->children != NULL) {
	for (GList *child_iter = rsc->private->children;
	child_iter != NULL; child_iter = child_iter->next) {

	pcmk_resource_t child = (pcmk_resource_t ) child_iter->data;

	if (rsc_probed_on(child, node)) {
	return true;
	}
	}
	return false;
	}

	if (rsc->private->probed_nodes != NULL) {
	GHashTableIter iter;
	pcmk_node_t *known_node = NULL;

	g_hash_table_iter_init(&iter, rsc->private->probed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &known_node)) {
	if (pcmk__same_node(node, known_node)) {
	return true;
	}
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Find clone instance that has been probed on given node
	*
	* \param[in] clone Clone resource to check
	* \param[in] node Node to check
	*
	* \return Instance of \p clone that has been probed on \p node if any,
	* otherwise NULL
	*/
	static pcmk_resource_t *
	find_probed_instance_on(const pcmk_resource_t clone, const pcmk_node_t node)
	{
	for (GList *iter = clone->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	if (rsc_probed_on(instance, node)) {
	return instance;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Probe an anonymous clone on a node
	*
	* \param[in,out] clone Anonymous clone to probe
	* \param[in,out] node Node to probe \p clone on
	*/
	static bool
	probe_anonymous_clone(pcmk_resource_t clone, pcmk_node_t node)
	{
	// Check whether we already probed an instance on this node
	pcmk_resource_t *child = find_probed_instance_on(clone, node);

	// Otherwise, check if we plan to start an instance on this node
	for (GList *iter = clone->private->children;
	(iter != NULL) && (child == NULL); iter = iter->next) {

	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;
	const pcmk_node_t *instance_node = NULL;

	instance_node = instance->private->fns->location(instance, NULL, 0);
	if (pcmk__same_node(instance_node, node)) {
	child = instance;
	}
	}

	// Otherwise, use the first clone instance
	if (child == NULL) {
	child = clone->private->children->data;
	}

	// Anonymous clones only need to probe a single instance
	return child->private->cmds->create_probe(child, node);
	}

	/*!
	* \internal
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__clone_create_probe(pcmk_resource_t rsc, pcmk_node_t node)
	{
	CRM_ASSERT((node != NULL) && pcmk__is_clone(rsc));

	if (pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)) {
	/* The clone is configured to be probed only where a location constraint
	* exists with PCMK_XA_RESOURCE_DISCOVERY set to exclusive.
	*
	* This check is not strictly necessary here since the instance's
	* create_probe() method would also check, but doing it here is more
	* efficient (especially for unique clones with a large number of
	* instances), and affects the CRM_meta_notify_available_uname variable
	* passed with notify actions.
	*/
	pcmk_node_t *allowed = g_hash_table_lookup(rsc->private->allowed_nodes,
	node->private->id);

	if ((allowed == NULL)
	- \|\| (allowed->rsc_discover_mode != pcmk_probe_exclusive)) {
	+ \|\| (allowed->assign->probe_mode != pcmk__probe_exclusive)) {
	/* This node is not marked for resource discovery. Remove it from
	* allowed nodes so that notifications contain only nodes that the
	* clone can possibly run on.
	*/
	pcmk__rsc_trace(rsc,
	"Skipping probe for %s on %s because resource has "
	"exclusive discovery but is not allowed on node",
	rsc->id, pcmk__node_name(node));
	g_hash_table_remove(rsc->private->allowed_nodes, node->private->id);
	return false;
	}
	}

	rsc->private->children = g_list_sort(rsc->private->children,
	pcmk__cmp_instance_number);
	if (pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
	return pcmk__probe_resource_list(rsc->private->children, node);
	} else {
	return probe_anonymous_clone(rsc, node);
	}
	}

	/*!
	* \internal
	* \brief Add meta-attributes relevant to transition graph actions to XML
	*
	* Add clone-specific meta-attributes needed for transition graph actions.
	*
	* \param[in] rsc Clone resource whose meta-attributes should be added
	* \param[in,out] xml Transition graph action attributes XML to add to
	*/
	void
	pcmk__clone_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml)
	{
	char *name = NULL;

	CRM_ASSERT(pcmk__is_clone(rsc) && (xml != NULL));

	name = crm_meta_name(PCMK_META_GLOBALLY_UNIQUE);
	crm_xml_add(xml, name, pcmk__flag_text(rsc->flags, pcmk__rsc_unique));
	free(name);

	name = crm_meta_name(PCMK_META_NOTIFY);
	crm_xml_add(xml, name, pcmk__flag_text(rsc->flags, pcmk__rsc_notify));
	free(name);

	name = crm_meta_name(PCMK_META_CLONE_MAX);
	crm_xml_add_int(xml, name, pe__clone_max(rsc));
	free(name);

	name = crm_meta_name(PCMK_META_CLONE_NODE_MAX);
	crm_xml_add_int(xml, name, pe__clone_node_max(rsc));
	free(name);

	if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
	int promoted_max = pe__clone_promoted_max(rsc);
	int promoted_node_max = pe__clone_promoted_node_max(rsc);

	name = crm_meta_name(PCMK_META_PROMOTED_MAX);
	crm_xml_add_int(xml, name, promoted_max);
	free(name);

	name = crm_meta_name(PCMK_META_PROMOTED_NODE_MAX);
	crm_xml_add_int(xml, name, promoted_node_max);
	free(name);

	/* @COMPAT Maintain backward compatibility with resource agents that
	* expect the old names (deprecated since 2.0.0).
	*/
	name = crm_meta_name(PCMK__META_PROMOTED_MAX_LEGACY);
	crm_xml_add_int(xml, name, promoted_max);
	free(name);

	name = crm_meta_name(PCMK__META_PROMOTED_NODE_MAX_LEGACY);
	crm_xml_add_int(xml, name, promoted_node_max);
	free(name);
	}
	}

	// Clone implementation of pcmk__assignment_methods_t:add_utilization()
	void
	pcmk__clone_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization)
	{
	bool existing = false;
	pcmk_resource_t *child = NULL;

	CRM_ASSERT(pcmk__is_clone(rsc) && (orig_rsc != NULL)
	&& (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
	return;
	}

	// Look for any child already existing in the list
	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	child = (pcmk_resource_t *) iter->data;
	if (g_list_find(all_rscs, child)) {
	existing = true; // Keep checking remaining children
	} else {
	// If this is a clone of a group, look for group's members
	for (GList *member_iter = child->private->children;
	member_iter != NULL; member_iter = member_iter->next) {

	pcmk_resource_t member = (pcmk_resource_t ) member_iter->data;

	if (g_list_find(all_rscs, member) != NULL) {
	// Add child's utilization, not group member's
	child->private->cmds->add_utilization(child, orig_rsc,
	all_rscs,
	utilization);
	existing = true;
	break;
	}
	}
	}
	}

	if (!existing && (rsc->private->children != NULL)) {
	// If nothing was found, still add first child's utilization
	child = (pcmk_resource_t *) rsc->private->children->data;

	child->private->cmds->add_utilization(child, orig_rsc, all_rscs,
	utilization);
	}
	}

	// Clone implementation of pcmk__assignment_methods_t:shutdown_lock()
	void
	pcmk__clone_shutdown_lock(pcmk_resource_t *rsc)
	{
	CRM_ASSERT(pcmk__is_clone(rsc));
	return; // Clones currently don't support shutdown locks
	}
	diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c
	index 9e32a3cb4d..aaa31a3a13 100644
	--- a/lib/pacemaker/pcmk_sched_colocation.c
	+++ b/lib/pacemaker/pcmk_sched_colocation.c
	@@ -1,1951 +1,1955 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include "crm/common/util.h"
	#include "crm/common/xml_internal.h"
	#include "crm/common/xml.h"
	#include "libpacemaker_private.h"

	// Used to temporarily mark a node as unusable
	#define INFINITY_HACK (PCMK_SCORE_INFINITY * -100)

	/*!
	* \internal
	* \brief Compare two colocations according to priority
	*
	* Compare two colocations according to the order in which they should be
	* considered, based on either their dependent resources or their primary
	* resources -- preferring (in order):
	* * Colocation that is not \c NULL
	* * Colocation whose resource has higher priority
	* * Colocation whose resource is of a higher-level variant
	* (bundle > clone > group > primitive)
	* * Colocation whose resource is promotable, if both are clones
	* * Colocation whose resource has lower ID in lexicographic order
	*
	* \param[in] colocation1 First colocation to compare
	* \param[in] colocation2 Second colocation to compare
	* \param[in] dependent If \c true, compare colocations by dependent
	* priority; otherwise compare them by primary priority
	*
	* \return A negative number if \p colocation1 should be considered first,
	* a positive number if \p colocation2 should be considered first,
	* or 0 if order doesn't matter
	*/
	static gint
	cmp_colocation_priority(const pcmk__colocation_t *colocation1,
	const pcmk__colocation_t *colocation2, bool dependent)
	{
	const pcmk_resource_t *rsc1 = NULL;
	const pcmk_resource_t *rsc2 = NULL;

	if (colocation1 == NULL) {
	return 1;
	}
	if (colocation2 == NULL) {
	return -1;
	}

	if (dependent) {
	rsc1 = colocation1->dependent;
	rsc2 = colocation2->dependent;
	CRM_ASSERT(colocation1->primary != NULL);
	} else {
	rsc1 = colocation1->primary;
	rsc2 = colocation2->primary;
	CRM_ASSERT(colocation1->dependent != NULL);
	}
	CRM_ASSERT((rsc1 != NULL) && (rsc2 != NULL));

	if (rsc1->private->priority > rsc2->private->priority) {
	return -1;
	}
	if (rsc1->private->priority < rsc2->private->priority) {
	return 1;
	}

	// Process clones before primitives and groups
	if (rsc1->private->variant > rsc2->private->variant) {
	return -1;
	}
	if (rsc1->private->variant < rsc2->private->variant) {
	return 1;
	}

	/* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
	* clones (probably unnecessary, but avoids having to update regression
	* tests)
	*/
	if (pcmk__is_clone(rsc1)) {
	if (pcmk_is_set(rsc1->flags, pcmk__rsc_promotable)
	&& !pcmk_is_set(rsc2->flags, pcmk__rsc_promotable)) {
	return -1;
	}
	if (!pcmk_is_set(rsc1->flags, pcmk__rsc_promotable)
	&& pcmk_is_set(rsc2->flags, pcmk__rsc_promotable)) {
	return 1;
	}
	}

	return strcmp(rsc1->id, rsc2->id);
	}

	/*!
	* \internal
	* \brief Compare two colocations according to priority based on dependents
	*
	* Compare two colocations according to the order in which they should be
	* considered, based on their dependent resources -- preferring (in order):
	* * Colocation that is not \c NULL
	* * Colocation whose resource has higher priority
	* * Colocation whose resource is of a higher-level variant
	* (bundle > clone > group > primitive)
	* * Colocation whose resource is promotable, if both are clones
	* * Colocation whose resource has lower ID in lexicographic order
	*
	* \param[in] a First colocation to compare
	* \param[in] b Second colocation to compare
	*
	* \return A negative number if \p a should be considered first,
	* a positive number if \p b should be considered first,
	* or 0 if order doesn't matter
	*/
	static gint
	cmp_dependent_priority(gconstpointer a, gconstpointer b)
	{
	return cmp_colocation_priority(a, b, true);
	}

	/*!
	* \internal
	* \brief Compare two colocations according to priority based on primaries
	*
	* Compare two colocations according to the order in which they should be
	* considered, based on their primary resources -- preferring (in order):
	* * Colocation that is not \c NULL
	* * Colocation whose primary has higher priority
	* * Colocation whose primary is of a higher-level variant
	* (bundle > clone > group > primitive)
	* * Colocation whose primary is promotable, if both are clones
	* * Colocation whose primary has lower ID in lexicographic order
	*
	* \param[in] a First colocation to compare
	* \param[in] b Second colocation to compare
	*
	* \return A negative number if \p a should be considered first,
	* a positive number if \p b should be considered first,
	* or 0 if order doesn't matter
	*/
	static gint
	cmp_primary_priority(gconstpointer a, gconstpointer b)
	{
	return cmp_colocation_priority(a, b, false);
	}

	/*!
	* \internal
	* \brief Add a "this with" colocation constraint to a sorted list
	*
	* \param[in,out] list List of constraints to add \p colocation to
	* \param[in] colocation Colocation constraint to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The list will be sorted using cmp_primary_priority().
	*/
	void
	pcmk__add_this_with(GList *list, const pcmk__colocation_t colocation,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (colocation != NULL) && (rsc != NULL));

	pcmk__rsc_trace(rsc,
	"Adding colocation %s (%s with %s using %s @%s) to "
	"'this with' list for %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, colocation->node_attribute,
	pcmk_readable_score(colocation->score), rsc->id);
	list = g_list_insert_sorted(list, (gpointer) colocation,
	cmp_primary_priority);
	}

	/*!
	* \internal
	* \brief Add a list of "this with" colocation constraints to a list
	*
	* \param[in,out] list List of constraints to add \p addition to
	* \param[in] addition List of colocation constraints to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The lists must be pre-sorted by cmp_primary_priority().
	*/
	void
	pcmk__add_this_with_list(GList *list, GList addition,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (rsc != NULL));

	pcmk__if_tracing(
	{}, // Always add each colocation individually if tracing
	{
	if (*list == NULL) {
	// Trivial case for efficiency if not tracing
	*list = g_list_copy(addition);
	return;
	}
	}
	);

	for (const GList *iter = addition; iter != NULL; iter = iter->next) {
	pcmk__add_this_with(list, addition->data, rsc);
	}
	}

	/*!
	* \internal
	* \brief Add a "with this" colocation constraint to a sorted list
	*
	* \param[in,out] list List of constraints to add \p colocation to
	* \param[in] colocation Colocation constraint to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The list will be sorted using cmp_dependent_priority().
	*/
	void
	pcmk__add_with_this(GList *list, const pcmk__colocation_t colocation,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (colocation != NULL) && (rsc != NULL));

	pcmk__rsc_trace(rsc,
	"Adding colocation %s (%s with %s using %s @%s) to "
	"'with this' list for %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, colocation->node_attribute,
	pcmk_readable_score(colocation->score), rsc->id);
	list = g_list_insert_sorted(list, (gpointer) colocation,
	cmp_dependent_priority);
	}

	/*!
	* \internal
	* \brief Add a list of "with this" colocation constraints to a list
	*
	* \param[in,out] list List of constraints to add \p addition to
	* \param[in] addition List of colocation constraints to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The lists must be pre-sorted by cmp_dependent_priority().
	*/
	void
	pcmk__add_with_this_list(GList *list, GList addition,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (rsc != NULL));

	pcmk__if_tracing(
	{}, // Always add each colocation individually if tracing
	{
	if (*list == NULL) {
	// Trivial case for efficiency if not tracing
	*list = g_list_copy(addition);
	return;
	}
	}
	);

	for (const GList *iter = addition; iter != NULL; iter = iter->next) {
	pcmk__add_with_this(list, addition->data, rsc);
	}
	}

	/*!
	* \internal
	* \brief Add orderings necessary for an anti-colocation constraint
	*
	* \param[in,out] first_rsc One resource in an anti-colocation
	* \param[in] first_role Anti-colocation role of \p first_rsc
	* \param[in] then_rsc Other resource in the anti-colocation
	* \param[in] then_role Anti-colocation role of \p then_rsc
	*/
	static void
	anti_colocation_order(pcmk_resource_t *first_rsc, int first_role,
	pcmk_resource_t *then_rsc, int then_role)
	{
	const char *first_tasks[] = { NULL, NULL };
	const char *then_tasks[] = { NULL, NULL };

	/* Actions to make first_rsc lose first_role */
	if (first_role == pcmk_role_promoted) {
	first_tasks[0] = PCMK_ACTION_DEMOTE;

	} else {
	first_tasks[0] = PCMK_ACTION_STOP;

	if (first_role == pcmk_role_unpromoted) {
	first_tasks[1] = PCMK_ACTION_PROMOTE;
	}
	}

	/* Actions to make then_rsc gain then_role */
	if (then_role == pcmk_role_promoted) {
	then_tasks[0] = PCMK_ACTION_PROMOTE;

	} else {
	then_tasks[0] = PCMK_ACTION_START;

	if (then_role == pcmk_role_unpromoted) {
	then_tasks[1] = PCMK_ACTION_DEMOTE;
	}
	}

	for (int first_lpc = 0;
	(first_lpc <= 1) && (first_tasks[first_lpc] != NULL); first_lpc++) {

	for (int then_lpc = 0;
	(then_lpc <= 1) && (then_tasks[then_lpc] != NULL); then_lpc++) {

	pcmk__order_resource_actions(first_rsc, first_tasks[first_lpc],
	then_rsc, then_tasks[then_lpc],
	pcmk__ar_if_required_on_same_node);
	}
	}
	}

	/*!
	* \internal
	* \brief Add a new colocation constraint to scheduler data
	*
	* \param[in] id XML ID for this constraint
	* \param[in] node_attr Colocate by this attribute (NULL for #uname)
	* \param[in] score Constraint score
	* \param[in,out] dependent Resource to be colocated
	* \param[in,out] primary Resource to colocate \p dependent with
	* \param[in] dependent_role Current role of \p dependent
	* \param[in] primary_role Current role of \p primary
	* \param[in] flags Group of enum pcmk__coloc_flags
	*/
	void
	pcmk__new_colocation(const char id, const char node_attr, int score,
	pcmk_resource_t dependent, pcmk_resource_t primary,
	const char dependent_role, const char primary_role,
	uint32_t flags)
	{
	pcmk__colocation_t *new_con = NULL;

	CRM_CHECK(id != NULL, return);

	if ((dependent == NULL) \|\| (primary == NULL)) {
	pcmk__config_err("Ignoring colocation '%s' because resource "
	"does not exist", id);
	return;
	}

	if (score == 0) {
	pcmk__rsc_trace(dependent,
	"Ignoring colocation '%s' (%s with %s) because score is 0",
	id, dependent->id, primary->id);
	return;
	}

	new_con = pcmk__assert_alloc(1, sizeof(pcmk__colocation_t));

	if (pcmk__str_eq(dependent_role, PCMK_ROLE_STARTED,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	dependent_role = PCMK__ROLE_UNKNOWN;
	}

	if (pcmk__str_eq(primary_role, PCMK_ROLE_STARTED,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	primary_role = PCMK__ROLE_UNKNOWN;
	}

	new_con->id = id;
	new_con->dependent = dependent;
	new_con->primary = primary;
	new_con->score = score;
	new_con->dependent_role = pcmk_parse_role(dependent_role);
	new_con->primary_role = pcmk_parse_role(primary_role);
	new_con->node_attribute = pcmk__s(node_attr, CRM_ATTR_UNAME);
	new_con->flags = flags;

	pcmk__add_this_with(&(dependent->private->this_with_colocations), new_con,
	dependent);
	pcmk__add_with_this(&(primary->private->with_this_colocations), new_con,
	primary);

	dependent->private->scheduler->colocation_constraints =
	g_list_prepend(dependent->private->scheduler->colocation_constraints,
	new_con);

	if (score <= -PCMK_SCORE_INFINITY) {
	anti_colocation_order(dependent, new_con->dependent_role, primary,
	new_con->primary_role);
	anti_colocation_order(primary, new_con->primary_role, dependent,
	new_con->dependent_role);
	}
	}

	/*!
	* \internal
	* \brief Return the boolean influence corresponding to configuration
	*
	* \param[in] coloc_id Colocation XML ID (for error logging)
	* \param[in] rsc Resource involved in constraint (for default)
	* \param[in] influence_s String value of \c PCMK_XA_INFLUENCE option
	*
	* \return \c pcmk__coloc_influence if string evaluates true, or string is
	* \c NULL or invalid and resource's \c PCMK_META_CRITICAL option
	* evaluates true, otherwise \c pcmk__coloc_none
	*/
	static uint32_t
	unpack_influence(const char coloc_id, const pcmk_resource_t rsc,
	const char *influence_s)
	{
	if (influence_s != NULL) {
	int influence_i = 0;

	if (crm_str_to_boolean(influence_s, &influence_i) < 0) {
	pcmk__config_err("Constraint '%s' has invalid value for "
	PCMK_XA_INFLUENCE " (using default)",
	coloc_id);
	} else {
	return (influence_i == 0)? pcmk__coloc_none : pcmk__coloc_influence;
	}
	}
	if (pcmk_is_set(rsc->flags, pcmk__rsc_critical)) {
	return pcmk__coloc_influence;
	}
	return pcmk__coloc_none;
	}

	static void
	unpack_colocation_set(xmlNode set, int score, const char coloc_id,
	const char influence_s, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_rsc = NULL;
	pcmk_resource_t *other = NULL;
	pcmk_resource_t *resource = NULL;
	const char *set_id = pcmk__xe_id(set);
	const char *role = crm_element_value(set, PCMK_XA_ROLE);
	bool with_previous = false;
	int local_score = score;
	bool sequential = false;
	uint32_t flags = pcmk__coloc_none;
	const char *xml_rsc_id = NULL;
	const char *score_s = crm_element_value(set, PCMK_XA_SCORE);

	if (score_s) {
	local_score = char2score(score_s);
	}
	if (local_score == 0) {
	crm_trace("Ignoring colocation '%s' for set '%s' because score is 0",
	coloc_id, set_id);
	return;
	}

	/* @COMPAT The deprecated PCMK__XA_ORDERING attribute specifies whether
	* resources in a positive-score set are colocated with the previous or next
	* resource.
	*/
	if (pcmk__str_eq(crm_element_value(set, PCMK__XA_ORDERING),
	PCMK__VALUE_GROUP,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	with_previous = true;
	} else {
	pcmk__warn_once(pcmk__wo_set_ordering,
	"Support for '" PCMK__XA_ORDERING "' other than"
	" '" PCMK__VALUE_GROUP "' in " PCMK_XE_RESOURCE_SET
	" (such as %s) is deprecated and will be removed in a"
	" future release",
	set_id);
	}

	if ((pcmk__xe_get_bool_attr(set, PCMK_XA_SEQUENTIAL,
	&sequential) == pcmk_rc_ok)
	&& !sequential) {
	return;
	}

	if (local_score > 0) {
	for (xml_rsc = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	xml_rsc_id = pcmk__xe_id(xml_rsc);
	resource = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (resource == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring %s and later resources in set %s: "
	"No such resource", xml_rsc_id, set_id);
	return;
	}
	if (other != NULL) {
	flags = pcmk__coloc_explicit
	\| unpack_influence(coloc_id, resource, influence_s);
	if (with_previous) {
	pcmk__rsc_trace(resource, "Colocating %s with %s in set %s",
	resource->id, other->id, set_id);
	pcmk__new_colocation(set_id, NULL, local_score, resource,
	other, role, role, flags);
	} else {
	pcmk__rsc_trace(resource, "Colocating %s with %s in set %s",
	other->id, resource->id, set_id);
	pcmk__new_colocation(set_id, NULL, local_score, other,
	resource, role, role, flags);
	}
	}
	other = resource;
	}

	} else {
	/* Anti-colocating with every prior resource is
	* the only way to ensure the intuitive result
	* (i.e. that no one in the set can run with anyone else in the set)
	*/

	for (xml_rsc = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	xmlNode *xml_rsc_with = NULL;

	xml_rsc_id = pcmk__xe_id(xml_rsc);
	resource = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (resource == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring %s and later resources in set %s: "
	"No such resource", xml_rsc_id, set_id);
	return;
	}
	flags = pcmk__coloc_explicit
	\| unpack_influence(coloc_id, resource, influence_s);
	for (xml_rsc_with = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF,
	NULL, NULL);
	xml_rsc_with != NULL;
	xml_rsc_with = pcmk__xe_next_same(xml_rsc_with)) {

	xml_rsc_id = pcmk__xe_id(xml_rsc_with);
	if (pcmk__str_eq(resource->id, xml_rsc_id, pcmk__str_none)) {
	break;
	}
	other = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	CRM_ASSERT(other != NULL); // We already processed it
	pcmk__new_colocation(set_id, NULL, local_score,
	resource, other, role, role, flags);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Colocate two resource sets relative to each other
	*
	* \param[in] id Colocation XML ID
	* \param[in] set1 Dependent set
	* \param[in] set2 Primary set
	* \param[in] score Colocation score
	* \param[in] influence_s Value of colocation's \c PCMK_XA_INFLUENCE
	* attribute
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	colocate_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	int score, const char *influence_s,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *xml_rsc = NULL;
	pcmk_resource_t *rsc_1 = NULL;
	pcmk_resource_t *rsc_2 = NULL;

	const char *xml_rsc_id = NULL;
	const char *role_1 = crm_element_value(set1, PCMK_XA_ROLE);
	const char *role_2 = crm_element_value(set2, PCMK_XA_ROLE);

	int rc = pcmk_rc_ok;
	bool sequential = false;
	uint32_t flags = pcmk__coloc_none;

	if (score == 0) {
	crm_trace("Ignoring colocation '%s' between sets %s and %s "
	"because score is 0",
	id, pcmk__xe_id(set1), pcmk__xe_id(set2));
	return;
	}

	rc = pcmk__xe_get_bool_attr(set1, PCMK_XA_SEQUENTIAL, &sequential);
	if ((rc != pcmk_rc_ok) \|\| sequential) {
	// Get the first one
	xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL, NULL);
	if (xml_rsc != NULL) {
	xml_rsc_id = pcmk__xe_id(xml_rsc);
	rsc_1 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_1 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s with set %s "
	"because first resource %s not found",
	pcmk__xe_id(set1), pcmk__xe_id(set2),
	xml_rsc_id);
	return;
	}
	}
	}

	rc = pcmk__xe_get_bool_attr(set2, PCMK_XA_SEQUENTIAL, &sequential);
	if ((rc != pcmk_rc_ok) \|\| sequential) {
	// Get the last one
	for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	xml_rsc_id = pcmk__xe_id(xml_rsc);
	}
	rsc_2 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_2 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s with set %s "
	"because last resource %s not found",
	pcmk__xe_id(set1), pcmk__xe_id(set2), xml_rsc_id);
	return;
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) { // Both sets are sequential
	flags = pcmk__coloc_explicit \| unpack_influence(id, rsc_1, influence_s);
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2,
	flags);

	} else if (rsc_1 != NULL) { // Only set1 is sequential
	flags = pcmk__coloc_explicit \| unpack_influence(id, rsc_1, influence_s);
	for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	xml_rsc_id = pcmk__xe_id(xml_rsc);
	rsc_2 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_2 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring set %s colocation with resource %s "
	"in set %s: No such resource",
	pcmk__xe_id(set1), xml_rsc_id,
	pcmk__xe_id(set2));
	continue;
	}
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	role_2, flags);
	}

	} else if (rsc_2 != NULL) { // Only set2 is sequential
	for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	xml_rsc_id = pcmk__xe_id(xml_rsc);
	rsc_1 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_1 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s resource %s "
	"with set %s: No such resource",
	pcmk__xe_id(set1), xml_rsc_id,
	pcmk__xe_id(set2));
	continue;
	}
	flags = pcmk__coloc_explicit
	\| unpack_influence(id, rsc_1, influence_s);
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	role_2, flags);
	}

	} else { // Neither set is sequential
	for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	xmlNode *xml_rsc_2 = NULL;

	xml_rsc_id = pcmk__xe_id(xml_rsc);
	rsc_1 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_1 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s resource %s "
	"with set %s: No such resource",
	pcmk__xe_id(set1), xml_rsc_id,
	pcmk__xe_id(set2));
	continue;
	}

	flags = pcmk__coloc_explicit
	\| unpack_influence(id, rsc_1, influence_s);
	for (xml_rsc_2 = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF,
	NULL, NULL);
	xml_rsc_2 != NULL; xml_rsc_2 = pcmk__xe_next_same(xml_rsc_2)) {

	xml_rsc_id = pcmk__xe_id(xml_rsc_2);
	rsc_2 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_2 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s resource "
	"%s with set %s resource %s: No such "
	"resource",
	pcmk__xe_id(set1), pcmk__xe_id(xml_rsc),
	pcmk__xe_id(set2), xml_rsc_id);
	continue;
	}
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2,
	role_1, role_2, flags);
	}
	}
	}
	}

	static void
	unpack_simple_colocation(xmlNode xml_obj, const char id,
	const char influence_s, pcmk_scheduler_t scheduler)
	{
	int score_i = 0;
	uint32_t flags = pcmk__coloc_none;

	const char *score = crm_element_value(xml_obj, PCMK_XA_SCORE);
	const char *dependent_id = crm_element_value(xml_obj, PCMK_XA_RSC);
	const char *primary_id = crm_element_value(xml_obj, PCMK_XA_WITH_RSC);
	const char *dependent_role = crm_element_value(xml_obj, PCMK_XA_RSC_ROLE);
	const char *primary_role = crm_element_value(xml_obj,
	PCMK_XA_WITH_RSC_ROLE);
	const char *attr = crm_element_value(xml_obj, PCMK_XA_NODE_ATTRIBUTE);

	const char *primary_instance = NULL;
	const char *dependent_instance = NULL;
	pcmk_resource_t *primary = NULL;
	pcmk_resource_t *dependent = NULL;

	primary = pcmk__find_constraint_resource(scheduler->resources, primary_id);
	dependent = pcmk__find_constraint_resource(scheduler->resources,
	dependent_id);

	// @COMPAT: Deprecated since 2.1.5
	primary_instance = crm_element_value(xml_obj, PCMK__XA_WITH_RSC_INSTANCE);
	dependent_instance = crm_element_value(xml_obj, PCMK__XA_RSC_INSTANCE);
	if (dependent_instance != NULL) {
	pcmk__warn_once(pcmk__wo_coloc_inst,
	"Support for " PCMK__XA_RSC_INSTANCE " is deprecated "
	"and will be removed in a future release");
	}
	if (primary_instance != NULL) {
	pcmk__warn_once(pcmk__wo_coloc_inst,
	"Support for " PCMK__XA_WITH_RSC_INSTANCE " is "
	"deprecated and will be removed in a future release");
	}

	if (dependent == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, dependent_id);
	return;

	} else if (primary == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, primary_id);
	return;

	} else if ((dependent_instance != NULL) && !pcmk__is_clone(dependent)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	id, dependent_id, dependent_instance);
	return;

	} else if ((primary_instance != NULL) && !pcmk__is_clone(primary)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	id, primary_id, primary_instance);
	return;
	}

	if (dependent_instance != NULL) {
	dependent = find_clone_instance(dependent, dependent_instance);
	if (dependent == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	id, dependent_id, dependent_instance);
	return;
	}
	}

	if (primary_instance != NULL) {
	primary = find_clone_instance(primary, primary_instance);
	if (primary == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", id, primary_id, primary_instance);
	return;
	}
	}

	if (pcmk__xe_attr_is_true(xml_obj, PCMK_XA_SYMMETRICAL)) {
	pcmk__config_warn("The colocation constraint "
	"'" PCMK_XA_SYMMETRICAL "' attribute has been "
	"removed");
	}

	if (score) {
	score_i = char2score(score);
	}

	flags = pcmk__coloc_explicit \| unpack_influence(id, dependent, influence_s);
	pcmk__new_colocation(id, attr, score_i, dependent, primary,
	dependent_role, primary_role, flags);
	}

	// \return Standard Pacemaker return code
	static int
	unpack_colocation_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	const char *id = NULL;
	const char *dependent_id = NULL;
	const char *primary_id = NULL;
	const char *dependent_role = NULL;
	const char *primary_role = NULL;

	pcmk_resource_t *dependent = NULL;
	pcmk_resource_t *primary = NULL;

	pcmk_tag_t *dependent_tag = NULL;
	pcmk_tag_t *primary_tag = NULL;

	xmlNode *dependent_set = NULL;
	xmlNode *primary_set = NULL;
	bool any_sets = false;

	*expanded_xml = NULL;

	CRM_CHECK(xml_obj != NULL, return EINVAL);

	id = pcmk__xe_id(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_COLOCATION);
	return pcmk_rc_ok;
	}

	dependent_id = crm_element_value(xml_obj, PCMK_XA_RSC);
	primary_id = crm_element_value(xml_obj, PCMK_XA_WITH_RSC);
	if ((dependent_id == NULL) \|\| (primary_id == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, dependent_id, &dependent,
	&dependent_tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, dependent_id);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, primary_id, &primary,
	&primary_tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, primary_id);
	return pcmk_rc_unpack_error;
	}

	if ((dependent != NULL) && (primary != NULL)) {
	/* Neither side references any template/tag. */
	return pcmk_rc_ok;
	}

	if ((dependent_tag != NULL) && (primary_tag != NULL)) {
	// A colocation constraint between two templates/tags makes no sense
	pcmk__config_err("Ignoring constraint '%s' because two templates or "
	"tags cannot be colocated", id);
	return pcmk_rc_unpack_error;
	}

	dependent_role = crm_element_value(xml_obj, PCMK_XA_RSC_ROLE);
	primary_role = crm_element_value(xml_obj, PCMK_XA_WITH_RSC_ROLE);

	*expanded_xml = pcmk__xml_copy(NULL, xml_obj);

	/* Convert dependent's template/tag reference into constraint
	* PCMK_XE_RESOURCE_SET
	*/
	if (!pcmk__tag_to_set(*expanded_xml, &dependent_set, PCMK_XA_RSC, true,
	scheduler)) {
	pcmk__xml_free(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (dependent_set != NULL) {
	if (dependent_role != NULL) {
	/* Move PCMK_XA_RSC_ROLE into converted PCMK_XE_RESOURCE_SET as
	* PCMK_XA_ROLE
	*/
	crm_xml_add(dependent_set, PCMK_XA_ROLE, dependent_role);
	pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_RSC_ROLE);
	}
	any_sets = true;
	}

	/* Convert primary's template/tag reference into constraint
	* PCMK_XE_RESOURCE_SET
	*/
	if (!pcmk__tag_to_set(*expanded_xml, &primary_set, PCMK_XA_WITH_RSC, true,
	scheduler)) {
	pcmk__xml_free(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (primary_set != NULL) {
	if (primary_role != NULL) {
	/* Move PCMK_XA_WITH_RSC_ROLE into converted PCMK_XE_RESOURCE_SET as
	* PCMK_XA_ROLE
	*/
	crm_xml_add(primary_set, PCMK_XA_ROLE, primary_role);
	pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_WITH_RSC_ROLE);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_COLOCATION);
	} else {
	pcmk__xml_free(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Parse a colocation constraint from XML into scheduler data
	*
	* \param[in,out] xml_obj Colocation constraint XML to unpack
	* \param[in,out] scheduler Scheduler data to add constraint to
	*/
	void
	pcmk__unpack_colocation(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	int score_i = 0;
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
	const char *score = NULL;
	const char *influence_s = NULL;

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring " PCMK_XE_RSC_COLOCATION
	" without " CRM_ATTR_ID);
	return;
	}

	if (unpack_colocation_tags(xml_obj, &expanded_xml,
	scheduler) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	score = crm_element_value(xml_obj, PCMK_XA_SCORE);
	if (score != NULL) {
	score_i = char2score(score);
	}
	influence_s = crm_element_value(xml_obj, PCMK_XA_INFLUENCE);

	for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL);
	set != NULL; set = pcmk__xe_next_same(set)) {

	set = pcmk__xe_resolve_idref(set, scheduler->input);
	if (set == NULL) { // Configuration error, message already logged
	if (expanded_xml != NULL) {
	pcmk__xml_free(expanded_xml);
	}
	return;
	}

	if (pcmk__str_empty(pcmk__xe_id(set))) {
	pcmk__config_err("Ignoring " PCMK_XE_RESOURCE_SET
	" without " CRM_ATTR_ID);
	continue;
	}
	unpack_colocation_set(set, score_i, id, influence_s, scheduler);

	if (last != NULL) {
	colocate_rsc_sets(id, last, set, score_i, influence_s, scheduler);
	}
	last = set;
	}

	if (expanded_xml) {
	pcmk__xml_free(expanded_xml);
	xml_obj = orig_xml;
	}

	if (last == NULL) {
	unpack_simple_colocation(xml_obj, id, influence_s, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Make actions of a given type unrunnable for a given resource
	*
	* \param[in,out] rsc Resource whose actions should be blocked
	* \param[in] task Name of action to block
	* \param[in] reason Unrunnable start action causing the block
	*/
	static void
	mark_action_blocked(pcmk_resource_t rsc, const char task,
	const pcmk_resource_t *reason)
	{
	GList *iter = NULL;
	char *reason_text = crm_strdup_printf("colocation with %s", reason->id);

	for (iter = rsc->private->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t *action = iter->data;

	if (pcmk_is_set(action->flags, pcmk_action_runnable)
	&& pcmk__str_eq(action->task, task, pcmk__str_none)) {

	pcmk__clear_action_flags(action, pcmk_action_runnable);
	pe_action_set_reason(action, reason_text, false);
	pcmk__block_colocation_dependents(action);
	pcmk__update_action_for_orderings(action, rsc->private->scheduler);
	}
	}

	// If parent resource can't perform an action, neither can any children
	for (iter = rsc->private->children; iter != NULL; iter = iter->next) {
	mark_action_blocked((pcmk_resource_t *) (iter->data), task, reason);
	}
	free(reason_text);
	}

	/*!
	* \internal
	* \brief If an action is unrunnable, block any relevant dependent actions
	*
	* If a given action is an unrunnable start or promote, block the start or
	* promote actions of resources colocated with it, as appropriate to the
	* colocations' configured roles.
	*
	* \param[in,out] action Action to check
	*/
	void
	pcmk__block_colocation_dependents(pcmk_action_t *action)
	{
	GList *iter = NULL;
	GList *colocations = NULL;
	pcmk_resource_t *rsc = NULL;
	bool is_start = false;

	if (pcmk_is_set(action->flags, pcmk_action_runnable)) {
	return; // Only unrunnable actions block dependents
	}

	is_start = pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_none);
	if (!is_start
	&& !pcmk__str_eq(action->task, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return; // Only unrunnable starts and promotes block dependents
	}

	CRM_ASSERT(action->rsc != NULL); // Start and promote are resource actions

	/* If this resource is part of a collective resource, dependents are blocked
	* only if all instances of the collective are unrunnable, so check the
	* collective resource.
	*/
	rsc = uber_parent(action->rsc);
	if (rsc->private->parent != NULL) {
	rsc = rsc->private->parent; // Bundle
	}

	// Colocation fails only if entire primary can't reach desired role
	for (iter = rsc->private->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t *child = iter->data;
	pcmk_action_t *child_action = NULL;

	child_action = find_first_action(child->private->actions, NULL,
	action->task, NULL);
	if ((child_action == NULL)
	\|\| pcmk_is_set(child_action->flags, pcmk_action_runnable)) {
	crm_trace("Not blocking %s colocation dependents because "
	"at least %s has runnable %s",
	rsc->id, child->id, action->task);
	return; // At least one child can reach desired role
	}
	}

	crm_trace("Blocking %s colocation dependents due to unrunnable %s %s",
	rsc->id, action->rsc->id, action->task);

	// Check each colocation where this resource is primary
	colocations = pcmk__with_this_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (colocation->score < PCMK_SCORE_INFINITY) {
	continue; // Only mandatory colocations block dependent
	}

	/* If the primary can't start, the dependent can't reach its colocated
	* role, regardless of what the primary or dependent colocation role is.
	*
	* If the primary can't be promoted, the dependent can't reach its
	* colocated role if the primary's colocation role is promoted.
	*/
	if (!is_start && (colocation->primary_role != pcmk_role_promoted)) {
	continue;
	}

	// Block the dependent from reaching its colocated role
	if (colocation->dependent_role == pcmk_role_promoted) {
	mark_action_blocked(colocation->dependent, PCMK_ACTION_PROMOTE,
	action->rsc);
	} else {
	mark_action_blocked(colocation->dependent, PCMK_ACTION_START,
	action->rsc);
	}
	}
	g_list_free(colocations);
	}

	/*!
	* \internal
	* \brief Get the resource to use for role comparisons
	*
	* A bundle replica includes a container and possibly an instance of the bundled
	* resource. The dependent in a "with bundle" colocation is colocated with a
	* particular bundle container. However, if the colocation includes a role, then
	* the role must be checked on the bundled resource instance inside the
	* container. The container itself will never be promoted; the bundled resource
	* may be.
	*
	* If the given resource is a bundle replica container, return the resource
	* inside it, if any. Otherwise, return the resource itself.
	*
	* \param[in] rsc Resource to check
	*
	* \return Resource to use for role comparisons
	*/
	static const pcmk_resource_t *
	get_resource_for_role(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk__rsc_replica_container)) {
	const pcmk_resource_t *child = pe__get_rsc_in_container(rsc);

	if (child != NULL) {
	return child;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine how a colocation constraint should affect a resource
	*
	* Colocation constraints have different effects at different points in the
	* scheduler sequence. Initially, they affect a resource's location; once that
	* is determined, then for promotable clones they can affect a resource
	* instance's role; after both are determined, the constraints no longer matter.
	* Given a specific colocation constraint, check what has been done so far to
	* determine what should be affected at the current point in the scheduler.
	*
	* \param[in] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint
	* \param[in] preview If true, pretend resources have already been assigned
	*
	* \return How colocation constraint should be applied at this point
	*/
	enum pcmk__coloc_affects
	pcmk__colocation_affects(const pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation, bool preview)
	{
	const pcmk_resource_t *dependent_role_rsc = NULL;
	const pcmk_resource_t *primary_role_rsc = NULL;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (!preview && pcmk_is_set(primary->flags, pcmk__rsc_unassigned)) {
	// Primary resource has not been assigned yet, so we can't do anything
	return pcmk__coloc_affects_nothing;
	}

	dependent_role_rsc = get_resource_for_role(dependent);

	primary_role_rsc = get_resource_for_role(primary);

	if ((colocation->dependent_role >= pcmk_role_unpromoted)
	&& (dependent_role_rsc->private->parent != NULL)
	&& pcmk_is_set(dependent_role_rsc->private->parent->flags,
	pcmk__rsc_promotable)
	&& !pcmk_is_set(dependent_role_rsc->flags, pcmk__rsc_unassigned)) {

	/* This is a colocation by role, and the dependent is a promotable clone
	* that has already been assigned, so the colocation should now affect
	* the role.
	*/
	return pcmk__coloc_affects_role;
	}

	if (!preview && !pcmk_is_set(dependent->flags, pcmk__rsc_unassigned)) {
	/* The dependent resource has already been through assignment, so the
	* constraint no longer has any effect. Log an error if a mandatory
	* colocation constraint has been violated.
	*/

	const pcmk_node_t *primary_node = primary->private->assigned_node;
	const pcmk_node_t *dependent_node = dependent->private->assigned_node;

	if (dependent_node == NULL) {
	crm_trace("Skipping colocation '%s': %s will not run anywhere",
	colocation->id, dependent->id);

	} else if (colocation->score >= PCMK_SCORE_INFINITY) {
	// Dependent resource must colocate with primary resource

	if (!pcmk__same_node(primary_node, dependent_node)) {
	pcmk__sched_err("%s must be colocated with %s but is not "
	"(%s vs. %s)",
	dependent->id, primary->id,
	pcmk__node_name(dependent_node),
	pcmk__node_name(primary_node));
	}

	} else if (colocation->score <= -PCMK_SCORE_INFINITY) {
	// Dependent resource must anti-colocate with primary resource

	if (pcmk__same_node(dependent_node, primary_node)) {
	pcmk__sched_err("%s and %s must be anti-colocated but are "
	"assigned to the same node (%s)",
	dependent->id, primary->id,
	pcmk__node_name(primary_node));
	}
	}
	return pcmk__coloc_affects_nothing;
	}

	if ((colocation->dependent_role != pcmk_role_unknown)
	&& (colocation->dependent_role != dependent_role_rsc->private->next_role)) {
	crm_trace("Skipping %scolocation '%s': dependent limited to %s role "

	"but %s next role is %s",
	((colocation->score < 0)? "anti-" : ""),
	colocation->id, pcmk_role_text(colocation->dependent_role),
	dependent_role_rsc->id,
	pcmk_role_text(dependent_role_rsc->private->next_role));
	return pcmk__coloc_affects_nothing;
	}

	if ((colocation->primary_role != pcmk_role_unknown)
	&& (colocation->primary_role != primary_role_rsc->private->next_role)) {
	crm_trace("Skipping %scolocation '%s': primary limited to %s role "
	"but %s next role is %s",
	((colocation->score < 0)? "anti-" : ""),
	colocation->id, pcmk_role_text(colocation->primary_role),
	primary_role_rsc->id,
	pcmk_role_text(primary_role_rsc->private->next_role));
	return pcmk__coloc_affects_nothing;
	}

	return pcmk__coloc_affects_location;
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent for assignment purposes
	*
	* Update the allowed node scores of the dependent resource in a colocation,
	* for the purposes of assigning it to a node.
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint
	*/
	void
	pcmk__apply_coloc_to_scores(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation)
	{
	const char *attr = colocation->node_attribute;
	const char *value = NULL;
	GHashTable *work = NULL;
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	if (primary->private->assigned_node != NULL) {
	value = pcmk__colocation_node_attr(primary->private->assigned_node,
	attr, primary);

	} else if (colocation->score < 0) {
	// Nothing to do (anti-colocation with something that is not running)
	return;
	}

	work = pcmk__copy_node_table(dependent->private->allowed_nodes);

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (primary->private->assigned_node == NULL) {
	- node->weight = pcmk__add_scores(-colocation->score, node->weight);
	+ node->assign->score = pcmk__add_scores(-colocation->score,
	+ node->assign->score);
	pcmk__rsc_trace(dependent,
	"Applied %s to %s score on %s (now %s after "
	"subtracting %s because primary %s inactive)",
	colocation->id, dependent->id,
	pcmk__node_name(node),
	- pcmk_readable_score(node->weight),
	+ pcmk_readable_score(node->assign->score),
	pcmk_readable_score(colocation->score), primary->id);
	continue;
	}

	if (pcmk__str_eq(pcmk__colocation_node_attr(node, attr, dependent),
	value, pcmk__str_casei)) {

	/* Add colocation score only if optional (or minus infinity). A
	* mandatory colocation is a requirement rather than a preference,
	* so we don't need to consider it for relative assignment purposes.
	* The resource will simply be forbidden from running on the node if
	* the primary isn't active there (via the condition above).
	*/
	if (colocation->score < PCMK_SCORE_INFINITY) {
	- node->weight = pcmk__add_scores(colocation->score,
	- node->weight);
	+ node->assign->score = pcmk__add_scores(colocation->score,
	+ node->assign->score);
	pcmk__rsc_trace(dependent,
	"Applied %s to %s score on %s (now %s after "
	"adding %s)",
	colocation->id, dependent->id,
	pcmk__node_name(node),
	- pcmk_readable_score(node->weight),
	+ pcmk_readable_score(node->assign->score),
	pcmk_readable_score(colocation->score));
	}
	continue;
	}

	if (colocation->score >= PCMK_SCORE_INFINITY) {
	/* Only mandatory colocations are relevant when the colocation
	* attribute doesn't match, because an attribute not matching is not
	* a negative preference -- the colocation is simply relevant only
	* where it matches.
	*/
	- node->weight = -PCMK_SCORE_INFINITY;
	+ node->assign->score = -PCMK_SCORE_INFINITY;
	pcmk__rsc_trace(dependent,
	"Banned %s from %s because colocation %s attribute %s "
	"does not match",
	dependent->id, pcmk__node_name(node),
	colocation->id, attr);
	}
	}

	if ((colocation->score <= -PCMK_SCORE_INFINITY)
	\|\| (colocation->score >= PCMK_SCORE_INFINITY)
	\|\| pcmk__any_node_available(work)) {

	g_hash_table_destroy(dependent->private->allowed_nodes);
	dependent->private->allowed_nodes = work;
	work = NULL;

	} else {
	pcmk__rsc_info(dependent,
	"%s: Rolling back scores from %s (no available nodes)",
	dependent->id, primary->id);
	}

	if (work != NULL) {
	g_hash_table_destroy(work);
	}
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent for role purposes
	*
	* Update the priority of the dependent resource in a colocation, for the
	* purposes of selecting its role
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint
	*/
	void
	pcmk__apply_coloc_to_priority(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation)
	{
	const char *dependent_value = NULL;
	const char *primary_value = NULL;
	const char *attr = colocation->node_attribute;
	int score_multiplier = 1;
	const pcmk_node_t *primary_node = NULL;
	const pcmk_node_t *dependent_node = NULL;

	const pcmk_resource_t *primary_role_rsc = NULL;

	CRM_ASSERT((dependent != NULL) && (primary != NULL) &&
	(colocation != NULL));

	primary_node = primary->private->assigned_node;
	dependent_node = dependent->private->assigned_node;

	if ((primary_node == NULL) \|\| (dependent_node == NULL)) {
	return;
	}

	dependent_value = pcmk__colocation_node_attr(dependent_node, attr,
	dependent);
	primary_value = pcmk__colocation_node_attr(primary_node, attr, primary);

	primary_role_rsc = get_resource_for_role(primary);

	if (!pcmk__str_eq(dependent_value, primary_value, pcmk__str_casei)) {
	if ((colocation->score == PCMK_SCORE_INFINITY)
	&& (colocation->dependent_role == pcmk_role_promoted)) {
	dependent->private->priority = -PCMK_SCORE_INFINITY;
	}
	return;
	}

	if ((colocation->primary_role != pcmk_role_unknown)
	&& (colocation->primary_role != primary_role_rsc->private->next_role)) {
	return;
	}

	if (colocation->dependent_role == pcmk_role_unpromoted) {
	score_multiplier = -1;
	}

	dependent->private->priority =
	pcmk__add_scores(score_multiplier * colocation->score,
	dependent->private->priority);
	pcmk__rsc_trace(dependent,
	"Applied %s to %s promotion priority (now %s after %s %s)",
	colocation->id, dependent->id,
	pcmk_readable_score(dependent->private->priority),
	((score_multiplier == 1)? "adding" : "subtracting"),
	pcmk_readable_score(colocation->score));
	}

	/*!
	* \internal
	* \brief Find score of highest-scored node that matches colocation attribute
	*
	* \param[in] rsc Resource whose allowed nodes should be searched
	* \param[in] attr Colocation attribute name (must not be NULL)
	* \param[in] value Colocation attribute value to require
	*/
	static int
	best_node_score_matching_attr(const pcmk_resource_t rsc, const char attr,
	const char *value)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	int best_score = -PCMK_SCORE_INFINITY;
	const char *best_node = NULL;

	// Find best allowed node with matching attribute
	g_hash_table_iter_init(&iter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {

	- if ((node->weight > best_score)
	+ if ((node->assign->score > best_score)
	&& pcmk__node_available(node, false, false)
	&& pcmk__str_eq(value, pcmk__colocation_node_attr(node, attr, rsc),
	pcmk__str_casei)) {

	- best_score = node->weight;
	+ best_score = node->assign->score;
	best_node = node->private->name;
	}
	}

	if (!pcmk__str_eq(attr, CRM_ATTR_UNAME, pcmk__str_none)) {
	if (best_node == NULL) {
	crm_info("No allowed node for %s matches node attribute %s=%s",
	rsc->id, attr, value);
	} else {
	crm_info("Allowed node %s for %s had best score (%d) "
	"of those matching node attribute %s=%s",
	best_node, rsc->id, best_score, attr, value);
	}
	}
	return best_score;
	}

	/*!
	* \internal
	* \brief Check whether a resource is allowed only on a single node
	*
	* \param[in] rsc Resource to check
	*
	* \return \c true if \p rsc is allowed only on one node, otherwise \c false
	*/
	static bool
	allowed_on_one(const pcmk_resource_t *rsc)
	{
	GHashTableIter iter;
	pcmk_node_t *allowed_node = NULL;
	int allowed_nodes = 0;

	g_hash_table_iter_init(&iter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &allowed_node)) {
	- if ((allowed_node->weight >= 0) && (++allowed_nodes > 1)) {
	+ if ((allowed_node->assign->score >= 0) && (++allowed_nodes > 1)) {
	pcmk__rsc_trace(rsc, "%s is allowed on multiple nodes", rsc->id);
	return false;
	}
	}
	pcmk__rsc_trace(rsc, "%s is allowed %s", rsc->id,
	((allowed_nodes == 1)? "on a single node" : "nowhere"));
	return (allowed_nodes == 1);
	}

	/*!
	* \internal
	* \brief Add resource's colocation matches to current node assignment scores
	*
	* For each node in a given table, if any of a given resource's allowed nodes
	* have a matching value for the colocation attribute, add the highest of those
	* nodes' scores to the node's score.
	*
	* \param[in,out] nodes Table of nodes with assignment scores so far
	* \param[in] source_rsc Resource whose node scores to add
	* \param[in] target_rsc Resource on whose behalf to update \p nodes
	* \param[in] colocation Original colocation constraint (used to get
	* configured primary resource's stickiness, and
	* to get colocation node attribute; pass NULL to
	* ignore stickiness and use default attribute)
	* \param[in] factor Factor by which to multiply scores being added
	* \param[in] only_positive Whether to add only positive scores
	*/
	static void
	add_node_scores_matching_attr(GHashTable *nodes,
	const pcmk_resource_t *source_rsc,
	const pcmk_resource_t *target_rsc,
	const pcmk__colocation_t *colocation,
	float factor, bool only_positive)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	const char *attr = colocation->node_attribute;

	// Iterate through each node
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	float delta_f = 0;
	int delta = 0;
	int score = 0;
	int new_score = 0;
	const char *value = pcmk__colocation_node_attr(node, attr, target_rsc);

	score = best_node_score_matching_attr(source_rsc, attr, value);

	if ((factor < 0) && (score < 0)) {
	/* If the dependent is anti-colocated, we generally don't want the
	* primary to prefer nodes that the dependent avoids. That could
	* lead to unnecessary shuffling of the primary when the dependent
	* hits its migration threshold somewhere, for example.
	*
	* However, there are cases when it is desirable. If the dependent
	* can't run anywhere but where the primary is, it would be
	* worthwhile to move the primary for the sake of keeping the
	* dependent active.
	*
	* We can't know that exactly at this point since we don't know
	* where the primary will be assigned, but we can limit considering
	* the preference to when the dependent is allowed only on one node.
	* This is less than ideal for multiple reasons:
	*
	* - the dependent could be allowed on more than one node but have
	* anti-colocation primaries on each;
	* - the dependent could be a clone or bundle with multiple
	* instances, and the dependent as a whole is allowed on multiple
	* nodes but some instance still can't run
	* - the dependent has considered node-specific criteria such as
	* location constraints and stickiness by this point, but might
	* have other factors that end up disallowing a node
	*
	* but the alternative is making the primary move when it doesn't
	* need to.
	*
	* We also consider the primary's stickiness and influence, so the
	* user has some say in the matter. (This is the configured primary,
	* not a particular instance of the primary, but that doesn't matter
	* unless stickiness uses a rule to vary by node, and that seems
	* acceptable to ignore.)
	*/
	if ((colocation->primary->private->stickiness >= -score)
	\|\| !pcmk__colocation_has_influence(colocation, NULL)
	\|\| !allowed_on_one(colocation->dependent)) {
	crm_trace("%s: Filtering %d + %f * %d "
	"(double negative disallowed)",
	- pcmk__node_name(node), node->weight, factor, score);
	+ pcmk__node_name(node), node->assign->score, factor,
	+ score);
	continue;
	}
	}

	- if (node->weight == INFINITY_HACK) {
	+ if (node->assign->score == INFINITY_HACK) {
	crm_trace("%s: Filtering %d + %f * %d (node was marked unusable)",
	- pcmk__node_name(node), node->weight, factor, score);
	+ pcmk__node_name(node), node->assign->score, factor,
	+ score);
	continue;
	}

	delta_f = factor * score;

	// Round the number; see http://c-faq.com/fp/round.html
	delta = (int) ((delta_f < 0)? (delta_f - 0.5) : (delta_f + 0.5));

	/* Small factors can obliterate the small scores that are often actually
	* used in configurations. If the score and factor are nonzero, ensure
	* that the result is nonzero as well.
	*/
	if ((delta == 0) && (score != 0)) {
	if (factor > 0.0) {
	delta = 1;
	} else if (factor < 0.0) {
	delta = -1;
	}
	}

	- new_score = pcmk__add_scores(delta, node->weight);
	+ new_score = pcmk__add_scores(delta, node->assign->score);

	- if (only_positive && (new_score < 0) && (node->weight > 0)) {
	+ if (only_positive && (new_score < 0) && (node->assign->score > 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d "
	"(negative disallowed, marking node unusable)",
	- pcmk__node_name(node), node->weight, factor, score,
	+ pcmk__node_name(node), node->assign->score, factor, score,
	new_score);
	- node->weight = INFINITY_HACK;
	+ node->assign->score = INFINITY_HACK;
	continue;
	}

	- if (only_positive && (new_score < 0) && (node->weight == 0)) {
	+ if (only_positive && (new_score < 0) && (node->assign->score == 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d (negative disallowed)",
	- pcmk__node_name(node), node->weight, factor, score,
	+ pcmk__node_name(node), node->assign->score, factor, score,
	new_score);
	continue;
	}

	crm_trace("%s: %d + %f * %d = %d", pcmk__node_name(node),
	- node->weight, factor, score, new_score);
	- node->weight = new_score;
	+ node->assign->score, factor, score, new_score);
	+ node->assign->score = new_score;
	}
	}

	/*!
	* \internal
	* \brief Update nodes with scores of colocated resources' nodes
	*
	* Given a table of nodes and a resource, update the nodes' scores with the
	* scores of the best nodes matching the attribute used for each of the
	* resource's relevant colocations.
	*
	* \param[in,out] source_rsc Resource whose node scores to add
	* \param[in] target_rsc Resource on whose behalf to update \p *nodes
	* \param[in] log_id Resource ID for logs (if \c NULL, use
	* \p source_rsc ID)
	* \param[in,out] nodes Nodes to update (set initial contents to \c NULL
	* to copy allowed nodes from \p source_rsc)
	* \param[in] colocation Original colocation constraint (used to get
	* configured primary resource's stickiness, and
	* to get colocation node attribute; if \c NULL,
	* <tt>source_rsc</tt>'s own matching node scores
	* will not be added, and \p *nodes must be \c NULL
	* as well)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pcmk__coloc_select values
	*
	* \note \c NULL \p target_rsc, \c NULL \p *nodes, \c NULL \p colocation, and
	* the \c pcmk__coloc_select_this_with flag are used together (and only by
	* \c cmp_resources()).
	* \note The caller remains responsible for freeing \p *nodes.
	* \note This is the shared implementation of
	* \c pcmk__assignment_methods_t:add_colocated_node_scores().
	*/
	void
	pcmk__add_colocated_node_scores(pcmk_resource_t *source_rsc,
	const pcmk_resource_t *target_rsc,
	const char *log_id,
	GHashTable **nodes,
	const pcmk__colocation_t *colocation,
	float factor, uint32_t flags)
	{
	GHashTable *work = NULL;

	CRM_ASSERT((source_rsc != NULL) && (nodes != NULL)
	&& ((colocation != NULL)
	\|\| ((target_rsc == NULL) && (*nodes == NULL))));

	if (log_id == NULL) {
	log_id = source_rsc->id;
	}

	// Avoid infinite recursion
	if (pcmk_is_set(source_rsc->flags, pcmk__rsc_updating_nodes)) {
	pcmk__rsc_info(source_rsc, "%s: Breaking dependency loop at %s",
	log_id, source_rsc->id);
	return;
	}
	pcmk__set_rsc_flags(source_rsc, pcmk__rsc_updating_nodes);

	if (*nodes == NULL) {
	work = pcmk__copy_node_table(source_rsc->private->allowed_nodes);
	target_rsc = source_rsc;
	} else {
	const bool pos = pcmk_is_set(flags, pcmk__coloc_select_nonnegative);

	pcmk__rsc_trace(source_rsc, "%s: Merging %s scores from %s (at %.6f)",
	log_id, (pos? "positive" : "all"), source_rsc->id, factor);
	work = pcmk__copy_node_table(*nodes);
	add_node_scores_matching_attr(work, source_rsc, target_rsc, colocation,
	factor, pos);
	}

	if (work == NULL) {
	pcmk__clear_rsc_flags(source_rsc, pcmk__rsc_updating_nodes);
	return;
	}

	if (pcmk__any_node_available(work)) {
	GList *colocations = NULL;

	if (pcmk_is_set(flags, pcmk__coloc_select_this_with)) {
	colocations = pcmk__this_with_colocations(source_rsc);
	pcmk__rsc_trace(source_rsc,
	"Checking additional %d optional '%s with' "
	"constraints",
	g_list_length(colocations), source_rsc->id);
	} else {
	colocations = pcmk__with_this_colocations(source_rsc);
	pcmk__rsc_trace(source_rsc,
	"Checking additional %d optional 'with %s' "
	"constraints",
	g_list_length(colocations), source_rsc->id);
	}
	flags \|= pcmk__coloc_select_active;

	for (GList *iter = colocations; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *constraint = iter->data;

	pcmk_resource_t *other = NULL;
	float other_factor = factor * constraint->score
	/ (float) PCMK_SCORE_INFINITY;

	if (pcmk_is_set(flags, pcmk__coloc_select_this_with)) {
	other = constraint->primary;
	} else if (!pcmk__colocation_has_influence(constraint, NULL)) {
	continue;
	} else {
	other = constraint->dependent;
	}

	pcmk__rsc_trace(source_rsc,
	"Optionally merging score of '%s' constraint "
	"(%s with %s)",
	constraint->id, constraint->dependent->id,
	constraint->primary->id);
	other->private->cmds->add_colocated_node_scores(other, target_rsc,
	log_id, &work,
	constraint,
	other_factor,
	flags);
	pe__show_node_scores(true, NULL, log_id, work,
	source_rsc->private->scheduler);
	}
	g_list_free(colocations);

	} else if (pcmk_is_set(flags, pcmk__coloc_select_active)) {
	pcmk__rsc_info(source_rsc, "%s: Rolling back optional scores from %s",
	log_id, source_rsc->id);
	g_hash_table_destroy(work);
	pcmk__clear_rsc_flags(source_rsc, pcmk__rsc_updating_nodes);
	return;
	}


	if (pcmk_is_set(flags, pcmk__coloc_select_nonnegative)) {
	pcmk_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	- if (node->weight == INFINITY_HACK) {
	- node->weight = 1;
	+ if (node->assign->score == INFINITY_HACK) {
	+ node->assign->score = 1;
	}
	}
	}

	if (*nodes != NULL) {
	g_hash_table_destroy(*nodes);
	}
	*nodes = work;

	pcmk__clear_rsc_flags(source_rsc, pcmk__rsc_updating_nodes);
	}

	/*!
	* \internal
	* \brief Apply a "with this" colocation to a resource's allowed node scores
	*
	* \param[in,out] data Colocation to apply
	* \param[in,out] user_data Resource being assigned
	*/
	void
	pcmk__add_dependent_scores(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t *colocation = data;
	pcmk_resource_t *primary = user_data;

	pcmk_resource_t *dependent = colocation->dependent;
	const float factor = colocation->score / (float) PCMK_SCORE_INFINITY;
	uint32_t flags = pcmk__coloc_select_active;

	if (!pcmk__colocation_has_influence(colocation, NULL)) {
	return;
	}
	if (pcmk__is_clone(primary)) {
	flags \|= pcmk__coloc_select_nonnegative;
	}
	pcmk__rsc_trace(primary,
	"%s: Incorporating attenuated %s assignment scores due "
	"to colocation %s",
	primary->id, dependent->id, colocation->id);
	dependent->private->cmds->add_colocated_node_scores(dependent, primary,
	dependent->id,
	&(primary->private->allowed_nodes),
	colocation, factor,
	flags);
	}

	/*!
	* \internal
	* \brief Exclude nodes from a dependent's node table if not in a given list
	*
	* Given a dependent resource in a colocation and a list of nodes where the
	* primary resource will run, set a node's score to \c -INFINITY in the
	* dependent's node table if not found in the primary nodes list.
	*
	* \param[in,out] dependent Dependent resource
	* \param[in] primary Primary resource (for logging only)
	* \param[in] colocation Colocation constraint (for logging only)
	* \param[in] primary_nodes List of nodes where the primary will have
	* unblocked instances in a suitable role
	* \param[in] merge_scores If \c true and a node is found in both \p table
	* and \p list, add the node's score in \p list to
	* the node's score in \p table
	*/
	void
	pcmk__colocation_intersect_nodes(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	const GList *primary_nodes, bool merge_scores)
	{
	GHashTableIter iter;
	pcmk_node_t *dependent_node = NULL;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	g_hash_table_iter_init(&iter, dependent->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &dependent_node)) {
	const pcmk_node_t *primary_node = NULL;

	primary_node = pe_find_node_id(primary_nodes,
	dependent_node->private->id);
	if (primary_node == NULL) {
	- dependent_node->weight = -PCMK_SCORE_INFINITY;
	+ dependent_node->assign->score = -PCMK_SCORE_INFINITY;
	pcmk__rsc_trace(dependent,
	"Banning %s from %s (no primary instance) for %s",
	dependent->id, pcmk__node_name(dependent_node),
	colocation->id);

	} else if (merge_scores) {
	- dependent_node->weight = pcmk__add_scores(dependent_node->weight,
	- primary_node->weight);
	+ dependent_node->assign->score =
	+ pcmk__add_scores(dependent_node->assign->score,
	+ primary_node->assign->score);
	pcmk__rsc_trace(dependent,
	- "Added %s's score %s to %s's score for %s (now %s) "
	+ "Added %s's score %s to %s's score for %s (now %d) "
	"for colocation %s",
	- primary->id, pcmk_readable_score(primary_node->weight),
	+ primary->id,
	+ pcmk_readable_score(primary_node->assign->score),
	dependent->id, pcmk__node_name(dependent_node),
	- pcmk_readable_score(dependent_node->weight),
	- colocation->id);
	+ dependent_node->assign->score, colocation->id);
	}
	}
	}

	/*!
	* \internal
	* \brief Get all colocations affecting a resource as the primary
	*
	* \param[in] rsc Resource to get colocations for
	*
	* \return Newly allocated list of colocations affecting \p rsc as primary
	*
	* \note This is a convenience wrapper for the with_this_colocations() method.
	*/
	GList *
	pcmk__with_this_colocations(const pcmk_resource_t *rsc)
	{
	GList *list = NULL;

	rsc->private->cmds->with_this_colocations(rsc, rsc, &list);
	return list;
	}

	/*!
	* \internal
	* \brief Get all colocations affecting a resource as the dependent
	*
	* \param[in] rsc Resource to get colocations for
	*
	* \return Newly allocated list of colocations affecting \p rsc as dependent
	*
	* \note This is a convenience wrapper for the this_with_colocations() method.
	*/
	GList *
	pcmk__this_with_colocations(const pcmk_resource_t *rsc)
	{
	GList *list = NULL;

	rsc->private->cmds->this_with_colocations(rsc, rsc, &list);
	return list;
	}
	diff --git a/lib/pacemaker/pcmk_sched_instances.c b/lib/pacemaker/pcmk_sched_instances.c
	index 3d0a9a3c65..d085de02f7 100644
	--- a/lib/pacemaker/pcmk_sched_instances.c
	+++ b/lib/pacemaker/pcmk_sched_instances.c
	@@ -1,1707 +1,1709 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	/* This file is intended for code usable with both clone instances and bundle
	* replica containers.
	*/

	#include <crm_internal.h>
	#include <crm/common/xml.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Check whether a node is allowed to run an instance
	*
	* \param[in] instance Clone instance or bundle container to check
	* \param[in] node Node to check
	* \param[in] max_per_node Maximum number of instances allowed to run on a node
	*
	* \return true if \p node is allowed to run \p instance, otherwise false
	*/
	static bool
	can_run_instance(const pcmk_resource_t instance, const pcmk_node_t node,
	int max_per_node)
	{
	pcmk_node_t *allowed_node = NULL;

	if (pcmk_is_set(instance->flags, pcmk__rsc_removed)) {
	pcmk__rsc_trace(instance, "%s cannot run on %s: orphaned",
	instance->id, pcmk__node_name(node));
	return false;
	}

	if (!pcmk__node_available(node, false, false)) {
	pcmk__rsc_trace(instance,
	"%s cannot run on %s: node cannot run resources",
	instance->id, pcmk__node_name(node));
	return false;
	}

	allowed_node = pcmk__top_allowed_node(instance, node);
	if (allowed_node == NULL) {
	crm_warn("%s cannot run on %s: node not allowed",
	instance->id, pcmk__node_name(node));
	return false;
	}

	- if (allowed_node->weight < 0) {
	+ if (allowed_node->assign->score < 0) {
	pcmk__rsc_trace(instance,
	"%s cannot run on %s: parent score is %s there",
	instance->id, pcmk__node_name(node),
	- pcmk_readable_score(allowed_node->weight));
	+ pcmk_readable_score(allowed_node->assign->score));
	return false;
	}

	- if (allowed_node->count >= max_per_node) {
	+ if (allowed_node->assign->count >= max_per_node) {
	pcmk__rsc_trace(instance,
	"%s cannot run on %s: node already has %d instance%s",
	instance->id, pcmk__node_name(node), max_per_node,
	pcmk__plural_s(max_per_node));
	return false;
	}

	pcmk__rsc_trace(instance, "%s can run on %s (%d already running)",
	- instance->id, pcmk__node_name(node), allowed_node->count);
	+ instance->id, pcmk__node_name(node),
	+ allowed_node->assign->count);
	return true;
	}

	/*!
	* \internal
	* \brief Ban a clone instance or bundle replica from unavailable allowed nodes
	*
	* \param[in,out] instance Clone instance or bundle replica to ban
	* \param[in] max_per_node Maximum instances allowed to run on a node
	*/
	static void
	ban_unavailable_allowed_nodes(pcmk_resource_t *instance, int max_per_node)
	{
	if (instance->private->allowed_nodes != NULL) {
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, instance->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (!can_run_instance(instance, node, max_per_node)) {
	pcmk__rsc_trace(instance, "Banning %s from unavailable node %s",
	instance->id, pcmk__node_name(node));
	- node->weight = -PCMK_SCORE_INFINITY;
	+ node->assign->score = -PCMK_SCORE_INFINITY;

	for (GList *child_iter = instance->private->children;
	child_iter != NULL; child_iter = child_iter->next) {

	pcmk_resource_t *child = child_iter->data;
	pcmk_node_t *child_node = NULL;

	child_node =
	g_hash_table_lookup(child->private->allowed_nodes,
	node->private->id);
	if (child_node != NULL) {
	pcmk__rsc_trace(instance,
	"Banning %s child %s "
	"from unavailable node %s",
	instance->id, child->id,
	pcmk__node_name(node));
	- child_node->weight = -PCMK_SCORE_INFINITY;
	+ child_node->assign->score = -PCMK_SCORE_INFINITY;
	}
	}
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Create a hash table with a single node in it
	*
	* \param[in] node Node to copy into new table
	*
	* \return Newly created hash table containing a copy of \p node
	* \note The caller is responsible for freeing the result with
	* g_hash_table_destroy().
	*/
	static GHashTable *
	new_node_table(pcmk_node_t *node)
	{
	GHashTable *table = pcmk__strkey_table(NULL, free);

	node = pe__copy_node(node);
	g_hash_table_insert(table, (gpointer) node->private->id, node);
	return table;
	}

	/*!
	* \internal
	* \brief Apply a resource's parent's colocation scores to a node table
	*
	* \param[in] rsc Resource whose colocations should be applied
	* \param[in,out] nodes Node table to apply colocations to
	*/
	static void
	apply_parent_colocations(const pcmk_resource_t rsc, GHashTable *nodes)
	{
	GList *colocations = pcmk__this_with_colocations(rsc);

	for (const GList *iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *colocation = iter->data;
	pcmk_resource_t *other = colocation->primary;
	float factor = colocation->score / (float) PCMK_SCORE_INFINITY;

	other->private->cmds->add_colocated_node_scores(other, rsc, rsc->id,
	nodes, colocation,
	factor,
	pcmk__coloc_select_default);
	}
	g_list_free(colocations);
	colocations = pcmk__with_this_colocations(rsc);

	for (const GList *iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *colocation = iter->data;
	pcmk_resource_t *other = colocation->dependent;
	float factor = colocation->score / (float) PCMK_SCORE_INFINITY;

	if (!pcmk__colocation_has_influence(colocation, rsc)) {
	continue;
	}
	other->private->cmds->add_colocated_node_scores(other, rsc, rsc->id,
	nodes, colocation,
	factor,
	pcmk__coloc_select_nonnegative);
	}
	g_list_free(colocations);
	}

	/*!
	* \internal
	* \brief Compare clone or bundle instances based on colocation scores
	*
	* Determine the relative order in which two clone or bundle instances should be
	* assigned to nodes, considering the scores of colocation constraints directly
	* or indirectly involving them.
	*
	* \param[in] instance1 First instance to compare
	* \param[in] instance2 Second instance to compare
	*
	* \return A negative number if \p instance1 should be assigned first,
	* a positive number if \p instance2 should be assigned first,
	* or 0 if assignment order doesn't matter
	*/
	static int
	cmp_instance_by_colocation(const pcmk_resource_t *instance1,
	const pcmk_resource_t *instance2)
	{
	int rc = 0;
	pcmk_node_t *node1 = NULL;
	pcmk_node_t *node2 = NULL;
	pcmk_node_t *current_node1 = pcmk__current_node(instance1);
	pcmk_node_t *current_node2 = pcmk__current_node(instance2);
	GHashTable *colocated_scores1 = NULL;
	GHashTable *colocated_scores2 = NULL;

	CRM_ASSERT((instance1 != NULL) && (instance1->private->parent != NULL)
	&& (instance2 != NULL) && (instance2->private->parent != NULL)
	&& (current_node1 != NULL) && (current_node2 != NULL));

	// Create node tables initialized with each node
	colocated_scores1 = new_node_table(current_node1);
	colocated_scores2 = new_node_table(current_node2);

	// Apply parental colocations
	apply_parent_colocations(instance1, &colocated_scores1);
	apply_parent_colocations(instance2, &colocated_scores2);

	// Find original nodes again, with scores updated for colocations
	node1 = g_hash_table_lookup(colocated_scores1, current_node1->private->id);
	node2 = g_hash_table_lookup(colocated_scores2, current_node2->private->id);

	// Compare nodes by updated scores
	- if (node1->weight < node2->weight) {
	+ if (node1->assign->score < node2->assign->score) {
	crm_trace("Assign %s (%d on %s) after %s (%d on %s)",
	- instance1->id, node1->weight, pcmk__node_name(node1),
	- instance2->id, node2->weight, pcmk__node_name(node2));
	+ instance1->id, node1->assign->score, pcmk__node_name(node1),
	+ instance2->id, node2->assign->score, pcmk__node_name(node2));
	rc = 1;

	- } else if (node1->weight > node2->weight) {
	+ } else if (node1->assign->score > node2->assign->score) {
	crm_trace("Assign %s (%d on %s) before %s (%d on %s)",
	- instance1->id, node1->weight, pcmk__node_name(node1),
	- instance2->id, node2->weight, pcmk__node_name(node2));
	+ instance1->id, node1->assign->score, pcmk__node_name(node1),
	+ instance2->id, node2->assign->score, pcmk__node_name(node2));
	rc = -1;
	}

	g_hash_table_destroy(colocated_scores1);
	g_hash_table_destroy(colocated_scores2);
	return rc;
	}

	/*!
	* \internal
	* \brief Check whether a resource or any of its children are failed
	*
	* \param[in] rsc Resource to check
	*
	* \return true if \p rsc or any of its children are failed, otherwise false
	*/
	static bool
	did_fail(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
	return true;
	}

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	if (did_fail((const pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node is allowed to run a resource
	*
	* \param[in] rsc Resource to check
	* \param[in,out] node Node to check (will be set NULL if not allowed)
	*
	* \return true if *node is either NULL or allowed for \p rsc, otherwise false
	*/
	static bool
	node_is_allowed(const pcmk_resource_t rsc, pcmk_node_t *node)
	{
	if (*node != NULL) {
	pcmk_node_t *allowed = g_hash_table_lookup(rsc->private->allowed_nodes,
	(*node)->private->id);

	- if ((allowed == NULL) \|\| (allowed->weight < 0)) {
	+ if ((allowed == NULL) \|\| (allowed->assign->score < 0)) {
	pcmk__rsc_trace(rsc, "%s: current location (%s) is unavailable",
	rsc->id, pcmk__node_name(*node));
	*node = NULL;
	return false;
	}
	}
	return true;
	}

	/*!
	* \internal
	* \brief Compare two clone or bundle instances' instance numbers
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a's instance number is lower,
	* a positive number if \p b's instance number is lower,
	* or 0 if their instance numbers are the same
	*/
	gint
	pcmk__cmp_instance_number(gconstpointer a, gconstpointer b)
	{
	const pcmk_resource_t instance1 = (const pcmk_resource_t ) a;
	const pcmk_resource_t instance2 = (const pcmk_resource_t ) b;
	char *div1 = NULL;
	char *div2 = NULL;

	CRM_ASSERT((instance1 != NULL) && (instance2 != NULL));

	// Clone numbers are after a colon, bundle numbers after a dash
	div1 = strrchr(instance1->id, ':');
	if (div1 == NULL) {
	div1 = strrchr(instance1->id, '-');
	}
	div2 = strrchr(instance2->id, ':');
	if (div2 == NULL) {
	div2 = strrchr(instance2->id, '-');
	}
	CRM_ASSERT((div1 != NULL) && (div2 != NULL));

	return (gint) (strtol(div1 + 1, NULL, 10) - strtol(div2 + 1, NULL, 10));
	}

	/*!
	* \internal
	* \brief Compare clone or bundle instances according to assignment order
	*
	* Compare two clone or bundle instances according to the order they should be
	* assigned to nodes, preferring (in order):
	*
	* - Active instance that is less multiply active
	* - Instance that is not active on a disallowed node
	* - Instance with higher configured priority
	* - Active instance whose current node can run resources
	* - Active instance whose parent is allowed on current node
	* - Active instance whose current node has fewer other instances
	* - Active instance
	* - Instance that isn't failed
	* - Instance whose colocations result in higher score on current node
	* - Instance with lower ID in lexicographic order
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a should be assigned first,
	* a positive number if \p b should be assigned first,
	* or 0 if assignment order doesn't matter
	*/
	gint
	pcmk__cmp_instance(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	pcmk_node_t *node1 = NULL;
	pcmk_node_t *node2 = NULL;
	unsigned int nnodes1 = 0;
	unsigned int nnodes2 = 0;

	bool can1 = true;
	bool can2 = true;

	const pcmk_resource_t instance1 = (const pcmk_resource_t ) a;
	const pcmk_resource_t instance2 = (const pcmk_resource_t ) b;

	CRM_ASSERT((instance1 != NULL) && (instance2 != NULL));

	node1 = instance1->private->fns->active_node(instance1, &nnodes1, NULL);
	node2 = instance2->private->fns->active_node(instance2, &nnodes2, NULL);

	/* If both instances are running and at least one is multiply
	* active, prefer instance that's running on fewer nodes.
	*/
	if ((nnodes1 > 0) && (nnodes2 > 0)) {
	if (nnodes1 < nnodes2) {
	crm_trace("Assign %s (active on %d) before %s (active on %d): "
	"less multiply active",
	instance1->id, nnodes1, instance2->id, nnodes2);
	return -1;

	} else if (nnodes1 > nnodes2) {
	crm_trace("Assign %s (active on %d) after %s (active on %d): "
	"more multiply active",
	instance1->id, nnodes1, instance2->id, nnodes2);
	return 1;
	}
	}

	/* An instance that is either inactive or active on an allowed node is
	* preferred over an instance that is active on a no-longer-allowed node.
	*/
	can1 = node_is_allowed(instance1, &node1);
	can2 = node_is_allowed(instance2, &node2);
	if (can1 && !can2) {
	crm_trace("Assign %s before %s: not active on a disallowed node",
	instance1->id, instance2->id);
	return -1;

	} else if (!can1 && can2) {
	crm_trace("Assign %s after %s: active on a disallowed node",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance with higher configured priority
	if (instance1->private->priority > instance2->private->priority) {
	crm_trace("Assign %s before %s: priority (%d > %d)",
	instance1->id, instance2->id,
	instance1->private->priority, instance2->private->priority);
	return -1;

	} else if (instance1->private->priority < instance2->private->priority) {
	crm_trace("Assign %s after %s: priority (%d < %d)",
	instance1->id, instance2->id,
	instance1->private->priority, instance2->private->priority);
	return 1;
	}

	// Prefer active instance
	if ((node1 == NULL) && (node2 == NULL)) {
	crm_trace("No assignment preference for %s vs. %s: inactive",
	instance1->id, instance2->id);
	return 0;

	} else if (node1 == NULL) {
	crm_trace("Assign %s after %s: active", instance1->id, instance2->id);
	return 1;

	} else if (node2 == NULL) {
	crm_trace("Assign %s before %s: active", instance1->id, instance2->id);
	return -1;
	}

	// Prefer instance whose current node can run resources
	can1 = pcmk__node_available(node1, false, false);
	can2 = pcmk__node_available(node2, false, false);
	if (can1 && !can2) {
	crm_trace("Assign %s before %s: current node can run resources",
	instance1->id, instance2->id);
	return -1;

	} else if (!can1 && can2) {
	crm_trace("Assign %s after %s: current node can't run resources",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance whose parent is allowed to run on instance's current node
	node1 = pcmk__top_allowed_node(instance1, node1);
	node2 = pcmk__top_allowed_node(instance2, node2);
	if ((node1 == NULL) && (node2 == NULL)) {
	crm_trace("No assignment preference for %s vs. %s: "
	"parent not allowed on either instance's current node",
	instance1->id, instance2->id);
	return 0;

	} else if (node1 == NULL) {
	crm_trace("Assign %s after %s: parent not allowed on current node",
	instance1->id, instance2->id);
	return 1;

	} else if (node2 == NULL) {
	crm_trace("Assign %s before %s: parent allowed on current node",
	instance1->id, instance2->id);
	return -1;
	}

	// Prefer instance whose current node is running fewer other instances
	- if (node1->count < node2->count) {
	+ if (node1->assign->count < node2->assign->count) {
	crm_trace("Assign %s before %s: fewer active instances on current node",
	instance1->id, instance2->id);
	return -1;

	- } else if (node1->count > node2->count) {
	+ } else if (node1->assign->count > node2->assign->count) {
	crm_trace("Assign %s after %s: more active instances on current node",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance that isn't failed
	can1 = did_fail(instance1);
	can2 = did_fail(instance2);
	if (!can1 && can2) {
	crm_trace("Assign %s before %s: not failed",
	instance1->id, instance2->id);
	return -1;
	} else if (can1 && !can2) {
	crm_trace("Assign %s after %s: failed",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance with higher cumulative colocation score on current node
	rc = cmp_instance_by_colocation(instance1, instance2);
	if (rc != 0) {
	return rc;
	}

	// Prefer instance with lower instance number
	rc = pcmk__cmp_instance_number(instance1, instance2);
	if (rc < 0) {
	crm_trace("Assign %s before %s: instance number",
	instance1->id, instance2->id);
	} else if (rc > 0) {
	crm_trace("Assign %s after %s: instance number",
	instance1->id, instance2->id);
	} else {
	crm_trace("No assignment preference for %s vs. %s",
	instance1->id, instance2->id);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Increment the parent's instance count after assigning an instance
	*
	* An instance's parent tracks how many instances have been assigned to each
	* node via its pcmk_node_t:count member. After assigning an instance to a node,
	* find the corresponding node in the parent's allowed table and increment it.
	*
	* \param[in,out] instance Instance whose parent to update
	* \param[in] assigned_to Node to which the instance was assigned
	*/
	static void
	increment_parent_count(pcmk_resource_t *instance,
	const pcmk_node_t *assigned_to)
	{
	pcmk_node_t *allowed = NULL;

	if (assigned_to == NULL) {
	return;
	}
	allowed = pcmk__top_allowed_node(instance, assigned_to);

	if (allowed == NULL) {
	/* The instance is allowed on the node, but its parent isn't. This
	* shouldn't be possible if the resource is managed, and we won't be
	* able to limit the number of instances assigned to the node.
	*/
	CRM_LOG_ASSERT(!pcmk_is_set(instance->flags, pcmk__rsc_managed));

	} else {
	- allowed->count++;
	+ allowed->assign->count++;
	}
	}

	/*!
	* \internal
	* \brief Assign an instance to a node
	*
	* \param[in,out] instance Clone instance or bundle replica container
	* \param[in] prefer If not NULL, attempt early assignment to this
	* node, if still the best choice; otherwise,
	* perform final assignment
	* \param[in] max_per_node Assign at most this many instances to one node
	*
	* \return Node to which \p instance is assigned
	*/
	static const pcmk_node_t *
	assign_instance(pcmk_resource_t instance, const pcmk_node_t prefer,
	int max_per_node)
	{
	pcmk_node_t *chosen = NULL;

	pcmk__rsc_trace(instance, "Assigning %s (preferring %s)", instance->id,
	((prefer == NULL)? "no node" : prefer->private->name));

	if (pcmk_is_set(instance->flags, pcmk__rsc_assigning)) {
	pcmk__rsc_debug(instance,
	"Assignment loop detected involving %s colocations",
	instance->id);
	return NULL;
	}
	ban_unavailable_allowed_nodes(instance, max_per_node);

	// Failed early assignments are reversible (stop_if_fail=false)
	chosen = instance->private->cmds->assign(instance, prefer,
	(prefer == NULL));
	increment_parent_count(instance, chosen);
	return chosen;
	}

	/*!
	* \internal
	* \brief Try to assign an instance to its current node early
	*
	* \param[in] rsc Clone or bundle being assigned (for logs only)
	* \param[in] instance Clone instance or bundle replica container
	* \param[in] current Instance's current node
	* \param[in] max_per_node Maximum number of instances per node
	* \param[in] available Number of instances still available for assignment
	*
	* \return \c true if \p instance was successfully assigned to its current node,
	* or \c false otherwise
	*/
	static bool
	assign_instance_early(const pcmk_resource_t rsc, pcmk_resource_t instance,
	const pcmk_node_t *current, int max_per_node,
	int available)
	{
	const pcmk_node_t *chosen = NULL;
	int reserved = 0;

	pcmk_resource_t *parent = instance->private->parent;
	GHashTable *allowed_orig = NULL;
	GHashTable *allowed_orig_parent = parent->private->allowed_nodes;
	const pcmk_node_t *allowed_node = NULL;

	pcmk__rsc_trace(instance, "Trying to assign %s to its current node %s",
	instance->id, pcmk__node_name(current));

	allowed_node = g_hash_table_lookup(instance->private->allowed_nodes,
	current->private->id);
	if (!pcmk__node_available(allowed_node, true, false)) {
	pcmk__rsc_info(instance,
	"Not assigning %s to current node %s: unavailable",
	instance->id, pcmk__node_name(current));
	return false;
	}

	/* On each iteration, if instance gets assigned to a node other than its
	* current one, we reserve one instance for the chosen node, unassign
	* instance, restore instance's original node tables, and try again. This
	* way, instances are proportionally assigned to nodes based on preferences,
	* but shuffling of specific instances is minimized. If a node will be
	* assigned instances at all, it preferentially receives instances that are
	* currently active there.
	*
	* parent->private->allowed_nodes tracks the number of instances assigned to
	* each node. If a node already has max_per_node instances assigned,
	* ban_unavailable_allowed_nodes() marks it as unavailable.
	*
	* In the end, we restore the original parent->private->allowed_nodes to
	* undo the changes to counts during tentative assignments. If we
	* successfully assigned an instance to its current node, we increment that
	* node's counter.
	*/

	// Back up the allowed node tables of instance and its children recursively
	pcmk__copy_node_tables(instance, &allowed_orig);

	// Update instances-per-node counts in a scratch table
	parent->private->allowed_nodes = pcmk__copy_node_table(allowed_orig_parent);

	while (reserved < available) {
	chosen = assign_instance(instance, current, max_per_node);

	if (pcmk__same_node(chosen, current)) {
	// Successfully assigned to current node
	break;
	}

	// Assignment updates scores, so restore to original state
	pcmk__rsc_debug(instance, "Rolling back node scores for %s",
	instance->id);
	pcmk__restore_node_tables(instance, allowed_orig);

	if (chosen == NULL) {
	// Assignment failed, so give up
	pcmk__rsc_info(instance,
	"Not assigning %s to current node %s: unavailable",
	instance->id, pcmk__node_name(current));
	pcmk__set_rsc_flags(instance, pcmk__rsc_unassigned);
	break;
	}

	// We prefer more strongly to assign an instance to the chosen node
	pcmk__rsc_debug(instance,
	"Not assigning %s to current node %s: %s is better",
	instance->id, pcmk__node_name(current),
	pcmk__node_name(chosen));

	// Reserve one instance for the chosen node and try again
	if (++reserved >= available) {
	pcmk__rsc_info(instance,
	"Not assigning %s to current node %s: "
	"other assignments are more important",
	instance->id, pcmk__node_name(current));

	} else {
	pcmk__rsc_debug(instance,
	"Reserved an instance of %s for %s. Retrying "
	"assignment of %s to %s",
	rsc->id, pcmk__node_name(chosen), instance->id,
	pcmk__node_name(current));
	}

	// Clear this assignment (frees chosen); leave instance counts in parent
	pcmk__unassign_resource(instance);
	chosen = NULL;
	}

	g_hash_table_destroy(allowed_orig);

	// Restore original instances-per-node counts
	g_hash_table_destroy(parent->private->allowed_nodes);
	parent->private->allowed_nodes = allowed_orig_parent;

	if (chosen == NULL) {
	// Couldn't assign instance to current node
	return false;
	}
	pcmk__rsc_trace(instance, "Assigned %s to current node %s",
	instance->id, pcmk__node_name(current));
	increment_parent_count(instance, chosen);
	return true;
	}

	/*!
	* \internal
	* \brief Reset the node counts of a resource's allowed nodes to zero
	*
	* \param[in,out] rsc Resource to reset
	*
	* \return Number of nodes that are available to run resources
	*/
	static unsigned int
	reset_allowed_node_counts(pcmk_resource_t *rsc)
	{
	unsigned int available_nodes = 0;
	pcmk_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	- node->count = 0;
	+ node->assign->count = 0;
	if (pcmk__node_available(node, false, false)) {
	available_nodes++;
	}
	}
	return available_nodes;
	}

	/*!
	* \internal
	* \brief Check whether an instance has a preferred node
	*
	* \param[in] instance Clone instance or bundle replica container
	* \param[in] optimal_per_node Optimal number of instances per node
	*
	* \return Instance's current node if still available, otherwise NULL
	*/
	static const pcmk_node_t *
	preferred_node(const pcmk_resource_t *instance, int optimal_per_node)
	{
	const pcmk_node_t *node = NULL;
	const pcmk_node_t *parent_node = NULL;

	// Check whether instance is active, healthy, and not yet assigned
	if ((instance->private->active_nodes == NULL)
	\|\| !pcmk_is_set(instance->flags, pcmk__rsc_unassigned)
	\|\| pcmk_is_set(instance->flags, pcmk__rsc_failed)) {
	return NULL;
	}

	// Check whether instance's current node can run resources
	node = pcmk__current_node(instance);
	if (!pcmk__node_available(node, true, false)) {
	pcmk__rsc_trace(instance, "Not assigning %s to %s early (unavailable)",
	instance->id, pcmk__node_name(node));
	return NULL;
	}

	// Check whether node already has optimal number of instances assigned
	parent_node = pcmk__top_allowed_node(instance, node);
	- if ((parent_node != NULL) && (parent_node->count >= optimal_per_node)) {
	+ if ((parent_node != NULL)
	+ && (parent_node->assign->count >= optimal_per_node)) {
	pcmk__rsc_trace(instance,
	"Not assigning %s to %s early "
	"(optimal instances already assigned)",
	instance->id, pcmk__node_name(node));
	return NULL;
	}

	return node;
	}

	/*!
	* \internal
	* \brief Assign collective instances to nodes
	*
	* \param[in,out] collective Clone or bundle resource being assigned
	* \param[in,out] instances List of clone instances or bundle containers
	* \param[in] max_total Maximum instances to assign in total
	* \param[in] max_per_node Maximum instances to assign to any one node
	*/
	void
	pcmk__assign_instances(pcmk_resource_t collective, GList instances,
	int max_total, int max_per_node)
	{
	// Reuse node count to track number of assigned instances
	unsigned int available_nodes = reset_allowed_node_counts(collective);

	int optimal_per_node = 0;
	int assigned = 0;
	GList *iter = NULL;
	pcmk_resource_t *instance = NULL;
	const pcmk_node_t *current = NULL;

	if (available_nodes > 0) {
	optimal_per_node = max_total / available_nodes;
	}
	if (optimal_per_node < 1) {
	optimal_per_node = 1;
	}

	pcmk__rsc_debug(collective,
	"Assigning up to %d %s instance%s to up to %u node%s "
	"(at most %d per host, %d optimal)",
	max_total, collective->id, pcmk__plural_s(max_total),
	available_nodes, pcmk__plural_s(available_nodes),
	max_per_node, optimal_per_node);

	// Assign as many instances as possible to their current location
	for (iter = instances; (iter != NULL) && (assigned < max_total);
	iter = iter->next) {
	int available = max_total - assigned;

	instance = iter->data;
	if (!pcmk_is_set(instance->flags, pcmk__rsc_unassigned)) {
	continue; // Already assigned
	}

	current = preferred_node(instance, optimal_per_node);
	if ((current != NULL)
	&& assign_instance_early(collective, instance, current,
	max_per_node, available)) {
	assigned++;
	}
	}

	pcmk__rsc_trace(collective, "Assigned %d of %d instance%s to current node",
	assigned, max_total, pcmk__plural_s(max_total));

	for (iter = instances; iter != NULL; iter = iter->next) {
	instance = (pcmk_resource_t *) iter->data;

	if (!pcmk_is_set(instance->flags, pcmk__rsc_unassigned)) {
	continue; // Already assigned
	}

	if (instance->private->active_nodes != NULL) {
	current = pcmk__current_node(instance);
	if (pcmk__top_allowed_node(instance, current) == NULL) {
	const char *unmanaged = "";

	if (!pcmk_is_set(instance->flags, pcmk__rsc_managed)) {
	unmanaged = "Unmanaged resource ";
	}
	crm_notice("%s%s is running on %s which is no longer allowed",
	unmanaged, instance->id, pcmk__node_name(current));
	}
	}

	if (assigned >= max_total) {
	pcmk__rsc_debug(collective,
	"Not assigning %s because maximum %d instances "
	"already assigned",
	instance->id, max_total);
	resource_location(instance, NULL, -PCMK_SCORE_INFINITY,
	"collective_limit_reached",
	collective->private->scheduler);

	} else if (assign_instance(instance, NULL, max_per_node) != NULL) {
	assigned++;
	}
	}

	pcmk__rsc_debug(collective, "Assigned %d of %d possible instance%s of %s",
	assigned, max_total, pcmk__plural_s(max_total),
	collective->id);
	}

	enum instance_state {
	instance_starting = (1 << 0),
	instance_stopping = (1 << 1),

	/* This indicates that some instance is restarting. It's not the same as
	* instance_starting\|instance_stopping, which would indicate that some
	* instance is starting, and some instance (not necessarily the same one) is
	* stopping.
	*/
	instance_restarting = (1 << 2),

	instance_active = (1 << 3),

	instance_all = instance_starting\|instance_stopping
	\|instance_restarting\|instance_active,
	};

	/*!
	* \internal
	* \brief Check whether an instance is active, starting, and/or stopping
	*
	* \param[in] instance Clone instance or bundle replica container
	* \param[in,out] state Whether any instance is starting, stopping, etc.
	*/
	static void
	check_instance_state(const pcmk_resource_t instance, uint32_t state)
	{
	const GList *iter = NULL;
	uint32_t instance_state = 0; // State of just this instance

	// No need to check further if all conditions have already been detected
	if (pcmk_all_flags_set(*state, instance_all)) {
	return;
	}

	// If instance is a collective (a cloned group), check its children instead
	if (instance->private->variant > pcmk__rsc_variant_primitive) {
	for (iter = instance->private->children;
	(iter != NULL) && !pcmk_all_flags_set(*state, instance_all);
	iter = iter->next) {
	check_instance_state((const pcmk_resource_t *) iter->data, state);
	}
	return;
	}

	// If we get here, instance is a primitive

	if (instance->private->active_nodes != NULL) {
	instance_state \|= instance_active;
	}

	// Check each of the instance's actions for runnable start or stop
	for (iter = instance->private->actions;
	(iter != NULL) && !pcmk_all_flags_set(instance_state,
	instance_starting
	\|instance_stopping);
	iter = iter->next) {

	const pcmk_action_t action = (const pcmk_action_t ) iter->data;
	const bool optional = pcmk_is_set(action->flags, pcmk_action_optional);

	if (pcmk__str_eq(PCMK_ACTION_START, action->task, pcmk__str_none)) {
	if (!optional
	&& pcmk_is_set(action->flags, pcmk_action_runnable)) {

	pcmk__rsc_trace(instance, "Instance is starting due to %s",
	action->uuid);
	instance_state \|= instance_starting;
	} else {
	pcmk__rsc_trace(instance, "%s doesn't affect %s state (%s)",
	action->uuid, instance->id,
	(optional? "optional" : "unrunnable"));
	}

	} else if (pcmk__str_eq(PCMK_ACTION_STOP, action->task,
	pcmk__str_none)) {
	/* Only stop actions can be pseudo-actions for primitives. That
	* indicates that the node they are on is being fenced, so the stop
	* is implied rather than actually executed.
	*/
	if (!optional
	&& pcmk_any_flags_set(action->flags, pcmk_action_pseudo
	\|pcmk_action_runnable)) {
	pcmk__rsc_trace(instance, "Instance is stopping due to %s",
	action->uuid);
	instance_state \|= instance_stopping;
	} else {
	pcmk__rsc_trace(instance, "%s doesn't affect %s state (%s)",
	action->uuid, instance->id,
	(optional? "optional" : "unrunnable"));
	}
	}
	}

	if (pcmk_all_flags_set(instance_state,
	instance_starting\|instance_stopping)) {
	instance_state \|= instance_restarting;
	}
	*state \|= instance_state;
	}

	/*!
	* \internal
	* \brief Create actions for collective resource instances
	*
	* \param[in,out] collective Clone or bundle resource to create actions for
	* \param[in,out] instances List of clone instances or bundle containers
	*/
	void
	pcmk__create_instance_actions(pcmk_resource_t collective, GList instances)
	{
	uint32_t state = 0;

	pcmk_action_t *stop = NULL;
	pcmk_action_t *stopped = NULL;

	pcmk_action_t *start = NULL;
	pcmk_action_t *started = NULL;

	pcmk__rsc_trace(collective, "Creating collective instance actions for %s",
	collective->id);

	// Create actions for each instance appropriate to its variant
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->private->cmds->create_actions(instance);
	check_instance_state(instance, &state);
	}

	// Create pseudo-actions for rsc start and started
	start = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_START,
	!pcmk_is_set(state, instance_starting),
	true);
	started = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_RUNNING,
	!pcmk_is_set(state, instance_starting),
	false);
	started->priority = PCMK_SCORE_INFINITY;
	if (pcmk_any_flags_set(state, instance_active\|instance_starting)) {
	pcmk__set_action_flags(started, pcmk_action_runnable);
	}

	// Create pseudo-actions for rsc stop and stopped
	stop = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_STOP,
	!pcmk_is_set(state, instance_stopping),
	true);
	stopped = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_STOPPED,
	!pcmk_is_set(state, instance_stopping),
	true);
	stopped->priority = PCMK_SCORE_INFINITY;
	if (!pcmk_is_set(state, instance_restarting)) {
	pcmk__set_action_flags(stop, pcmk_action_migratable);
	}

	if (pcmk__is_clone(collective)) {
	pe__create_clone_notif_pseudo_ops(collective, start, started, stop,
	stopped);
	}
	}

	/*!
	* \internal
	* \brief Get a list of clone instances or bundle replica containers
	*
	* \param[in] rsc Clone or bundle resource
	*
	* \return Clone instances if \p rsc is a clone, or a newly created list of
	* \p rsc's replica containers if \p rsc is a bundle
	* \note The caller must call free_instance_list() on the result when the list
	* is no longer needed.
	*/
	static inline GList *
	get_instance_list(const pcmk_resource_t *rsc)
	{
	if (pcmk__is_bundle(rsc)) {
	return pe__bundle_containers(rsc);
	} else {
	return rsc->private->children;
	}
	}

	/*!
	* \internal
	* \brief Free any memory created by get_instance_list()
	*
	* \param[in] rsc Clone or bundle resource passed to get_instance_list()
	* \param[in,out] list Return value of get_instance_list() for \p rsc
	*/
	static inline void
	free_instance_list(const pcmk_resource_t rsc, GList list)
	{
	if (list != rsc->private->children) {
	g_list_free(list);
	}
	}

	/*!
	* \internal
	* \brief Check whether an instance is compatible with a role and node
	*
	* \param[in] instance Clone instance or bundle replica container
	* \param[in] node Instance must match this node
	* \param[in] role If not pcmk_role_unknown, instance must match this role
	* \param[in] current If true, compare instance's original node and role,
	* otherwise compare assigned next node and role
	*
	* \return true if \p instance is compatible with \p node and \p role,
	* otherwise false
	*/
	bool
	pcmk__instance_matches(const pcmk_resource_t instance, const pcmk_node_t node,
	enum rsc_role_e role, bool current)
	{
	pcmk_node_t *instance_node = NULL;

	CRM_CHECK((instance != NULL) && (node != NULL), return false);

	if ((role != pcmk_role_unknown)
	&& (role != instance->private->fns->state(instance, current))) {
	pcmk__rsc_trace(instance,
	"%s is not a compatible instance (role is not %s)",
	instance->id, pcmk_role_text(role));
	return false;
	}

	if (!is_set_recursive(instance, pcmk__rsc_blocked, true)) {
	// We only want instances that haven't failed
	instance_node = instance->private->fns->location(instance, NULL,
	current);
	}

	if (instance_node == NULL) {
	pcmk__rsc_trace(instance,
	"%s is not a compatible instance "
	"(not assigned to a node)",
	instance->id);
	return false;
	}

	if (!pcmk__same_node(instance_node, node)) {
	pcmk__rsc_trace(instance,
	"%s is not a compatible instance "
	"(assigned to %s not %s)",
	instance->id, pcmk__node_name(instance_node),
	pcmk__node_name(node));
	return false;
	}

	return true;
	}

	#define display_role(r) \
	(((r) == pcmk_role_unknown)? "matching" : pcmk_role_text(r))

	/*!
	* \internal
	* \brief Find an instance that matches a given resource by node and role
	*
	* \param[in] match_rsc Resource that instance must match (for logging only)
	* \param[in] rsc Clone or bundle resource to check for matching instance
	* \param[in] node Instance must match this node
	* \param[in] role If not pcmk_role_unknown, instance must match this role
	* \param[in] current If true, compare instance's original node and role,
	* otherwise compare assigned next node and role
	*
	* \return \p rsc instance matching \p node and \p role if any, otherwise NULL
	*/
	static pcmk_resource_t *
	find_compatible_instance_on_node(const pcmk_resource_t *match_rsc,
	const pcmk_resource_t *rsc,
	const pcmk_node_t *node, enum rsc_role_e role,
	bool current)
	{
	GList *instances = NULL;

	instances = get_instance_list(rsc);
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	if (pcmk__instance_matches(instance, node, role, current)) {
	pcmk__rsc_trace(match_rsc,
	"Found %s %s instance %s compatible with %s on %s",
	display_role(role), rsc->id, instance->id,
	match_rsc->id, pcmk__node_name(node));
	free_instance_list(rsc, instances); // Only frees list, not contents
	return instance;
	}
	}
	free_instance_list(rsc, instances);

	pcmk__rsc_trace(match_rsc,
	"No %s %s instance found compatible with %s on %s",
	display_role(role), rsc->id, match_rsc->id,
	pcmk__node_name(node));
	return NULL;
	}

	/*!
	* \internal
	* \brief Find a clone instance or bundle container compatible with a resource
	*
	* \param[in] match_rsc Resource that instance must match
	* \param[in] rsc Clone or bundle resource to check for matching instance
	* \param[in] role If not pcmk_role_unknown, instance must match this role
	* \param[in] current If true, compare instance's original node and role,
	* otherwise compare assigned next node and role
	*
	* \return Compatible (by \p role and \p match_rsc location) instance of \p rsc
	* if any, otherwise NULL
	*/
	pcmk_resource_t *
	pcmk__find_compatible_instance(const pcmk_resource_t *match_rsc,
	const pcmk_resource_t *rsc, enum rsc_role_e role,
	bool current)
	{
	pcmk_resource_t *instance = NULL;
	GList *nodes = NULL;
	const pcmk_node_t *node = NULL;
	GHashTable *allowed_nodes = match_rsc->private->allowed_nodes;

	// If match_rsc has a node, check only that node
	node = match_rsc->private->fns->location(match_rsc, NULL, current);
	if (node != NULL) {
	return find_compatible_instance_on_node(match_rsc, rsc, node, role,
	current);
	}

	// Otherwise check for an instance matching any of match_rsc's allowed nodes
	nodes = pcmk__sort_nodes(g_hash_table_get_values(allowed_nodes), NULL);
	for (GList *iter = nodes; (iter != NULL) && (instance == NULL);
	iter = iter->next) {
	instance = find_compatible_instance_on_node(match_rsc, rsc,
	(pcmk_node_t *) iter->data,
	role, current);
	}

	if (instance == NULL) {
	pcmk__rsc_debug(rsc, "No %s instance found compatible with %s",
	rsc->id, match_rsc->id);
	}
	g_list_free(nodes);
	return instance;
	}

	/*!
	* \internal
	* \brief Unassign an instance if mandatory ordering has no interleave match
	*
	* \param[in] first 'First' action in an ordering
	* \param[in] then 'Then' action in an ordering
	* \param[in,out] then_instance 'Then' instance that has no interleave match
	* \param[in] type Group of enum pcmk__action_relation_flags
	* \param[in] current If true, "then" action is stopped or demoted
	*
	* \return true if \p then_instance was unassigned, otherwise false
	*/
	static bool
	unassign_if_mandatory(const pcmk_action_t first, const pcmk_action_t then,
	pcmk_resource_t *then_instance, uint32_t type,
	bool current)
	{
	// Allow "then" instance to go down even without an interleave match
	if (current) {
	pcmk__rsc_trace(then->rsc,
	"%s has no instance to order before stopping "
	"or demoting %s",
	first->rsc->id, then_instance->id);

	/* If the "first" action must be runnable, but there is no "first"
	* instance, the "then" instance must not be allowed to come up.
	*/
	} else if (pcmk_any_flags_set(type, pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_first_implies_then)) {
	pcmk__rsc_info(then->rsc,
	"Inhibiting %s from being active "
	"because there is no %s instance to interleave",
	then_instance->id, first->rsc->id);
	return pcmk__assign_resource(then_instance, NULL, true, true);
	}
	return false;
	}

	/*!
	* \internal
	* \brief Find first matching action for a clone instance or bundle container
	*
	* \param[in] action Action in an interleaved ordering
	* \param[in] instance Clone instance or bundle container being interleaved
	* \param[in] action_name Action to look for
	* \param[in] node If not NULL, require action to be on this node
	* \param[in] for_first If true, \p instance is the 'first' resource in the
	* ordering, otherwise it is the 'then' resource
	*
	* \return First action for \p instance (or in some cases if \p instance is a
	* bundle container, its containerized resource) that matches
	* \p action_name and \p node if any, otherwise NULL
	*/
	static pcmk_action_t *
	find_instance_action(const pcmk_action_t action, const pcmk_resource_t instance,
	const char action_name, const pcmk_node_t node,
	bool for_first)
	{
	const pcmk_resource_t *rsc = NULL;
	pcmk_action_t *matching_action = NULL;

	/* If instance is a bundle container, sometimes we should interleave the
	* action for the container itself, and sometimes for the containerized
	* resource.
	*
	* For example, given "start bundle A then bundle B", B likely requires the
	* service inside A's container to be active, rather than just the
	* container, so we should interleave the action for A's containerized
	* resource. On the other hand, it's possible B's container itself requires
	* something from A, so we should interleave the action for B's container.
	*
	* Essentially, for 'first', we should use the containerized resource for
	* everything except stop, and for 'then', we should use the container for
	* everything except promote and demote (which can only be performed on the
	* containerized resource).
	*/
	if ((for_first && !pcmk__str_any_of(action->task, PCMK_ACTION_STOP,
	PCMK_ACTION_STOPPED, NULL))

	\|\| (!for_first && pcmk__str_any_of(action->task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_PROMOTED,
	PCMK_ACTION_DEMOTE,
	PCMK_ACTION_DEMOTED, NULL))) {

	rsc = pe__get_rsc_in_container(instance);
	}
	if (rsc == NULL) {
	rsc = instance; // No containerized resource, use instance itself
	} else {
	node = NULL; // Containerized actions are on bundle-created guest
	}

	matching_action = find_first_action(rsc->private->actions, NULL,
	action_name, node);
	if (matching_action != NULL) {
	return matching_action;
	}

	if (pcmk_is_set(instance->flags, pcmk__rsc_removed)
	\|\| pcmk__str_any_of(action_name, PCMK_ACTION_STOP, PCMK_ACTION_DEMOTE,
	NULL)) {
	crm_trace("No %s action found for %s%s",
	action_name,
	pcmk_is_set(instance->flags, pcmk__rsc_removed)? "orphan " : "",
	instance->id);
	} else {
	crm_err("No %s action found for %s to interleave (bug?)",
	action_name, instance->id);
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get the original action name of a bundle or clone action
	*
	* Given an action for a bundle or clone, get the original action name,
	* mapping notify to the action being notified, and if the instances are
	* primitives, mapping completion actions to the action that was completed
	* (for example, stopped to stop).
	*
	* \param[in] action Clone or bundle action to check
	*
	* \return Original action name for \p action
	*/
	static const char *
	orig_action_name(const pcmk_action_t *action)
	{
	// Any instance will do
	const pcmk_resource_t *instance = action->rsc->private->children->data;

	char *action_type = NULL;
	const char *action_name = action->task;
	enum action_tasks orig_task = pcmk_action_unspecified;

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_NOTIFIED, NULL)) {
	// action->uuid is RSC_(confirmed-){pre,post}_notify_ACTION_INTERVAL
	CRM_CHECK(parse_op_key(action->uuid, NULL, &action_type, NULL),
	return pcmk_action_text(pcmk_action_unspecified));
	action_name = strstr(action_type, "_notify_");
	CRM_CHECK(action_name != NULL,
	return pcmk_action_text(pcmk_action_unspecified));
	action_name += strlen("_notify_");
	}
	orig_task = get_complex_task(instance, action_name);
	free(action_type);
	return pcmk_action_text(orig_task);
	}

	/*!
	* \internal
	* \brief Update two interleaved actions according to an ordering between them
	*
	* Given information about an ordering of two interleaved actions, update the
	* actions' flags (and runnable_before members if appropriate) as appropriate
	* for the ordering. Effects may cascade to other orderings involving the
	* actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static uint32_t
	update_interleaved_actions(pcmk_action_t first, pcmk_action_t then,
	const pcmk_node_t *node, uint32_t filter,
	uint32_t type)
	{
	GList *instances = NULL;
	uint32_t changed = pcmk__updated_none;
	const char *orig_first_task = orig_action_name(first);

	// Stops and demotes must be interleaved with instance on current node
	bool current = pcmk__ends_with(first->uuid, "_" PCMK_ACTION_STOPPED "_0")
	\|\| pcmk__ends_with(first->uuid,
	"_" PCMK_ACTION_DEMOTED "_0");

	// Update the specified actions for each "then" instance individually
	instances = get_instance_list(then->rsc);
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pcmk_resource_t *first_instance = NULL;
	pcmk_resource_t *then_instance = iter->data;

	pcmk_action_t *first_action = NULL;
	pcmk_action_t *then_action = NULL;

	// Find a "first" instance to interleave with this "then" instance
	first_instance = pcmk__find_compatible_instance(then_instance,
	first->rsc,
	pcmk_role_unknown,
	current);

	if (first_instance == NULL) { // No instance can be interleaved
	if (unassign_if_mandatory(first, then, then_instance, type,
	current)) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	continue;
	}

	first_action = find_instance_action(first, first_instance,
	orig_first_task, node, true);
	if (first_action == NULL) {
	continue;
	}

	then_action = find_instance_action(then, then_instance, then->task,
	node, false);
	if (then_action == NULL) {
	continue;
	}

	if (order_actions(first_action, then_action, type)) {
	pcmk__set_updated_flags(changed, first,
	pcmk__updated_first\|pcmk__updated_then);
	}

	changed \|= then_instance->private->cmds->update_ordered_actions(
	first_action, then_action, node,
	first_instance->private->cmds->action_flags(first_action, node),
	filter, type, then->rsc->private->scheduler);
	}
	free_instance_list(then->rsc, instances);
	return changed;
	}

	/*!
	* \internal
	* \brief Check whether two actions in an ordering can be interleaved
	*
	* \param[in] first 'First' action in the ordering
	* \param[in] then 'Then' action in the ordering
	*
	* \return true if \p first and \p then can be interleaved, otherwise false
	*/
	static bool
	can_interleave_actions(const pcmk_action_t first, const pcmk_action_t then)
	{
	bool interleave = false;
	pcmk_resource_t *rsc = NULL;

	if ((first->rsc == NULL) \|\| (then->rsc == NULL)) {
	crm_trace("Not interleaving %s with %s: not resource actions",
	first->uuid, then->uuid);
	return false;
	}

	if (first->rsc == then->rsc) {
	crm_trace("Not interleaving %s with %s: same resource",
	first->uuid, then->uuid);
	return false;
	}

	if ((first->rsc->private->variant < pcmk__rsc_variant_clone)
	\|\| (then->rsc->private->variant < pcmk__rsc_variant_clone)) {
	crm_trace("Not interleaving %s with %s: not clones or bundles",
	first->uuid, then->uuid);
	return false;
	}

	if (pcmk__ends_with(then->uuid, "_stop_0")
	\|\| pcmk__ends_with(then->uuid, "_demote_0")) {
	rsc = first->rsc;
	} else {
	rsc = then->rsc;
	}

	interleave = crm_is_true(g_hash_table_lookup(rsc->private->meta,
	PCMK_META_INTERLEAVE));
	pcmk__rsc_trace(rsc, "'%s then %s' will %sbe interleaved (based on %s)",
	first->uuid, then->uuid, (interleave? "" : "not "),
	rsc->id);
	return interleave;
	}

	/*!
	* \internal
	* \brief Update non-interleaved instance actions according to an ordering
	*
	* Given information about an ordering of two non-interleaved actions, update
	* the actions' flags (and runnable_before members if appropriate) as
	* appropriate for the ordering. Effects may cascade to other orderings
	* involving the actions as well.
	*
	* \param[in,out] instance Clone instance or bundle container
	* \param[in,out] first "First" action in ordering
	* \param[in] then "Then" action in ordering (for \p instance's parent)
	* \param[in] node If not NULL, limit scope of ordering to this node
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static uint32_t
	update_noninterleaved_actions(pcmk_resource_t instance, pcmk_action_t first,
	const pcmk_action_t then, const pcmk_node_t node,
	uint32_t flags, uint32_t filter, uint32_t type)
	{
	pcmk_action_t *instance_action = NULL;
	pcmk_scheduler_t *scheduler = instance->private->scheduler;
	uint32_t instance_flags = 0;
	uint32_t changed = pcmk__updated_none;

	// Check whether instance has an equivalent of "then" action
	instance_action = find_first_action(instance->private->actions, NULL,
	then->task, node);
	if (instance_action == NULL) {
	return changed;
	}

	// Check whether action is runnable
	instance_flags = instance->private->cmds->action_flags(instance_action,
	node);
	if (!pcmk_is_set(instance_flags, pcmk_action_runnable)) {
	return changed;
	}

	// If so, update actions for the instance
	changed = instance->private->cmds->update_ordered_actions(first,
	instance_action,
	node, flags,
	filter, type,
	scheduler);

	// Propagate any changes to later actions
	if (pcmk_is_set(changed, pcmk__updated_then)) {
	for (GList *after_iter = instance_action->actions_after;
	after_iter != NULL; after_iter = after_iter->next) {
	pcmk__related_action_t *after = after_iter->data;

	pcmk__update_action_for_orderings(after->action, scheduler);
	}
	}

	return changed;
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two clone or bundle actions, update
	* the actions' flags (and runnable_before members if appropriate) as
	* appropriate for the ordering. Effects may cascade to other orderings
	* involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__instance_update_ordered_actions(pcmk_action_t first, pcmk_action_t then,
	const pcmk_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pcmk_scheduler_t *scheduler)
	{
	CRM_ASSERT((first != NULL) && (then != NULL) && (scheduler != NULL));

	if (then->rsc == NULL) {
	return pcmk__updated_none;

	} else if (can_interleave_actions(first, then)) {
	return update_interleaved_actions(first, then, node, filter, type);

	} else {
	uint32_t changed = pcmk__updated_none;
	GList *instances = get_instance_list(then->rsc);

	// Update actions for the clone or bundle resource itself
	changed \|= pcmk__update_ordered_actions(first, then, node, flags,
	filter, type, scheduler);

	// Update the 'then' clone instances or bundle containers individually
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pcmk_resource_t *instance = iter->data;

	changed \|= update_noninterleaved_actions(instance, first, then,
	node, flags, filter, type);
	}
	free_instance_list(then->rsc, instances);
	return changed;
	}
	}

	#define pe__clear_action_summary_flags(flags, action, flag) do { \
	flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Action summary", action->rsc->id, \
	flags, flag, #flag); \
	} while (0)

	/*!
	* \internal
	* \brief Return action flags for a given clone or bundle action
	*
	* \param[in,out] action Action for a clone or bundle
	* \param[in] instances Clone instances or bundle containers
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__collective_action_flags(pcmk_action_t action, const GList instances,
	const pcmk_node_t *node)
	{
	bool any_runnable = false;
	const char *action_name = orig_action_name(action);

	// Set original assumptions (optional and runnable may be cleared below)
	uint32_t flags = pcmk_action_optional
	\|pcmk_action_runnable
	\|pcmk_action_pseudo;

	for (const GList *iter = instances; iter != NULL; iter = iter->next) {
	const pcmk_resource_t *instance = iter->data;
	const pcmk_node_t *instance_node = NULL;
	pcmk_action_t *instance_action = NULL;
	uint32_t instance_flags;

	// Node is relevant only to primitive instances
	if (pcmk__is_primitive(instance)) {
	instance_node = node;
	}

	instance_action = find_first_action(instance->private->actions, NULL,
	action_name, instance_node);
	if (instance_action == NULL) {
	pcmk__rsc_trace(action->rsc, "%s has no %s action on %s",
	instance->id, action_name, pcmk__node_name(node));
	continue;
	}

	pcmk__rsc_trace(action->rsc, "%s has %s for %s on %s",
	instance->id, instance_action->uuid, action_name,
	pcmk__node_name(node));

	instance_flags = instance->private->cmds->action_flags(instance_action,
	node);

	// If any instance action is mandatory, so is the collective action
	if (pcmk_is_set(flags, pcmk_action_optional)
	&& !pcmk_is_set(instance_flags, pcmk_action_optional)) {
	pcmk__rsc_trace(instance, "%s is mandatory because %s is",
	action->uuid, instance_action->uuid);
	pe__clear_action_summary_flags(flags, action,
	pcmk_action_optional);
	pcmk__clear_action_flags(action, pcmk_action_optional);
	}

	// If any instance action is runnable, so is the collective action
	if (pcmk_is_set(instance_flags, pcmk_action_runnable)) {
	any_runnable = true;
	}
	}

	if (!any_runnable) {
	pcmk__rsc_trace(action->rsc,
	"%s is not runnable because no instance can run %s",
	action->uuid, action_name);
	pe__clear_action_summary_flags(flags, action, pcmk_action_runnable);
	if (node == NULL) {
	pcmk__clear_action_flags(action, pcmk_action_runnable);
	}
	}

	return flags;
	}
	diff --git a/lib/pacemaker/pcmk_sched_location.c b/lib/pacemaker/pcmk_sched_location.c
	index a407930eae..c37aecc1c3 100644
	--- a/lib/pacemaker/pcmk_sched_location.c
	+++ b/lib/pacemaker/pcmk_sched_location.c
	@@ -1,732 +1,732 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/common/rules_internal.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/rules.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	static int
	get_node_score(const char rule, const char score, bool raw,
	pcmk_node_t node, pcmk_resource_t rsc)
	{
	int score_f = 0;

	if (score == NULL) {
	pcmk__config_warn("Rule %s: no score specified (assuming 0)", rule);

	} else if (raw) {
	score_f = char2score(score);

	} else {
	const char *target = NULL;
	const char *attr_score = NULL;

	target = g_hash_table_lookup(rsc->private->meta,
	PCMK_META_CONTAINER_ATTRIBUTE_TARGET);

	attr_score = pcmk__node_attr(node, score, target,
	pcmk__rsc_node_current);
	if (attr_score == NULL) {
	crm_debug("Rule %s: %s did not have a value for %s",
	rule, pcmk__node_name(node), score);
	score_f = -PCMK_SCORE_INFINITY;

	} else {
	crm_debug("Rule %s: %s had value %s for %s",
	rule, pcmk__node_name(node), attr_score, score);
	score_f = char2score(attr_score);
	}
	}
	return score_f;
	}

	/*!
	* \internal
	* \brief Parse a role configuration for a location constraint
	*
	* \param[in] role_spec Role specification
	* \param[out] role Where to store parsed role
	*
	* \return true if role specification is valid, otherwise false
	*/
	static bool
	parse_location_role(const char role_spec, enum rsc_role_e role)
	{
	if (role_spec == NULL) {
	*role = pcmk_role_unknown;
	return true;
	}

	*role = pcmk_parse_role(role_spec);
	switch (*role) {
	case pcmk_role_unknown:
	return false;

	case pcmk_role_started:
	case pcmk_role_unpromoted:
	/* Any promotable clone instance cannot be promoted without being in
	* the unpromoted role first. Therefore, any constraint for the
	* started or unpromoted role applies to every role.
	*/
	*role = pcmk_role_unknown;
	break;

	default:
	break;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Generate a location constraint from a rule
	*
	* \param[in,out] rsc Resource that constraint is for
	* \param[in] rule_xml Rule XML (sub-element of location constraint)
	* \param[in] discovery Value of \c PCMK_XA_RESOURCE_DISCOVERY for
	* constraint
	* \param[out] next_change Where to set when rule evaluation will change
	* \param[in,out] rule_input Values used to evaluate rule criteria
	* (node-specific values will be overwritten by
	* this function)
	*
	* \return true if rule is valid, otherwise false
	*/
	static bool
	generate_location_rule(pcmk_resource_t rsc, xmlNode rule_xml,
	const char discovery, crm_time_t next_change,
	pcmk_rule_input_t *rule_input)
	{
	const char *rule_id = NULL;
	const char *score = NULL;
	const char *boolean = NULL;
	const char *role_spec = NULL;

	GList *iter = NULL;

	bool raw_score = true;
	bool score_allocated = false;

	pcmk__location_t *location_rule = NULL;
	enum rsc_role_e role = pcmk_role_unknown;
	enum pcmk__combine combine = pcmk__combine_unknown;

	rule_xml = pcmk__xe_resolve_idref(rule_xml, rsc->private->scheduler->input);
	if (rule_xml == NULL) {
	return false; // Error already logged
	}

	rule_id = crm_element_value(rule_xml, PCMK_XA_ID);
	if (rule_id == NULL) {
	pcmk__config_err("Ignoring " PCMK_XE_RULE " without " PCMK_XA_ID
	" in location constraint");
	return false;
	}

	boolean = crm_element_value(rule_xml, PCMK_XA_BOOLEAN_OP);
	role_spec = crm_element_value(rule_xml, PCMK_XA_ROLE);

	if (parse_location_role(role_spec, &role)) {
	crm_trace("Setting rule %s role filter to %s", rule_id, role_spec);
	} else {
	pcmk__config_err("Ignoring rule %s: Invalid " PCMK_XA_ROLE " '%s'",
	rule_id, role_spec);
	return false;
	}

	crm_trace("Processing location constraint rule %s", rule_id);

	score = crm_element_value(rule_xml, PCMK_XA_SCORE);
	if (score == NULL) {
	score = crm_element_value(rule_xml, PCMK_XA_SCORE_ATTRIBUTE);
	if (score != NULL) {
	raw_score = false;
	}
	}

	combine = pcmk__parse_combine(boolean);
	switch (combine) {
	case pcmk__combine_and:
	case pcmk__combine_or:
	break;

	default:
	/* @COMPAT When we can break behavioral backward compatibility,
	* return false
	*/
	pcmk__config_warn("Location constraint rule %s has invalid "
	PCMK_XA_BOOLEAN_OP " value '%s', using default "
	"'" PCMK_VALUE_AND "'",
	rule_id, boolean);
	combine = pcmk__combine_and;
	break;
	}

	location_rule = pcmk__new_location(rule_id, rsc, 0, discovery, NULL);
	CRM_CHECK(location_rule != NULL, return NULL);

	location_rule->role_filter = role;

	if ((rule_input->rsc_id != NULL) && (rule_input->rsc_id_nmatches > 0)
	&& !raw_score) {

	char *result = pcmk__replace_submatches(score, rule_input->rsc_id,
	rule_input->rsc_id_submatches,
	rule_input->rsc_id_nmatches);

	if (result != NULL) {
	score = result;
	score_allocated = true;
	}
	}

	for (iter = rsc->private->scheduler->nodes;
	iter != NULL; iter = iter->next) {

	pcmk_node_t *node = iter->data;

	rule_input->node_attrs = node->private->attrs;
	rule_input->rsc_params = pe_rsc_params(rsc, node,
	rsc->private->scheduler);

	if (pcmk_evaluate_rule(rule_xml, rule_input,
	next_change) == pcmk_rc_ok) {
	pcmk_node_t *local = pe__copy_node(node);

	location_rule->nodes = g_list_prepend(location_rule->nodes, local);
	- local->weight = get_node_score(rule_id, score, raw_score, node,
	- rsc);
	+ local->assign->score = get_node_score(rule_id, score, raw_score,
	+ node, rsc);
	crm_trace("%s has score %s after %s", pcmk__node_name(node),
	- pcmk_readable_score(local->weight), rule_id);
	+ pcmk_readable_score(local->assign->score), rule_id);
	}
	}

	if (score_allocated) {
	free((char *)score);
	}

	if (location_rule->nodes == NULL) {
	crm_trace("No matching nodes for location constraint rule %s", rule_id);
	} else {
	crm_trace("Location constraint rule %s matched %d nodes",
	rule_id, g_list_length(location_rule->nodes));
	}
	return true;
	}

	static void
	unpack_rsc_location(xmlNode xml_obj, pcmk_resource_t rsc,
	const char role_spec, const char score,
	char *rsc_id_match, int rsc_id_nmatches,
	regmatch_t *rsc_id_submatches)
	{
	const char *rsc_id = crm_element_value(xml_obj, PCMK_XA_RSC);
	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
	const char *node = crm_element_value(xml_obj, PCMK_XA_NODE);
	const char *discovery = crm_element_value(xml_obj,
	PCMK_XA_RESOURCE_DISCOVERY);

	if (rsc == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, rsc_id);
	return;
	}

	if (score == NULL) {
	score = crm_element_value(xml_obj, PCMK_XA_SCORE);
	}

	if ((node != NULL) && (score != NULL)) {
	int score_i = char2score(score);
	pcmk_node_t *match = pcmk_find_node(rsc->private->scheduler, node);
	enum rsc_role_e role = pcmk_role_unknown;
	pcmk__location_t *location = NULL;

	if (match == NULL) {
	crm_info("Ignoring location constraint %s "
	"because '%s' is not a known node",
	pcmk__s(id, "without ID"), node);
	return;
	}

	if (role_spec == NULL) {
	role_spec = crm_element_value(xml_obj, PCMK_XA_ROLE);
	}
	if (parse_location_role(role_spec, &role)) {
	crm_trace("Setting location constraint %s role filter: %s",
	id, role_spec);
	} else {
	/* @COMPAT The previous behavior of creating the constraint ignoring
	* the role is retained for now, but we should ignore the entire
	* constraint when we can break backward compatibility.
	*/
	pcmk__config_err("Ignoring role in constraint %s: "
	"Invalid value '%s'", id, role_spec);
	}

	location = pcmk__new_location(id, rsc, score_i, discovery, match);
	if (location == NULL) {
	return; // Error already logged
	}
	location->role_filter = role;

	} else {
	bool empty = true;
	crm_time_t *next_change = crm_time_new_undefined();
	pcmk_rule_input_t rule_input = {
	.now = rsc->private->scheduler->now,
	.rsc_meta = rsc->private->meta,
	.rsc_id = rsc_id_match,
	.rsc_id_submatches = rsc_id_submatches,
	.rsc_id_nmatches = rsc_id_nmatches,
	};

	/* This loop is logically parallel to pcmk__evaluate_rules(), except
	* instead of checking whether any rule is active, we set up location
	* constraints for each active rule.
	*
	* @COMPAT When we can break backward compatibility, limit location
	* constraints to a single rule, for consistency with other contexts.
	* Since a rule may contain other rules, this does not prohibit any
	* existing use cases.
	*/
	for (xmlNode *rule_xml = pcmk__xe_first_child(xml_obj, PCMK_XE_RULE,
	NULL, NULL);
	rule_xml != NULL; rule_xml = pcmk__xe_next_same(rule_xml)) {

	if (generate_location_rule(rsc, rule_xml, discovery, next_change,
	&rule_input)) {
	if (empty) {
	empty = false;
	continue;
	}
	pcmk__warn_once(pcmk__wo_location_rules,
	"Support for multiple " PCMK_XE_RULE
	" elements in a location constraint is "
	"deprecated and will be removed in a future "
	"release (use a single new rule combining the "
	"previous rules with " PCMK_XA_BOOLEAN_OP
	" set to '" PCMK_VALUE_OR "' instead)");
	}
	}

	if (empty) {
	pcmk__config_err("Ignoring constraint '%s' because it contains "
	"no valid rules", id);
	}

	/* If there is a point in the future when the evaluation of a rule will
	* change, make sure the scheduler is re-run by that time.
	*/
	if (crm_time_is_defined(next_change)) {
	time_t t = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(t, rsc->private->scheduler,
	"location rule evaluation");
	}
	crm_time_free(next_change);
	}
	}

	static void
	unpack_simple_location(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
	const char *value = crm_element_value(xml_obj, PCMK_XA_RSC);

	if (value) {
	pcmk_resource_t *rsc;

	rsc = pcmk__find_constraint_resource(scheduler->resources, value);
	unpack_rsc_location(xml_obj, rsc, NULL, NULL, NULL, 0, NULL);
	}

	value = crm_element_value(xml_obj, PCMK_XA_RSC_PATTERN);
	if (value) {
	regex_t regex;
	bool invert = false;

	if (value[0] == '!') {
	value++;
	invert = true;
	}

	if (regcomp(&regex, value, REG_EXTENDED) != 0) {
	pcmk__config_err("Ignoring constraint '%s' because "
	PCMK_XA_RSC_PATTERN
	" has invalid value '%s'", id, value);
	return;
	}

	for (GList *iter = scheduler->resources; iter != NULL;
	iter = iter->next) {

	pcmk_resource_t *r = iter->data;
	int nregs = 0;
	regmatch_t *pmatch = NULL;
	int status;

	if (regex.re_nsub > 0) {
	nregs = regex.re_nsub + 1;
	} else {
	nregs = 1;
	}
	pmatch = pcmk__assert_alloc(nregs, sizeof(regmatch_t));

	status = regexec(&regex, r->id, nregs, pmatch, 0);

	if (!invert && (status == 0)) {
	crm_debug("'%s' matched '%s' for %s", r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, r->id, nregs,
	pmatch);

	} else if (invert && (status != 0)) {
	crm_debug("'%s' is an inverted match of '%s' for %s",
	r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, NULL, 0, NULL);

	} else {
	crm_trace("'%s' does not match '%s' for %s", r->id, value, id);
	}

	free(pmatch);
	}

	regfree(&regex);
	}
	}

	// \return Standard Pacemaker return code
	static int
	unpack_location_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	const char *id = NULL;
	const char *rsc_id = NULL;
	const char *state = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_tag_t *tag = NULL;
	xmlNode *rsc_set = NULL;

	*expanded_xml = NULL;

	CRM_CHECK(xml_obj != NULL, return EINVAL);

	id = pcmk__xe_id(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_LOCATION);
	return pcmk_rc_ok;
	}

	rsc_id = crm_element_value(xml_obj, PCMK_XA_RSC);
	if (rsc_id == NULL) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, rsc_id, &rsc, &tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, rsc_id);
	return pcmk_rc_unpack_error;

	} else if (rsc != NULL) {
	// No template is referenced
	return pcmk_rc_ok;
	}

	state = crm_element_value(xml_obj, PCMK_XA_ROLE);

	*expanded_xml = pcmk__xml_copy(NULL, xml_obj);

	/* Convert any template or tag reference into constraint
	* PCMK_XE_RESOURCE_SET
	*/
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set, PCMK_XA_RSC,
	false, scheduler)) {
	pcmk__xml_free(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set != NULL) {
	if (state != NULL) {
	/* Move PCMK_XA_RSC_ROLE into converted PCMK_XE_RESOURCE_SET as
	* PCMK_XA_ROLE attribute
	*/
	crm_xml_add(rsc_set, PCMK_XA_ROLE, state);
	pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_ROLE);
	}
	crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_LOCATION);

	} else {
	// No sets
	pcmk__xml_free(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	// \return Standard Pacemaker return code
	static int
	unpack_location_set(xmlNode location, xmlNode set,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *xml_rsc = NULL;
	pcmk_resource_t *resource = NULL;
	const char *set_id;
	const char *role;
	const char *local_score;

	CRM_CHECK(set != NULL, return EINVAL);

	set_id = pcmk__xe_id(set);
	if (set_id == NULL) {
	pcmk__config_err("Ignoring " PCMK_XE_RESOURCE_SET " without "
	PCMK_XA_ID " in constraint '%s'",
	pcmk__s(pcmk__xe_id(location), "(missing ID)"));
	return pcmk_rc_unpack_error;
	}

	role = crm_element_value(set, PCMK_XA_ROLE);
	local_score = crm_element_value(set, PCMK_XA_SCORE);

	for (xml_rsc = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL, NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	resource = pcmk__find_constraint_resource(scheduler->resources,
	pcmk__xe_id(xml_rsc));
	if (resource == NULL) {
	pcmk__config_err("%s: No resource found for %s",
	set_id, pcmk__xe_id(xml_rsc));
	return pcmk_rc_unpack_error;
	}

	unpack_rsc_location(location, resource, role, local_score, NULL, 0,
	NULL);
	}

	return pcmk_rc_ok;
	}

	void
	pcmk__unpack_location(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	xmlNode *set = NULL;
	bool any_sets = false;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	if (unpack_location_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
	return;
	}

	if (expanded_xml) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL);
	set != NULL; set = pcmk__xe_next_same(set)) {

	any_sets = true;
	set = pcmk__xe_resolve_idref(set, scheduler->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_location_set(xml_obj, set, scheduler) != pcmk_rc_ok)) {

	if (expanded_xml) {
	pcmk__xml_free(expanded_xml);
	}
	return;
	}
	}

	if (expanded_xml) {
	pcmk__xml_free(expanded_xml);
	xml_obj = orig_xml;
	}

	if (!any_sets) {
	unpack_simple_location(xml_obj, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Add a new location constraint to scheduler data
	*
	* \param[in] id XML ID of location constraint
	* \param[in,out] rsc Resource in location constraint
	* \param[in] node_score Constraint score
	- * \param[in] discover_mode Resource discovery option for constraint
	+ * \param[in] probe_mode When resource should be probed on node
	* \param[in] node Node in constraint (or NULL if rule-based)
	*
	* \return Newly allocated location constraint on success, otherwise NULL
	* \note The result will be added to the cluster (via \p rsc) and should not be
	* freed separately.
	*/
	pcmk__location_t *
	pcmk__new_location(const char id, pcmk_resource_t rsc,
	- int node_score, const char discover_mode, pcmk_node_t node)
	+ int node_score, const char probe_mode, pcmk_node_t node)
	{
	pcmk__location_t *new_con = NULL;

	CRM_CHECK((node != NULL) \|\| (node_score == 0), return NULL);

	if (id == NULL) {
	pcmk__config_err("Invalid constraint: no ID specified");
	return NULL;
	}

	if (rsc == NULL) {
	pcmk__config_err("Invalid constraint %s: no resource specified", id);
	return NULL;
	}

	new_con = pcmk__assert_alloc(1, sizeof(pcmk__location_t));
	new_con->id = pcmk__str_copy(id);
	new_con->rsc = rsc;
	new_con->nodes = NULL;
	new_con->role_filter = pcmk_role_unknown;

	- if (pcmk__str_eq(discover_mode, PCMK_VALUE_ALWAYS,
	+ if (pcmk__str_eq(probe_mode, PCMK_VALUE_ALWAYS,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	- new_con->discover_mode = pcmk_probe_always;
	+ new_con->probe_mode = pcmk__probe_always;

	- } else if (pcmk__str_eq(discover_mode, PCMK_VALUE_NEVER,
	- pcmk__str_casei)) {
	- new_con->discover_mode = pcmk_probe_never;
	+ } else if (pcmk__str_eq(probe_mode, PCMK_VALUE_NEVER, pcmk__str_casei)) {
	+ new_con->probe_mode = pcmk__probe_never;

	- } else if (pcmk__str_eq(discover_mode, PCMK_VALUE_EXCLUSIVE,
	+ } else if (pcmk__str_eq(probe_mode, PCMK_VALUE_EXCLUSIVE,
	pcmk__str_casei)) {
	- new_con->discover_mode = pcmk_probe_exclusive;
	+ new_con->probe_mode = pcmk__probe_exclusive;
	pcmk__set_rsc_flags(rsc, pcmk__rsc_exclusive_probes);

	} else {
	pcmk__config_err("Invalid " PCMK_XA_RESOURCE_DISCOVERY " value %s "
	- "in location constraint", discover_mode);
	+ "in location constraint", probe_mode);
	}

	if (node != NULL) {
	pcmk_node_t *copy = pe__copy_node(node);

	- copy->weight = node_score;
	+ copy->assign->score = node_score;
	new_con->nodes = g_list_prepend(NULL, copy);
	}

	rsc->private->scheduler->placement_constraints =
	g_list_prepend(rsc->private->scheduler->placement_constraints, new_con);
	rsc->private->location_constraints =
	g_list_prepend(rsc->private->location_constraints, new_con);

	return new_con;
	}

	/*!
	* \internal
	* \brief Apply all location constraints
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__apply_locations(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->placement_constraints;
	iter != NULL; iter = iter->next) {
	pcmk__location_t *location = iter->data;

	location->rsc->private->cmds->apply_location(location->rsc, location);
	}
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a resource's allowed node scores
	*
	* \param[in,out] rsc Resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*
	* \note This does not consider the resource's children, so the resource's
	* apply_location() method should be used instead in most cases.
	*/
	void
	pcmk__apply_location(pcmk_resource_t rsc, pcmk__location_t location)
	{
	bool need_role = false;

	CRM_ASSERT((rsc != NULL) && (location != NULL));

	// If a role was specified, ensure constraint is applicable
	need_role = (location->role_filter > pcmk_role_unknown);
	if (need_role && (location->role_filter != rsc->private->next_role)) {
	pcmk__rsc_trace(rsc,
	"Not applying %s to %s because role will be %s not %s",
	location->id, rsc->id,
	pcmk_role_text(rsc->private->next_role),
	pcmk_role_text(location->role_filter));
	return;
	}

	if (location->nodes == NULL) {
	pcmk__rsc_trace(rsc, "Not applying %s to %s because no nodes match",
	location->id, rsc->id);
	return;
	}

	pcmk__rsc_trace(rsc, "Applying %s%s%s to %s", location->id,
	(need_role? " for role " : ""),
	(need_role? pcmk_role_text(location->role_filter) : ""),
	rsc->id);

	for (GList *iter = location->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t *node = iter->data;
	pcmk_node_t *allowed_node = NULL;

	allowed_node = g_hash_table_lookup(rsc->private->allowed_nodes,
	node->private->id);
	if (allowed_node == NULL) {
	pcmk__rsc_trace(rsc, "* = %d on %s",
	- node->weight, pcmk__node_name(node));
	+ node->assign->score, pcmk__node_name(node));
	allowed_node = pe__copy_node(node);
	g_hash_table_insert(rsc->private->allowed_nodes,
	(gpointer) allowed_node->private->id,
	allowed_node);
	} else {
	pcmk__rsc_trace(rsc, "* + %d on %s",
	- node->weight, pcmk__node_name(node));
	- allowed_node->weight = pcmk__add_scores(allowed_node->weight,
	- node->weight);
	+ node->assign->score, pcmk__node_name(node));
	+ allowed_node->assign->score =
	+ pcmk__add_scores(allowed_node->assign->score,
	+ node->assign->score);
	}

	- if (allowed_node->rsc_discover_mode < location->discover_mode) {
	- if (location->discover_mode == pcmk_probe_exclusive) {
	+ if (allowed_node->assign->probe_mode < location->probe_mode) {
	+ if (location->probe_mode == pcmk__probe_exclusive) {
	pcmk__set_rsc_flags(rsc, pcmk__rsc_exclusive_probes);
	}
	/* exclusive > never > always... always is default */
	- allowed_node->rsc_discover_mode = location->discover_mode;
	+ allowed_node->assign->probe_mode = location->probe_mode;
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_nodes.c b/lib/pacemaker/pcmk_sched_nodes.c
	index b5685710fe..b7c53aaad7 100644
	--- a/lib/pacemaker/pcmk_sched_nodes.c
	+++ b/lib/pacemaker/pcmk_sched_nodes.c
	@@ -1,444 +1,444 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Check whether a node is available to run resources
	*
	* \param[in] node Node to check
	* \param[in] consider_score If true, consider a negative score unavailable
	* \param[in] consider_guest If true, consider a guest node unavailable whose
	* resource will not be active
	*
	* \return true if node is online and not shutting down, unclean, or in standby
	* or maintenance mode, otherwise false
	*/
	bool
	pcmk__node_available(const pcmk_node_t *node, bool consider_score,
	bool consider_guest)
	{
	if ((node == NULL) \|\| (node->details == NULL) \|\| !node->details->online
	\|\| node->details->shutdown \|\| node->details->unclean
	\|\| pcmk_is_set(node->private->flags, pcmk__node_standby)
	\|\| node->details->maintenance) {
	return false;
	}

	- if (consider_score && (node->weight < 0)) {
	+ if (consider_score && (node->assign->score < 0)) {
	return false;
	}

	// @TODO Go through all callers to see which should set consider_guest
	if (consider_guest && pcmk__is_guest_or_bundle_node(node)) {
	pcmk_resource_t *guest = node->private->remote->private->launcher;

	if (guest->private->fns->location(guest, NULL, FALSE) == NULL) {
	return false;
	}
	}

	return true;
	}

	/*!
	* \internal
	* \brief Copy a hash table of node objects
	*
	* \param[in] nodes Hash table to copy
	*
	* \return New copy of nodes (or NULL if nodes is NULL)
	*/
	GHashTable *
	pcmk__copy_node_table(GHashTable *nodes)
	{
	GHashTable *new_table = NULL;
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	if (nodes == NULL) {
	return NULL;
	}
	new_table = pcmk__strkey_table(NULL, free);
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	pcmk_node_t *new_node = pe__copy_node(node);

	g_hash_table_insert(new_table, (gpointer) new_node->private->id,
	new_node);
	}
	return new_table;
	}

	/*!
	* \internal
	* \brief Free a table of node tables
	*
	* \param[in,out] data Table to free
	*
	* \note This is a \c GDestroyNotify wrapper for \c g_hash_table_destroy().
	*/
	static void
	destroy_node_tables(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \internal
	* \brief Recursively copy the node tables of a resource
	*
	* Build a hash table containing copies of the allowed nodes tables of \p rsc
	* and its entire tree of descendants. The key is the resource ID, and the value
	* is a copy of the resource's node table.
	*
	* \param[in] rsc Resource whose node table to copy
	* \param[in,out] copy Where to store the copied node tables
	*
	* \note \p *copy should be \c NULL for the top-level call.
	* \note The caller is responsible for freeing \p copy using
	* \c g_hash_table_destroy().
	*/
	void
	pcmk__copy_node_tables(const pcmk_resource_t rsc, GHashTable *copy)
	{
	CRM_ASSERT((rsc != NULL) && (copy != NULL));

	if (*copy == NULL) {
	*copy = pcmk__strkey_table(NULL, destroy_node_tables);
	}

	g_hash_table_insert(*copy, rsc->id,
	pcmk__copy_node_table(rsc->private->allowed_nodes));

	for (const GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk__copy_node_tables((const pcmk_resource_t *) iter->data, copy);
	}
	}

	/*!
	* \internal
	* \brief Recursively restore the node tables of a resource from backup
	*
	* Given a hash table containing backup copies of the allowed nodes tables of
	* \p rsc and its entire tree of descendants, replace the resources' current
	* node tables with the backed-up copies.
	*
	* \param[in,out] rsc Resource whose node tables to restore
	* \param[in] backup Table of backup node tables (created by
	* \c pcmk__copy_node_tables())
	*
	* \note This function frees the resources' current node tables.
	*/
	void
	pcmk__restore_node_tables(pcmk_resource_t rsc, GHashTable backup)
	{
	CRM_ASSERT((rsc != NULL) && (backup != NULL));

	g_hash_table_destroy(rsc->private->allowed_nodes);

	// Copy to avoid danger with multiple restores
	rsc->private->allowed_nodes =
	pcmk__copy_node_table(g_hash_table_lookup(backup, rsc->id));

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk__restore_node_tables((pcmk_resource_t *) iter->data, backup);
	}
	}

	/*!
	* \internal
	* \brief Copy a list of node objects
	*
	* \param[in] list List to copy
	* \param[in] reset Set copies' scores to 0
	*
	* \return New list of shallow copies of nodes in original list
	*/
	GList *
	pcmk__copy_node_list(const GList *list, bool reset)
	{
	GList *result = NULL;

	for (const GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk_node_t *new_node = NULL;
	pcmk_node_t *this_node = iter->data;

	new_node = pe__copy_node(this_node);
	if (reset) {
	- new_node->weight = 0;
	+ new_node->assign->score = 0;
	}
	result = g_list_prepend(result, new_node);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Compare two nodes for assignment preference
	*
	* Given two nodes, check which one is more preferred by assignment criteria
	* such as node score and utilization.
	*
	* \param[in] a First node to compare
	* \param[in] b Second node to compare
	* \param[in] data Node to prefer if all else equal
	*
	* \return -1 if \p a is preferred, +1 if \p b is preferred, or 0 if they are
	* equally preferred
	*/
	static gint
	compare_nodes(gconstpointer a, gconstpointer b, gpointer data)
	{
	const pcmk_node_t node1 = (const pcmk_node_t ) a;
	const pcmk_node_t node2 = (const pcmk_node_t ) b;
	const pcmk_node_t preferred = (const pcmk_node_t ) data;

	int node1_score = -PCMK_SCORE_INFINITY;
	int node2_score = -PCMK_SCORE_INFINITY;

	int result = 0;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	// Compare node scores

	if (pcmk__node_available(node1, false, false)) {
	- node1_score = node1->weight;
	+ node1_score = node1->assign->score;
	}
	if (pcmk__node_available(node2, false, false)) {
	- node2_score = node2->weight;
	+ node2_score = node2->assign->score;
	}

	if (node1_score > node2_score) {
	crm_trace("%s before %s (score %d > %d)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	node1_score, node2_score);
	return -1;
	}

	if (node1_score < node2_score) {
	crm_trace("%s after %s (score %d < %d)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	node1_score, node2_score);
	return 1;
	}

	// If appropriate, compare node utilization

	if (pcmk__str_eq(node1->private->scheduler->placement_strategy,
	PCMK_VALUE_MINIMAL, pcmk__str_casei)) {
	goto equal;
	}

	if (pcmk__str_eq(node1->private->scheduler->placement_strategy,
	PCMK_VALUE_BALANCED, pcmk__str_casei)) {

	result = pcmk__compare_node_capacities(node1, node2);
	if (result < 0) {
	crm_trace("%s before %s (greater capacity by %d attributes)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	result * -1);
	return -1;
	} else if (result > 0) {
	crm_trace("%s after %s (lower capacity by %d attributes)",
	pcmk__node_name(node1), pcmk__node_name(node2), result);
	return 1;
	}
	}

	// Compare number of resources already assigned to node

	if (node1->private->num_resources < node2->private->num_resources) {
	crm_trace("%s before %s (%d resources < %d)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	node1->private->num_resources, node2->private->num_resources);
	return -1;

	} else if (node1->private->num_resources > node2->private->num_resources) {
	crm_trace("%s after %s (%d resources > %d)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	node1->private->num_resources, node2->private->num_resources);
	return 1;
	}

	// Check whether one node is already running desired resource

	if (preferred != NULL) {
	if (pcmk__same_node(preferred, node1)) {
	crm_trace("%s before %s (preferred node)",
	pcmk__node_name(node1), pcmk__node_name(node2));
	return -1;
	} else if (pcmk__same_node(preferred, node2)) {
	crm_trace("%s after %s (not preferred node)",
	pcmk__node_name(node1), pcmk__node_name(node2));
	return 1;
	}
	}

	// If all else is equal, prefer node with lowest-sorting name
	equal:
	result = strcmp(node1->private->name, node2->private->name);
	if (result < 0) {
	crm_trace("%s before %s (name)",
	pcmk__node_name(node1), pcmk__node_name(node2));
	return -1;
	} else if (result > 0) {
	crm_trace("%s after %s (name)",
	pcmk__node_name(node1), pcmk__node_name(node2));
	return 1;
	}

	crm_trace("%s == %s", pcmk__node_name(node1), pcmk__node_name(node2));
	return 0;
	}

	/*!
	* \internal
	* \brief Sort a list of nodes by assigment preference
	*
	* \param[in,out] nodes Node list to sort
	* \param[in] active_node Node where resource being assigned is active
	*
	* \return New head of sorted list
	*/
	GList *
	pcmk__sort_nodes(GList nodes, pcmk_node_t active_node)
	{
	return g_list_sort_with_data(nodes, compare_nodes, active_node);
	}

	/*!
	* \internal
	* \brief Check whether any node is available to run resources
	*
	* \param[in] nodes Nodes to check
	*
	* \return true if any node in \p nodes is available to run resources,
	* otherwise false
	*/
	bool
	pcmk__any_node_available(GHashTable *nodes)
	{
	GHashTableIter iter;
	const pcmk_node_t *node = NULL;

	if (nodes == NULL) {
	return false;
	}
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (pcmk__node_available(node, true, false)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Apply node health values for all nodes in cluster
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__apply_node_health(pcmk_scheduler_t *scheduler)
	{
	int base_health = 0;
	enum pcmk__health_strategy strategy;
	const char *strategy_str =
	pcmk__cluster_option(scheduler->config_hash,
	PCMK_OPT_NODE_HEALTH_STRATEGY);

	strategy = pcmk__parse_health_strategy(strategy_str);
	if (strategy == pcmk__health_strategy_none) {
	return;
	}
	crm_info("Applying node health strategy '%s'", strategy_str);

	// The progressive strategy can use a base health score
	if (strategy == pcmk__health_strategy_progressive) {
	base_health = pe__health_score(PCMK_OPT_NODE_HEALTH_BASE, scheduler);
	}

	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	int health = pe__sum_node_health_scores(node, base_health);

	// An overall health score of 0 has no effect
	if (health == 0) {
	continue;
	}
	crm_info("Overall system health of %s is %d",
	pcmk__node_name(node), health);

	// Use node health as a location score for each resource on the node
	for (GList *r = scheduler->resources; r != NULL; r = r->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) r->data;

	bool constrain = true;

	if (health < 0) {
	/* Negative health scores do not apply to resources with
	* PCMK_META_ALLOW_UNHEALTHY_NODES=true.
	*/
	constrain = !crm_is_true(g_hash_table_lookup(rsc->private->meta,
	PCMK_META_ALLOW_UNHEALTHY_NODES));
	}
	if (constrain) {
	pcmk__new_location(strategy_str, rsc, health, NULL, node);
	} else {
	pcmk__rsc_trace(rsc, "%s is immune from health ban on %s",
	rsc->id, pcmk__node_name(node));
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Check for a node in a resource's parent's allowed nodes
	*
	* \param[in] rsc Resource whose parent should be checked
	* \param[in] node Node to check for
	*
	* \return Equivalent of \p node from \p rsc's parent's allowed nodes if any,
	* otherwise NULL
	*/
	pcmk_node_t *
	pcmk__top_allowed_node(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	GHashTable *allowed_nodes = NULL;

	if ((rsc == NULL) \|\| (node == NULL)) {
	return NULL;
	}

	if (rsc->private->parent == NULL) {
	allowed_nodes = rsc->private->allowed_nodes;
	} else {
	allowed_nodes = rsc->private->parent->private->allowed_nodes;
	}
	return g_hash_table_lookup(allowed_nodes, node->private->id);
	}
	diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c
	index 45bb56fe3d..a443780d30 100644
	--- a/lib/pacemaker/pcmk_sched_primitive.c
	+++ b/lib/pacemaker/pcmk_sched_primitive.c
	@@ -1,1716 +1,1716 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <stdint.h> // uint8_t, uint32_t

	#include <crm/common/xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	static void stop_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void start_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void demote_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void promote_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void assert_role_error(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);

	#define RSC_ROLE_MAX (pcmk_role_promoted + 1)

	static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the immediate next role when transitioning from one role
	* to a target role. For example, when going from Stopped to Promoted, the
	* next role is Unpromoted, because the resource must be started before it
	* can be promoted. The current state then becomes Started, which is fed
	* into this array again, giving a next role of Promoted.
	*
	* Current role Immediate next role Final target role
	* ------------ ------------------- -----------------
	*/
	/* Unknown / { pcmk_role_unknown, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_stopped, /* Started */
	pcmk_role_stopped, /* Unpromoted */
	pcmk_role_stopped, /* Promoted */
	},
	/* Stopped / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_started, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_unpromoted, /* Promoted */
	},
	/* Started / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_started, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	/* Unpromoted / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_stopped, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	/* Promoted / { pcmk_role_stopped, / Unknown */
	pcmk_role_unpromoted, /* Stopped */
	pcmk_role_unpromoted, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Function to schedule actions needed for a role change
	*
	* \param[in,out] rsc Resource whose role is changing
	* \param[in,out] node Node where resource will be in its next role
	* \param[in] optional Whether scheduled actions should be optional
	*/
	typedef void (rsc_transition_fn)(pcmk_resource_t rsc, pcmk_node_t *node,
	bool optional);

	static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the function needed to transition directly from one role
	* to another. NULL indicates that nothing is needed.
	*
	* Current role Transition function Next role
	* ------------ ------------------- ----------
	*/
	/* Unknown / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	assert_role_error, /* Started */
	assert_role_error, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Stopped / { assert_role_error, / Unknown */
	NULL, /* Stopped */
	start_resource, /* Started */
	start_resource, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Started / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	NULL, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Unpromoted / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	stop_resource, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Promoted / { assert_role_error, / Unknown */
	demote_resource, /* Stopped */
	demote_resource, /* Started */
	demote_resource, /* Unpromoted */
	NULL, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Get a list of a resource's allowed nodes sorted by node score
	*
	* \param[in] rsc Resource to check
	*
	* \return List of allowed nodes sorted by node score
	*/
	static GList *
	sorted_allowed_nodes(const pcmk_resource_t *rsc)
	{
	if (rsc->private->allowed_nodes != NULL) {
	GList *nodes = g_hash_table_get_values(rsc->private->allowed_nodes);

	if (nodes != NULL) {
	return pcmk__sort_nodes(nodes, pcmk__current_node(rsc));
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Assign a resource to its best allowed node, if possible
	*
	* \param[in,out] rsc Resource to choose a node for
	* \param[in] prefer If not \c NULL, prefer this node when all else
	* equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return true if \p rsc could be assigned to a node, otherwise false
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	static bool
	assign_best_node(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *nodes = NULL;
	pcmk_node_t *chosen = NULL;
	pcmk_node_t *best = NULL;
	const pcmk_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc);

	if (prefer == NULL) {
	prefer = most_free_node;
	}

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
	// We've already finished assignment of resources to nodes
	return rsc->private->assigned_node != NULL;
	}

	// Sort allowed nodes by score
	nodes = sorted_allowed_nodes(rsc);
	if (nodes != NULL) {
	best = (pcmk_node_t *) nodes->data; // First node has best score
	}

	if ((prefer != NULL) && (nodes != NULL)) {
	// Get the allowed node version of prefer
	chosen = g_hash_table_lookup(rsc->private->allowed_nodes,
	prefer->private->id);

	if (chosen == NULL) {
	pcmk__rsc_trace(rsc, "Preferred node %s for %s was unknown",
	pcmk__node_name(prefer), rsc->id);

	/* Favor the preferred node as long as its score is at least as good as
	* the best allowed node's.
	*
	* An alternative would be to favor the preferred node even if the best
	* node is better, when the best node's score is less than INFINITY.
	*/
	- } else if (chosen->weight < best->weight) {
	+ } else if (chosen->assign->score < best->assign->score) {
	pcmk__rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
	pcmk__node_name(chosen), rsc->id);
	chosen = NULL;

	} else if (!pcmk__node_available(chosen, true, false)) {
	pcmk__rsc_trace(rsc, "Preferred node %s for %s was unavailable",
	pcmk__node_name(chosen), rsc->id);
	chosen = NULL;

	} else {
	pcmk__rsc_trace(rsc,
	"Chose preferred node %s for %s "
	"(ignoring %d candidates)",
	pcmk__node_name(chosen), rsc->id,
	g_list_length(nodes));
	}
	}

	if ((chosen == NULL) && (best != NULL)) {
	/* Either there is no preferred node, or the preferred node is not
	* suitable, but another node is allowed to run the resource.
	*/

	chosen = best;

	if (!pcmk__is_unique_clone(rsc->private->parent)
	- && (chosen->weight > 0) // Zero not acceptable
	+ && (chosen->assign->score > 0) // Zero not acceptable
	&& pcmk__node_available(chosen, false, false)) {
	/* If the resource is already running on a node, prefer that node if
	* it is just as good as the chosen node.
	*
	* We don't do this for unique clone instances, because
	* pcmk__assign_instances() has already assigned instances to their
	* running nodes when appropriate, and if we get here, we don't want
	* remaining unassigned instances to prefer a node that's already
	* running another instance.
	*/
	pcmk_node_t *running = pcmk__current_node(rsc);

	if (running == NULL) {
	// Nothing to do

	} else if (!pcmk__node_available(running, true, false)) {
	pcmk__rsc_trace(rsc,
	"Current node for %s (%s) can't run resources",
	rsc->id, pcmk__node_name(running));

	} else {
	int nodes_with_best_score = 1;

	for (GList *iter = nodes->next; iter; iter = iter->next) {
	pcmk_node_t allowed = (pcmk_node_t ) iter->data;

	- if (allowed->weight != chosen->weight) {
	+ if (allowed->assign->score != chosen->assign->score) {
	// The nodes are sorted by score, so no more are equal
	break;
	}
	if (pcmk__same_node(allowed, running)) {
	// Scores are equal, so prefer the current node
	chosen = allowed;
	}
	nodes_with_best_score++;
	}

	if (nodes_with_best_score > 1) {
	uint8_t log_level = LOG_INFO;

	- if (chosen->weight >= PCMK_SCORE_INFINITY) {
	+ if (chosen->assign->score >= PCMK_SCORE_INFINITY) {
	log_level = LOG_WARNING;
	}
	do_crm_log(log_level,
	"Chose %s for %s from %d nodes with score %s",
	pcmk__node_name(chosen), rsc->id,
	nodes_with_best_score,
	- pcmk_readable_score(chosen->weight));
	+ pcmk_readable_score(chosen->assign->score));
	}
	}
	}

	pcmk__rsc_trace(rsc, "Chose %s for %s from %d candidates",
	pcmk__node_name(chosen), rsc->id, g_list_length(nodes));
	}

	pcmk__assign_resource(rsc, chosen, false, stop_if_fail);
	g_list_free(nodes);
	return rsc->private->assigned_node != NULL;
	}

	/*!
	* \internal
	* \brief Apply a "this with" colocation to a node's allowed node scores
	*
	* \param[in,out] colocation Colocation to apply
	* \param[in,out] rsc Resource being assigned
	*/
	static void
	apply_this_with(pcmk__colocation_t colocation, pcmk_resource_t rsc)
	{
	GHashTable *archive = NULL;
	pcmk_resource_t *other = colocation->primary;

	// In certain cases, we will need to revert the node scores
	if ((colocation->dependent_role >= pcmk_role_promoted)
	\|\| ((colocation->score < 0)
	&& (colocation->score > -PCMK_SCORE_INFINITY))) {
	archive = pcmk__copy_node_table(rsc->private->allowed_nodes);
	}

	if (pcmk_is_set(other->flags, pcmk__rsc_unassigned)) {
	pcmk__rsc_trace(rsc,
	"%s: Assigning colocation %s primary %s first"
	"(score=%d role=%s)",
	rsc->id, colocation->id, other->id,
	colocation->score,
	pcmk_role_text(colocation->dependent_role));
	other->private->cmds->assign(other, NULL, true);
	}

	// Apply the colocation score to this resource's allowed node scores
	rsc->private->cmds->apply_coloc_score(rsc, other, colocation, true);
	if ((archive != NULL)
	&& !pcmk__any_node_available(rsc->private->allowed_nodes)) {
	pcmk__rsc_info(rsc,
	"%s: Reverting scores from colocation with %s "
	"because no nodes allowed",
	rsc->id, other->id);
	g_hash_table_destroy(rsc->private->allowed_nodes);
	rsc->private->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive != NULL) {
	g_hash_table_destroy(archive);
	}
	}

	/*!
	* \internal
	* \brief Update a Pacemaker Remote node once its connection has been assigned
	*
	* \param[in] connection Connection resource that has been assigned
	*/
	static void
	remote_connection_assigned(const pcmk_resource_t *connection)
	{
	pcmk_node_t *remote_node = pcmk_find_node(connection->private->scheduler,
	connection->id);

	CRM_CHECK(remote_node != NULL, return);

	if ((connection->private->assigned_node != NULL)
	&& (connection->private->next_role != pcmk_role_stopped)) {

	crm_trace("Pacemaker Remote node %s will be online",
	remote_node->private->id);
	remote_node->details->online = TRUE;
	if (!pcmk_is_set(remote_node->private->flags, pcmk__node_seen)) {
	// Avoid unnecessary fence, since we will attempt connection
	remote_node->details->unclean = FALSE;
	}

	} else {
	crm_trace("Pacemaker Remote node %s will be shut down "
	"(%sassigned connection's next role is %s)",
	remote_node->private->id,
	((connection->private->assigned_node == NULL)? "un" : ""),
	pcmk_role_text(connection->private->next_role));
	remote_node->details->shutdown = TRUE;
	}
	}

	/*!
	* \internal
	* \brief Assign a primitive resource to a node
	*
	* \param[in,out] rsc Resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pcmk_node_t *
	pcmk__primitive_assign(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *this_with_colocations = NULL;
	GList *with_this_colocations = NULL;
	GList *iter = NULL;
	pcmk_resource_t *parent = NULL;
	pcmk__colocation_t *colocation = NULL;
	pcmk_scheduler_t *scheduler = NULL;

	CRM_ASSERT(pcmk__is_primitive(rsc));
	scheduler = rsc->private->scheduler;
	parent = rsc->private->parent;

	// Never assign a child without parent being assigned first
	if ((parent != NULL) && !pcmk_is_set(parent->flags, pcmk__rsc_assigning)) {
	pcmk__rsc_debug(rsc, "%s: Assigning parent %s first",
	rsc->id, parent->id);
	parent->private->cmds->assign(parent, prefer, stop_if_fail);
	}

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
	// Assignment has already been done
	const char *node_name = "no node";

	if (rsc->private->assigned_node != NULL) {
	node_name = pcmk__node_name(rsc->private->assigned_node);
	}
	pcmk__rsc_debug(rsc, "%s: pre-assigned to %s", rsc->id, node_name);
	return rsc->private->assigned_node;
	}

	// Ensure we detect assignment loops
	if (pcmk_is_set(rsc->flags, pcmk__rsc_assigning)) {
	pcmk__rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
	return NULL;
	}
	pcmk__set_rsc_flags(rsc, pcmk__rsc_assigning);

	pe__show_node_scores(true, rsc, "Pre-assignment",
	rsc->private->allowed_nodes, scheduler);

	this_with_colocations = pcmk__this_with_colocations(rsc);
	with_this_colocations = pcmk__with_this_colocations(rsc);

	// Apply mandatory colocations first, to satisfy as many as possible
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -PCMK_SCORE_INFINITY)
	\|\| (colocation->score >= PCMK_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -PCMK_SCORE_INFINITY)
	\|\| (colocation->score >= PCMK_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	pe__show_node_scores(true, rsc, "Mandatory-colocations",
	rsc->private->allowed_nodes, scheduler);

	// Then apply optional colocations
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -PCMK_SCORE_INFINITY)
	&& (colocation->score < PCMK_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -PCMK_SCORE_INFINITY)
	&& (colocation->score < PCMK_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	g_list_free(this_with_colocations);
	g_list_free(with_this_colocations);

	if (rsc->private->next_role == pcmk_role_stopped) {
	pcmk__rsc_trace(rsc,
	"Banning %s from all nodes because it will be stopped",
	rsc->id);
	resource_location(rsc, NULL, -PCMK_SCORE_INFINITY,
	PCMK_META_TARGET_ROLE, scheduler);

	} else if ((rsc->private->next_role > rsc->private->orig_role)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	&& (scheduler->no_quorum_policy == pcmk_no_quorum_freeze)) {
	crm_notice("Resource %s cannot be elevated from %s to %s due to "
	PCMK_OPT_NO_QUORUM_POLICY "=" PCMK_VALUE_FREEZE,
	rsc->id, pcmk_role_text(rsc->private->orig_role),
	pcmk_role_text(rsc->private->next_role));
	pe__set_next_role(rsc, rsc->private->orig_role,
	PCMK_OPT_NO_QUORUM_POLICY "=" PCMK_VALUE_FREEZE);
	}

	pe__show_node_scores(!pcmk_is_set(scheduler->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->private->allowed_nodes, scheduler);

	// Unmanage resource if fencing is enabled but no device is configured
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {
	pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed);
	}

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	// Unmanaged resources stay on their current node
	const char *reason = NULL;
	pcmk_node_t *assign_to = NULL;

	pe__set_next_role(rsc, rsc->private->orig_role, "unmanaged");
	assign_to = pcmk__current_node(rsc);
	if (assign_to == NULL) {
	reason = "inactive";
	} else if (rsc->private->orig_role == pcmk_role_promoted) {
	reason = "promoted";
	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
	reason = "failed";
	} else {
	reason = "active";
	}
	pcmk__rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id,
	(assign_to? assign_to->private->name : "no node"),
	reason);
	pcmk__assign_resource(rsc, assign_to, true, stop_if_fail);

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	// Must stop at some point, but be consistent with stop_if_fail
	if (stop_if_fail) {
	pcmk__rsc_debug(rsc,
	"Forcing %s to stop: " PCMK_OPT_STOP_ALL_RESOURCES,
	rsc->id);
	}
	pcmk__assign_resource(rsc, NULL, true, stop_if_fail);

	} else if (!assign_best_node(rsc, prefer, stop_if_fail)) {
	// Assignment failed
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
	pcmk__rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
	} else if ((rsc->private->active_nodes != NULL) && stop_if_fail) {
	pcmk__rsc_info(rsc, "Stopping removed resource %s", rsc->id);
	}
	}

	pcmk__clear_rsc_flags(rsc, pcmk__rsc_assigning);

	if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
	remote_connection_assigned(rsc);
	}

	return rsc->private->assigned_node;
	}

	/*!
	* \internal
	* \brief Schedule actions to bring resource down and back to current role
	*
	* \param[in,out] rsc Resource to restart
	* \param[in,out] current Node that resource should be brought down on
	* \param[in] need_stop Whether the resource must be stopped
	* \param[in] need_promote Whether the resource must be promoted
	*
	* \return Role that resource would have after scheduled actions are taken
	*/
	static void
	schedule_restart_actions(pcmk_resource_t rsc, pcmk_node_t current,
	bool need_stop, bool need_promote)
	{
	enum rsc_role_e role = rsc->private->orig_role;
	enum rsc_role_e next_role;
	rsc_transition_fn fn = NULL;

	pcmk__set_rsc_flags(rsc, pcmk__rsc_restarting);

	// Bring resource down to a stop on its current node
	while (role != pcmk_role_stopped) {
	next_role = rsc_state_matrix[role][pcmk_role_stopped];
	pcmk__rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
	(need_stop? "required" : "optional"), rsc->id,
	pcmk_role_text(role), pcmk_role_text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, current, !need_stop);
	role = next_role;
	}

	// Bring resource up to its next role on its next node
	while ((rsc->private->orig_role <= rsc->private->next_role)
	&& (role != rsc->private->orig_role)
	&& !pcmk_is_set(rsc->flags, pcmk__rsc_blocked)) {
	bool required = need_stop;

	next_role = rsc_state_matrix[role][rsc->private->orig_role];
	if ((next_role == pcmk_role_promoted) && need_promote) {
	required = true;
	}
	pcmk__rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
	(required? "required" : "optional"), rsc->id,
	pcmk_role_text(role), pcmk_role_text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->private->assigned_node, !required);
	role = next_role;
	}

	pcmk__clear_rsc_flags(rsc, pcmk__rsc_restarting);
	}

	/*!
	* \internal
	* \brief If a resource's next role is not explicitly specified, set a default
	*
	* \param[in,out] rsc Resource to set next role for
	*
	* \return "explicit" if next role was explicitly set, otherwise "implicit"
	*/
	static const char *
	set_default_next_role(pcmk_resource_t *rsc)
	{
	if (rsc->private->next_role != pcmk_role_unknown) {
	return "explicit";
	}

	if (rsc->private->assigned_node == NULL) {
	pe__set_next_role(rsc, pcmk_role_stopped, "assignment");
	} else {
	pe__set_next_role(rsc, pcmk_role_started, "assignment");
	}
	return "implicit";
	}

	/*!
	* \internal
	* \brief Create an action to represent an already pending start
	*
	* \param[in,out] rsc Resource to create start action for
	*/
	static void
	create_pending_start(pcmk_resource_t *rsc)
	{
	pcmk_action_t *start = NULL;

	pcmk__rsc_trace(rsc,
	"Creating action for %s to represent already pending start",
	rsc->id);
	start = start_action(rsc, rsc->private->assigned_node, TRUE);
	pcmk__set_action_flags(start, pcmk_action_always_in_graph);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to take a resource to its next role
	*
	* \param[in,out] rsc Resource to schedule actions for
	*/
	static void
	schedule_role_transition_actions(pcmk_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->private->orig_role;

	while (role != rsc->private->next_role) {
	enum rsc_role_e next_role =
	rsc_state_matrix[role][rsc->private->next_role];
	rsc_transition_fn fn = NULL;

	pcmk__rsc_trace(rsc,
	"Creating action to take %s from %s to %s "
	"(ending at %s)",
	rsc->id, pcmk_role_text(role),
	pcmk_role_text(next_role),
	pcmk_role_text(rsc->private->next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->private->assigned_node, false);
	role = next_role;
	}
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given primitive resource
	*
	* \param[in,out] rsc Primitive resource to create actions for
	*/
	void
	pcmk__primitive_create_actions(pcmk_resource_t *rsc)
	{
	bool need_stop = false;
	bool need_promote = false;
	bool is_moving = false;
	bool allow_migrate = false;
	bool multiply_active = false;

	pcmk_node_t *current = NULL;
	pcmk_node_t *migration_target = NULL;
	unsigned int num_all_active = 0;
	unsigned int num_clean_active = 0;
	const char *next_role_source = NULL;

	CRM_ASSERT(pcmk__is_primitive(rsc));

	next_role_source = set_default_next_role(rsc);
	pcmk__rsc_trace(rsc,
	"Creating all actions for %s transition from %s to %s "
	"(%s) on %s",
	rsc->id, pcmk_role_text(rsc->private->orig_role),
	pcmk_role_text(rsc->private->next_role), next_role_source,
	pcmk__node_name(rsc->private->assigned_node));

	current = rsc->private->fns->active_node(rsc, &num_all_active,
	&num_clean_active);

	g_list_foreach(rsc->private->dangling_migration_sources,
	pcmk__abort_dangling_migration, rsc);

	if ((current != NULL) && (rsc->private->assigned_node != NULL)
	&& !pcmk__same_node(current, rsc->private->assigned_node)
	&& (rsc->private->next_role >= pcmk_role_started)) {

	pcmk__rsc_trace(rsc, "Moving %s from %s to %s",
	rsc->id, pcmk__node_name(current),
	pcmk__node_name(rsc->private->assigned_node));
	is_moving = true;
	allow_migrate = pcmk__rsc_can_migrate(rsc, current);

	// This is needed even if migrating (though I'm not sure why ...)
	need_stop = true;
	}

	// Check whether resource is partially migrated and/or multiply active
	migration_target = rsc->private->partial_migration_target;
	if ((rsc->private->partial_migration_source != NULL)
	&& (migration_target != NULL) && allow_migrate && (num_all_active == 2)
	&& pcmk__same_node(current, rsc->private->partial_migration_source)
	&& pcmk__same_node(rsc->private->assigned_node, migration_target)) {
	/* A partial migration is in progress, and the migration target remains
	* the same as when the migration began.
	*/
	pcmk__rsc_trace(rsc,
	"Partial migration of %s from %s to %s will continue",
	rsc->id,
	pcmk__node_name(rsc->private->partial_migration_source),
	pcmk__node_name(migration_target));

	} else if ((rsc->private->partial_migration_source != NULL)
	\|\| (migration_target != NULL)) {
	// A partial migration is in progress but can't be continued

	if (num_all_active > 2) {
	// The resource is migrating and multiply active!
	crm_notice("Forcing recovery of %s because it is migrating "
	"from %s to %s and possibly active elsewhere",
	rsc->id,
	pcmk__node_name(rsc->private->partial_migration_source),
	pcmk__node_name(migration_target));
	} else {
	// The migration source or target isn't available
	crm_notice("Forcing recovery of %s because it can no longer "
	"migrate from %s to %s",
	rsc->id,
	pcmk__node_name(rsc->private->partial_migration_source),
	pcmk__node_name(migration_target));
	}
	need_stop = true;
	rsc->private->partial_migration_source = NULL;
	rsc->private->partial_migration_target = NULL;
	allow_migrate = false;

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
	multiply_active = (num_all_active > 1);
	} else {
	/* If a resource has PCMK_META_REQUIRES set to PCMK_VALUE_NOTHING or
	* PCMK_VALUE_QUORUM, don't consider it active on unclean nodes (similar
	* to how all resources behave when PCMK_OPT_STONITH_ENABLED is false).
	* We can start such resources elsewhere before fencing completes, and
	* if we considered the resource active on the failed node, we would
	* attempt recovery for being active on multiple nodes.
	*/
	multiply_active = (num_clean_active > 1);
	}

	if (multiply_active) {
	const char *class = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);

	// Resource was (possibly) incorrectly multiply active
	pcmk__sched_err("%s resource %s might be active on %u nodes (%s)",
	pcmk__s(class, "Untyped"), rsc->id, num_all_active,
	pcmk__multiply_active_text(rsc));
	crm_notice("For more information, see \"What are multiply active "
	"resources?\" at "
	"https://projects.clusterlabs.org/w/clusterlabs/faq/");

	switch (rsc->private->multiply_active_policy) {
	case pcmk__multiply_active_restart:
	need_stop = true;
	break;
	case pcmk__multiply_active_unexpected:
	need_stop = true; // stop_resource() will skip expected node
	pcmk__set_rsc_flags(rsc, pcmk__rsc_stop_unexpected);
	break;
	default:
	break;
	}

	} else {
	pcmk__clear_rsc_flags(rsc, pcmk__rsc_stop_unexpected);
	}

	if (pcmk_is_set(rsc->flags, pcmk__rsc_start_pending)) {
	create_pending_start(rsc);
	}

	if (is_moving) {
	// Remaining tests are only for resources staying where they are

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
	if (pcmk_is_set(rsc->flags, pcmk__rsc_stop_if_failed)) {
	need_stop = true;
	pcmk__rsc_trace(rsc, "Recovering %s", rsc->id);
	} else {
	pcmk__rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
	if (rsc->private->next_role == pcmk_role_promoted) {
	need_promote = true;
	}
	}

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_blocked)) {
	pcmk__rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
	need_stop = true;

	} else if ((rsc->private->orig_role > pcmk_role_started)
	&& (current != NULL)
	&& (rsc->private->assigned_node != NULL)) {
	pcmk_action_t *start = NULL;

	pcmk__rsc_trace(rsc, "Creating start action for promoted resource %s",
	rsc->id);
	start = start_action(rsc, rsc->private->assigned_node, TRUE);
	if (!pcmk_is_set(start->flags, pcmk_action_optional)) {
	// Recovery of a promoted resource
	pcmk__rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
	need_stop = true;
	}
	}

	// Create any actions needed to bring resource down and back up to same role
	schedule_restart_actions(rsc, current, need_stop, need_promote);

	// Create any actions needed to take resource from this role to the next
	schedule_role_transition_actions(rsc);

	pcmk__create_recurring_actions(rsc);

	if (allow_migrate) {
	pcmk__create_migration_actions(rsc, current);
	}
	}

	/*!
	* \internal
	* \brief Ban a resource from any allowed nodes that are Pacemaker Remote nodes
	*
	* \param[in] rsc Resource to check
	*/
	static void
	rsc_avoids_remote_nodes(const pcmk_resource_t *rsc)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (node->private->remote != NULL) {
	- node->weight = -PCMK_SCORE_INFINITY;
	+ node->assign->score = -PCMK_SCORE_INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Return allowed nodes as (possibly sorted) list
	*
	* Convert a resource's hash table of allowed nodes to a list. If printing to
	* stdout, sort the list, to keep action ID numbers consistent for regression
	* test output (while avoiding the performance hit on a live cluster).
	*
	* \param[in] rsc Resource to check for allowed nodes
	*
	* \return List of resource's allowed nodes
	* \note Callers should take care not to rely on the list being sorted.
	*/
	static GList *
	allowed_nodes_as_list(const pcmk_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;

	if (rsc->private->allowed_nodes != NULL) {
	allowed_nodes = g_hash_table_get_values(rsc->private->allowed_nodes);
	}

	if (!pcmk__is_daemon) {
	allowed_nodes = g_list_sort(allowed_nodes, pe__cmp_node_name);
	}

	return allowed_nodes;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a primitive resource
	*
	* \param[in,out] rsc Primitive resource to create implicit constraints for
	*/
	void
	pcmk__primitive_internal_constraints(pcmk_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;
	bool check_unfencing = false;
	bool check_utilization = false;
	pcmk_scheduler_t *scheduler = NULL;

	CRM_ASSERT(pcmk__is_primitive(rsc));
	scheduler = rsc->private->scheduler;

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	pcmk__rsc_trace(rsc,
	"Skipping implicit constraints for unmanaged resource "
	"%s", rsc->id);
	return;
	}

	// Whether resource requires unfencing
	check_unfencing = !pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)
	&& pcmk_is_set(scheduler->flags,
	pcmk_sched_enable_unfencing)
	&& pcmk_is_set(rsc->flags, pcmk__rsc_needs_unfencing);

	// Whether a non-default placement strategy is used
	check_utilization = (g_hash_table_size(rsc->private->utilization) > 0)
	&& !pcmk__str_eq(scheduler->placement_strategy,
	PCMK_VALUE_DEFAULT, pcmk__str_casei);

	// Order stops before starts (i.e. restart)
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0), NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0), NULL,
	pcmk__ar_ordered
	\|pcmk__ar_first_implies_then
	\|pcmk__ar_intermediate_stop, scheduler);

	// Promotable ordering: demote before stop, start before promote
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk__rsc_promotable)
	\|\| (rsc->private->orig_role > pcmk_role_unpromoted)) {

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
	NULL,
	pcmk__ar_promoted_then_implies_first, scheduler);

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0),
	NULL,
	pcmk__ar_unrunnable_first_blocks, scheduler);
	}

	// Don't clear resource history if probing on same node
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	pcmk__ar_if_on_same_node\|pcmk__ar_then_cancels_first,
	scheduler);

	// Certain checks need allowed nodes
	if (check_unfencing \|\| check_utilization
	\|\| (rsc->private->launcher != NULL)) {

	allowed_nodes = allowed_nodes_as_list(rsc);
	}

	if (check_unfencing) {
	g_list_foreach(allowed_nodes, pcmk__order_restart_vs_unfence, rsc);
	}

	if (check_utilization) {
	pcmk__create_utilization_constraints(rsc, allowed_nodes);
	}

	if (rsc->private->launcher != NULL) {
	pcmk_resource_t *remote_rsc = NULL;

	if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
	// rsc is the implicit remote connection for a guest or bundle node

	/* Guest resources are not allowed to run on Pacemaker Remote nodes,
	* to avoid nesting remotes. However, bundles are allowed.
	*/
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_remote_nesting_allowed)) {
	rsc_avoids_remote_nodes(rsc->private->launcher);
	}

	/* If someone cleans up a guest or bundle node's launcher, we will
	* likely schedule a (re-)probe of the launcher and recovery of the
	* connection. Order the connection stop after the launcher probe,
	* so that if we detect the launcher running, we will trigger a new
	* transition and avoid the unnecessary recovery.
	*/
	pcmk__order_resource_actions(rsc->private->launcher,
	PCMK_ACTION_MONITOR,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);

	/* A user can specify that a resource must start on a Pacemaker Remote
	* node by explicitly configuring it with the PCMK__META_CONTAINER
	* meta-attribute. This is of questionable merit, since location
	* constraints can accomplish the same thing. But we support it, so here
	* we check whether a resource (that is not itself a remote connection)
	* has PCMK__META_CONTAINER set to a remote node or guest node resource.
	*/
	} else if (pcmk_is_set(rsc->private->launcher->flags,
	pcmk__rsc_is_remote_connection)) {
	remote_rsc = rsc->private->launcher;
	} else {
	remote_rsc =
	pe__resource_contains_guest_node(scheduler,
	rsc->private->launcher);
	}

	if (remote_rsc != NULL) {
	/* Force the resource on the Pacemaker Remote node instead of
	* colocating the resource with the launcher.
	*/
	for (GList *item = allowed_nodes; item; item = item->next) {
	pcmk_node_t *node = item->data;

	if (node->private->remote != remote_rsc) {
	- node->weight = -PCMK_SCORE_INFINITY;
	+ node->assign->score = -PCMK_SCORE_INFINITY;
	}
	}

	} else {
	/* This resource is either launched by a resource that does NOT
	* represent a Pacemaker Remote node, or a Pacemaker Remote
	* connection resource for a guest node or bundle.
	*/
	int score;

	crm_trace("Order and colocate %s relative to its launcher %s",
	rsc->id, rsc->private->launcher->id);

	pcmk__new_ordering(rsc->private->launcher,
	pcmk__op_key(rsc->private->launcher->id,
	PCMK_ACTION_START, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_first_implies_then
	\|pcmk__ar_unrunnable_first_blocks, scheduler);

	pcmk__new_ordering(rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
	NULL,
	rsc->private->launcher,
	pcmk__op_key(rsc->private->launcher->id,
	PCMK_ACTION_STOP, 0),
	NULL, pcmk__ar_then_implies_first, scheduler);

	if (pcmk_is_set(rsc->flags, pcmk__rsc_remote_nesting_allowed)) {
	score = 10000; /* Highly preferred but not essential */
	} else {
	score = PCMK_SCORE_INFINITY; // Force to run on same host
	}
	pcmk__new_colocation("#resource-with-container", NULL, score, rsc,
	rsc->private->launcher, NULL, NULL,
	pcmk__coloc_influence);
	}
	}

	if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)
	\|\| pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)) {
	/* Remote connections and fencing devices are not allowed to run on
	* Pacemaker Remote nodes
	*/
	rsc_avoids_remote_nodes(rsc);
	}
	g_list_free(allowed_nodes);
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__primitive_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	enum pcmk__coloc_affects filter_results;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (for_dependent) {
	// Always process on behalf of primary resource
	primary->private->cmds->apply_coloc_score(dependent, primary,
	colocation, false);
	return;
	}

	filter_results = pcmk__colocation_affects(dependent, primary, colocation,
	false);
	pcmk__rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
	((colocation->score > 0)? "Colocating" : "Anti-colocating"),
	dependent->id, primary->id, colocation->id,
	colocation->score,
	filter_results);

	switch (filter_results) {
	case pcmk__coloc_affects_role:
	pcmk__apply_coloc_to_priority(dependent, primary, colocation);
	break;
	case pcmk__coloc_affects_location:
	pcmk__apply_coloc_to_scores(dependent, primary, colocation);
	break;
	default: // pcmk__coloc_affects_nothing
	return;
	}
	}

	/* Primitive implementation of
	* pcmk__assignment_methods_t:with_this_colocations()
	*/
	void
	pcmk__with_primitive_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	const pcmk_resource_t *parent = NULL;

	CRM_ASSERT(pcmk__is_primitive(rsc) && (list != NULL));
	parent = rsc->private->parent;

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_with_this_list(list, rsc->private->with_this_colocations,
	orig_rsc);
	if (parent != NULL) {
	parent->private->cmds->with_this_colocations(parent, orig_rsc,
	list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->private->with_this_colocations;
	iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_with_this(list, colocation, orig_rsc);
	}
	}
	}
	}

	/* Primitive implementation of
	* pcmk__assignment_methods_t:this_with_colocations()
	*/
	void
	pcmk__primitive_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	const pcmk_resource_t *parent = NULL;

	CRM_ASSERT(pcmk__is_primitive(rsc) && (list != NULL));
	parent = rsc->private->parent;

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_this_with_list(list, rsc->private->this_with_colocations,
	orig_rsc);
	if (parent != NULL) {
	parent->private->cmds->this_with_colocations(parent, orig_rsc,
	list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->private->this_with_colocations;
	iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_this_with(list, colocation, orig_rsc);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given primitive resource action
	*
	* \param[in,out] action Action to get flags for
	* \param[in] node If not NULL, limit effects to this node (ignored)
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__primitive_action_flags(pcmk_action_t action, const pcmk_node_t node)
	{
	CRM_ASSERT(action != NULL);
	return (uint32_t) action->flags;
	}

	/*!
	* \internal
	* \brief Check whether a node is a multiply active resource's expected node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return \c true if \p rsc is multiply active with
	* \c PCMK_META_MULTIPLE_ACTIVE set to \c PCMK_VALUE_STOP_UNEXPECTED,
	* and \p node is the node where it will remain active
	* \note This assumes that the resource's next role cannot be changed to stopped
	* after this is called, which should be reasonable if status has already
	* been unpacked and resources have been assigned to nodes.
	*/
	static bool
	is_expected_node(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	return pcmk_all_flags_set(rsc->flags,
	pcmk__rsc_stop_unexpected\|pcmk__rsc_restarting)
	&& (rsc->private->next_role > pcmk_role_stopped)
	&& pcmk__same_node(rsc->private->assigned_node, node);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to stop a resource wherever it is active
	*
	* \param[in,out] rsc Resource being stopped
	* \param[in] node Node where resource is being stopped (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	stop_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	for (GList *iter = rsc->private->active_nodes;
	iter != NULL; iter = iter->next) {

	pcmk_node_t current = (pcmk_node_t ) iter->data;
	pcmk_action_t *stop = NULL;

	if (is_expected_node(rsc, current)) {
	/* We are scheduling restart actions for a multiply active resource
	* with PCMK_META_MULTIPLE_ACTIVE=PCMK_VALUE_STOP_UNEXPECTED, and
	* this is where it should not be stopped.
	*/
	pcmk__rsc_trace(rsc,
	"Skipping stop of multiply active resource %s "
	"on expected node %s",
	rsc->id, pcmk__node_name(current));
	continue;
	}

	if (rsc->private->partial_migration_target != NULL) {
	// Continue migration if node originally was and remains target
	if (pcmk__same_node(current, rsc->private->partial_migration_target)
	&& pcmk__same_node(current, rsc->private->assigned_node)) {
	pcmk__rsc_trace(rsc,
	"Skipping stop of %s on %s "
	"because partial migration there will continue",
	rsc->id, pcmk__node_name(current));
	continue;
	} else {
	pcmk__rsc_trace(rsc,
	"Forcing stop of %s on %s "
	"because migration target changed",
	rsc->id, pcmk__node_name(current));
	optional = false;
	}
	}

	pcmk__rsc_trace(rsc, "Scheduling stop of %s on %s",
	rsc->id, pcmk__node_name(current));
	stop = stop_action(rsc, current, optional);

	if (rsc->private->assigned_node == NULL) {
	pe_action_set_reason(stop, "node availability", true);
	} else if (pcmk_all_flags_set(rsc->flags, pcmk__rsc_restarting
	\|pcmk__rsc_stop_unexpected)) {
	/* We are stopping a multiply active resource on a node that is
	* not its expected node, and we are still scheduling restart
	* actions, so the stop is for being multiply active.
	*/
	pe_action_set_reason(stop, "being multiply active", true);
	}

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	pcmk__clear_action_flags(stop, pcmk_action_runnable);
	}

	if (pcmk_is_set(rsc->private->scheduler->flags,
	pcmk_sched_remove_after_stop)) {
	pcmk__schedule_cleanup(rsc, current, optional);
	}

	if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_unfencing)) {
	pcmk_action_t *unfence = pe_fence_op(current, PCMK_ACTION_ON, true,
	NULL, false,
	rsc->private->scheduler);

	order_actions(stop, unfence, pcmk__ar_then_implies_first);
	if (!pcmk__node_unfenced(current)) {
	pcmk__sched_err("Stopping %s until %s can be unfenced",
	rsc->id, pcmk__node_name(current));
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to start a resource on a node
	*
	* \param[in,out] rsc Resource being started
	* \param[in,out] node Node where resource should be started
	* \param[in] optional Whether actions should be optional
	*/
	static void
	start_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	pcmk_action_t *start = NULL;

	CRM_ASSERT(node != NULL);

	pcmk__rsc_trace(rsc, "Scheduling %s start of %s on %s (score %d)",
	(optional? "optional" : "required"), rsc->id,
	- pcmk__node_name(node), node->weight);
	+ pcmk__node_name(node), node->assign->score);
	start = start_action(rsc, node, TRUE);

	pcmk__order_vs_unfence(rsc, node, start, pcmk__ar_first_implies_then);

	if (pcmk_is_set(start->flags, pcmk_action_runnable) && !optional) {
	pcmk__clear_action_flags(start, pcmk_action_optional);
	}

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the start becomes necessary for other
	* reasons later.
	*/
	pcmk__rsc_trace(rsc,
	"Start of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pcmk__node_name(node));
	pcmk__set_action_flags(start, pcmk_action_pseudo);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to promote a resource on a node
	*
	* \param[in,out] rsc Resource being promoted
	* \param[in] node Node where resource should be promoted
	* \param[in] optional Whether actions should be optional
	*/
	static void
	promote_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	GList *iter = NULL;
	GList *action_list = NULL;
	bool runnable = true;

	CRM_ASSERT(node != NULL);

	// Any start must be runnable for promotion to be runnable
	action_list = pe__resource_actions(rsc, node, PCMK_ACTION_START, true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pcmk_action_t start = (pcmk_action_t ) iter->data;

	if (!pcmk_is_set(start->flags, pcmk_action_runnable)) {
	runnable = false;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	pcmk_action_t *promote = promote_action(rsc, node, optional);

	pcmk__rsc_trace(rsc, "Scheduling %s promotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pcmk__node_name(node));

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the promote becomes necessary for
	* other reasons later.
	*/
	pcmk__rsc_trace(rsc,
	"Promotion of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pcmk__node_name(node));
	pcmk__set_action_flags(promote, pcmk_action_pseudo);
	}
	} else {
	pcmk__rsc_trace(rsc, "Not promoting %s on %s: start unrunnable",
	rsc->id, pcmk__node_name(node));
	action_list = pe__resource_actions(rsc, node, PCMK_ACTION_PROMOTE,
	true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pcmk_action_t promote = (pcmk_action_t ) iter->data;

	pcmk__clear_action_flags(promote, pcmk_action_runnable);
	}
	g_list_free(action_list);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to demote a resource wherever it is active
	*
	* \param[in,out] rsc Resource being demoted
	* \param[in] node Node where resource should be demoted (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	demote_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	/* Since this will only be called for a primitive (possibly as an instance
	* of a collective resource), the resource is multiply active if it is
	* running on more than one node, so we want to demote on all of them as
	* part of recovery, regardless of which one is the desired node.
	*/
	for (GList *iter = rsc->private->active_nodes;
	iter != NULL; iter = iter->next) {

	pcmk_node_t current = (pcmk_node_t ) iter->data;

	if (is_expected_node(rsc, current)) {
	pcmk__rsc_trace(rsc,
	"Skipping demote of multiply active resource %s "
	"on expected node %s",
	rsc->id, pcmk__node_name(current));
	} else {
	pcmk__rsc_trace(rsc, "Scheduling %s demotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pcmk__node_name(current));
	demote_action(rsc, current, optional);
	}
	}
	}

	static void
	assert_role_error(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	CRM_ASSERT(false);
	}

	/*!
	* \internal
	* \brief Schedule cleanup of a resource
	*
	* \param[in,out] rsc Resource to clean up
	* \param[in] node Node to clean up on
	* \param[in] optional Whether clean-up should be optional
	*/
	void
	pcmk__schedule_cleanup(pcmk_resource_t rsc, const pcmk_node_t node,
	bool optional)
	{
	/* If the cleanup is required, its orderings are optional, because they're
	* relevant only if both actions are required. Conversely, if the cleanup is
	* optional, the orderings make the then action required if the first action
	* becomes required.
	*/
	uint32_t flag = optional? pcmk__ar_first_implies_then : pcmk__ar_ordered;

	CRM_CHECK((rsc != NULL) && (node != NULL), return);

	if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
	pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: resource failed",
	rsc->id, pcmk__node_name(node));
	return;
	}

	if (node->details->unclean \|\| !node->details->online) {
	pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: node unavailable",
	rsc->id, pcmk__node_name(node));
	return;
	}

	crm_notice("Scheduling clean-up of %s on %s",
	rsc->id, pcmk__node_name(node));
	delete_action(rsc, node, optional);

	// stop -> clean-up -> start
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_DELETE, flag);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DELETE,
	rsc, PCMK_ACTION_START, flag);
	}

	/*!
	* \internal
	* \brief Add primitive meta-attributes relevant to graph actions to XML
	*
	* \param[in] rsc Primitive resource whose meta-attributes should be added
	* \param[in,out] xml Transition graph action attributes XML to add to
	*/
	void
	pcmk__primitive_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml)
	{
	char *name = NULL;
	char *value = NULL;
	const pcmk_resource_t *parent = NULL;

	CRM_ASSERT(pcmk__is_primitive(rsc) && (xml != NULL));

	/* Clone instance numbers get set internally as meta-attributes, and are
	* needed in the transition graph (for example, to tell unique clone
	* instances apart).
	*/
	value = g_hash_table_lookup(rsc->private->meta, PCMK__META_CLONE);
	if (value != NULL) {
	name = crm_meta_name(PCMK__META_CLONE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	// Not sure if this one is really needed ...
	value = g_hash_table_lookup(rsc->private->meta, PCMK_META_REMOTE_NODE);
	if (value != NULL) {
	name = crm_meta_name(PCMK_META_REMOTE_NODE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	/* The PCMK__META_CONTAINER meta-attribute can be set on the primitive
	* itself or one of its ancestors, so check them all and keep the highest.
	*/
	for (parent = rsc; parent != NULL; parent = parent->private->parent) {
	if (parent->private->launcher != NULL) {
	crm_xml_add(xml, CRM_META "_" PCMK__META_CONTAINER,
	parent->private->launcher->id);
	}
	}

	/* Bundle replica children will get their external-ip set internally as a
	* meta-attribute. The graph action needs it, but under a different naming
	* convention than other meta-attributes.
	*/
	value = g_hash_table_lookup(rsc->private->meta, "external-ip");
	if (value != NULL) {
	crm_xml_add(xml, "pcmk_external_ip", value);
	}
	}

	// Primitive implementation of pcmk__assignment_methods_t:add_utilization()
	void
	pcmk__primitive_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList all_rscs, GHashTable utilization)
	{
	CRM_ASSERT(pcmk__is_primitive(rsc)
	&& (orig_rsc != NULL) && (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
	return;
	}

	pcmk__rsc_trace(orig_rsc,
	"%s: Adding primitive %s as colocated utilization",
	orig_rsc->id, rsc->id);
	pcmk__release_node_capacity(utilization, rsc);
	}

	/*!
	* \internal
	* \brief Get epoch time of node's shutdown attribute (or now if none)
	*
	* \param[in,out] node Node to check
	*
	* \return Epoch time corresponding to shutdown attribute if set or now if not
	*/
	static time_t
	shutdown_time(pcmk_node_t *node)
	{
	const char *shutdown = pcmk__node_attr(node, PCMK__NODE_ATTR_SHUTDOWN, NULL,
	pcmk__rsc_node_current);
	time_t result = 0;

	if (shutdown != NULL) {
	long long result_ll;

	if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
	result = (time_t) result_ll;
	}
	}
	return (result == 0)? get_effective_time(node->private->scheduler) : result;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it's not locked to the node
	*
	* \param[in] data Node to check
	* \param[in,out] user_data Resource to check
	*/
	static void
	ban_if_not_locked(gpointer data, gpointer user_data)
	{
	const pcmk_node_t node = (const pcmk_node_t ) data;
	pcmk_resource_t rsc = (pcmk_resource_t ) user_data;

	if (!pcmk__same_node(node, rsc->private->lock_node)) {
	resource_location(rsc, node, -PCMK_SCORE_INFINITY,
	PCMK_OPT_SHUTDOWN_LOCK, rsc->private->scheduler);
	}
	}

	// Primitive implementation of pcmk__assignment_methods_t:shutdown_lock()
	void
	pcmk__primitive_shutdown_lock(pcmk_resource_t *rsc)
	{
	const char *class = NULL;
	pcmk_scheduler_t *scheduler = NULL;

	CRM_ASSERT(pcmk__is_primitive(rsc));
	scheduler = rsc->private->scheduler;

	class = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);

	// Fence devices and remote connections can't be locked
	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
	\|\| pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
	return;
	}

	if (rsc->private->lock_node != NULL) {
	// The lock was obtained from resource history

	if (rsc->private->active_nodes != NULL) {
	/* The resource was started elsewhere even though it is now
	* considered locked. This shouldn't be possible, but as a
	* failsafe, we don't want to disturb the resource now.
	*/
	pcmk__rsc_info(rsc,
	"Cancelling shutdown lock "
	"because %s is already active", rsc->id);
	pe__clear_resource_history(rsc, rsc->private->lock_node);
	rsc->private->lock_node = NULL;
	rsc->private->lock_time = 0;
	}

	// Only a resource active on exactly one node can be locked
	} else if (pcmk__list_of_1(rsc->private->active_nodes)) {
	pcmk_node_t *node = rsc->private->active_nodes->data;

	if (node->details->shutdown) {
	if (node->details->unclean) {
	pcmk__rsc_debug(rsc,
	"Not locking %s to unclean %s for shutdown",
	rsc->id, pcmk__node_name(node));
	} else {
	rsc->private->lock_node = node;
	rsc->private->lock_time = shutdown_time(node);
	}
	}
	}

	if (rsc->private->lock_node == NULL) {
	// No lock needed
	return;
	}

	if (scheduler->shutdown_lock > 0) {
	time_t lock_expiration;

	lock_expiration = rsc->private->lock_time + scheduler->shutdown_lock;
	pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
	rsc->id, pcmk__node_name(rsc->private->lock_node),
	(long long) lock_expiration);
	pe__update_recheck_time(++lock_expiration, scheduler,
	"shutdown lock expiration");
	} else {
	pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown",
	rsc->id, pcmk__node_name(rsc->private->lock_node));
	}

	// If resource is locked to one node, ban it from all other nodes
	g_list_foreach(scheduler->nodes, ban_if_not_locked, rsc);
	}
	diff --git a/lib/pacemaker/pcmk_sched_probes.c b/lib/pacemaker/pcmk_sched_probes.c
	index 2340835a69..600ea03bea 100644
	--- a/lib/pacemaker/pcmk_sched_probes.c
	+++ b/lib/pacemaker/pcmk_sched_probes.c
	@@ -1,905 +1,905 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add the expected result to a newly created probe
	*
	* \param[in,out] probe Probe action to add expected result to
	* \param[in] rsc Resource that probe is for
	* \param[in] node Node that probe will run on
	*/
	static void
	add_expected_result(pcmk_action_t probe, const pcmk_resource_t rsc,
	const pcmk_node_t *node)
	{
	// Check whether resource is currently active on node
	pcmk_node_t *running = pe_find_node_id(rsc->private->active_nodes,
	node->private->id);

	// The expected result is what we think the resource's current state is
	if (running == NULL) {
	pe__add_action_expected_result(probe, CRM_EX_NOT_RUNNING);

	} else if (rsc->private->orig_role == pcmk_role_promoted) {
	pe__add_action_expected_result(probe, CRM_EX_PROMOTED);
	}
	}

	/*!
	* \internal
	* \brief Create any needed robes on a node for a list of resources
	*
	* \param[in,out] rscs List of resources to create probes for
	* \param[in,out] node Node to create probes on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_resource_list(GList rscs, pcmk_node_t node)
	{
	bool any_created = false;

	for (GList *iter = rscs; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->private->cmds->create_probe(rsc, node)) {
	any_created = true;
	}
	}
	return any_created;
	}

	/*!
	* \internal
	* \brief Order one resource's start after another's start-up probe
	*
	* \param[in,out] rsc1 Resource that might get start-up probe
	* \param[in] rsc2 Resource that might be started
	*/
	static void
	probe_then_start(pcmk_resource_t rsc1, pcmk_resource_t rsc2)
	{
	const pcmk_node_t *rsc1_node = rsc1->private->assigned_node;

	if ((rsc1_node != NULL)
	&& (g_hash_table_lookup(rsc1->private->probed_nodes,
	rsc1_node->private->id) == NULL)) {

	pcmk__new_ordering(rsc1,
	pcmk__op_key(rsc1->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	rsc2, pcmk__op_key(rsc2->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_ordered, rsc1->private->scheduler);
	}
	}

	/*!
	* \internal
	* \brief Check whether a guest resource will stop
	*
	* \param[in] node Guest node to check
	*
	* \return true if guest resource will likely stop, otherwise false
	*/
	static bool
	guest_resource_will_stop(const pcmk_node_t *node)
	{
	const pcmk_resource_t *guest_rsc = NULL;
	const pcmk_node_t *guest_node = NULL;

	guest_rsc = node->private->remote->private->launcher;
	guest_node = guest_rsc->private->assigned_node;

	/* Ideally, we'd check whether the guest has a required stop, but that
	* information doesn't exist yet, so approximate it ...
	*/
	return pcmk_is_set(node->private->flags, pcmk__node_remote_reset)
	\|\| node->details->unclean
	\|\| pcmk_is_set(guest_rsc->flags, pcmk__rsc_failed)
	\|\| (guest_rsc->private->next_role == pcmk_role_stopped)

	// Guest is moving
	\|\| ((guest_rsc->private->orig_role > pcmk_role_stopped)
	&& (guest_node != NULL)
	&& pcmk__find_node_in_list(guest_rsc->private->active_nodes,
	guest_node->private->name) == NULL);
	}

	/*!
	* \internal
	* \brief Create a probe action for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return Newly created probe action
	*/
	static pcmk_action_t *
	probe_action(pcmk_resource_t rsc, pcmk_node_t node)
	{
	pcmk_action_t *probe = NULL;
	char *key = pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0);

	crm_debug("Scheduling probe of %s %s on %s",
	pcmk_role_text(rsc->private->orig_role), rsc->id,
	pcmk__node_name(node));

	probe = custom_action(rsc, key, PCMK_ACTION_MONITOR, node, FALSE,
	rsc->private->scheduler);
	pcmk__clear_action_flags(probe, pcmk_action_optional);

	pcmk__order_vs_unfence(rsc, node, probe, pcmk__ar_ordered);
	add_expected_result(probe, rsc, node);
	return probe;
	}

	/*!
	* \internal
	* \brief Create probes for a resource on a node, if needed
	*
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_rsc_on_node(pcmk_resource_t rsc, pcmk_node_t node)
	{
	uint32_t flags = pcmk__ar_ordered;
	pcmk_action_t *probe = NULL;
	pcmk_node_t *allowed = NULL;
	pcmk_resource_t *top = uber_parent(rsc);
	const char *reason = NULL;

	CRM_ASSERT((rsc != NULL) && (node != NULL));

	if (!pcmk_is_set(rsc->private->scheduler->flags,
	pcmk_sched_probe_resources)) {
	reason = "start-up probes are disabled";
	goto no_probe;
	}

	if (pcmk__is_pacemaker_remote_node(node)) {
	const char *class = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_none)) {
	reason = "Pacemaker Remote nodes cannot run stonith agents";
	goto no_probe;

	} else if (pcmk__is_guest_or_bundle_node(node)
	&& pe__resource_contains_guest_node(rsc->private->scheduler,
	rsc)) {
	reason = "guest nodes cannot run resources containing guest nodes";
	goto no_probe;

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
	reason = "Pacemaker Remote nodes cannot host remote connections";
	goto no_probe;
	}
	}

	// If this is a collective resource, probes are created for its children
	if (rsc->private->children != NULL) {
	return pcmk__probe_resource_list(rsc->private->children, node);
	}

	if ((rsc->private->launcher != NULL)
	&& !pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
	reason = "resource is inside a container";
	goto no_probe;

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
	reason = "resource is orphaned";
	goto no_probe;

	} else if (g_hash_table_lookup(rsc->private->probed_nodes,
	node->private->id) != NULL) {
	reason = "resource state is already known";
	goto no_probe;
	}

	allowed = g_hash_table_lookup(rsc->private->allowed_nodes,
	node->private->id);

	if (pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)
	\|\| pcmk_is_set(top->flags, pcmk__rsc_exclusive_probes)) {
	// Exclusive discovery is enabled ...

	if (allowed == NULL) {
	// ... but this node is not allowed to run the resource
	reason = "resource has exclusive discovery but is not allowed "
	"on node";
	goto no_probe;

	- } else if (allowed->rsc_discover_mode != pcmk_probe_exclusive) {
	+ } else if (allowed->assign->probe_mode != pcmk__probe_exclusive) {
	// ... but no constraint marks this node for discovery of resource
	reason = "resource has exclusive discovery but is not enabled "
	"on node";
	goto no_probe;
	}
	}

	if (allowed == NULL) {
	allowed = node;
	}
	- if (allowed->rsc_discover_mode == pcmk_probe_never) {
	+ if (allowed->assign->probe_mode == pcmk__probe_never) {
	reason = "node has discovery disabled";
	goto no_probe;
	}

	if (pcmk__is_guest_or_bundle_node(node)) {
	pcmk_resource_t *guest = node->private->remote->private->launcher;

	if (guest->private->orig_role == pcmk_role_stopped) {
	// The guest is stopped, so we know no resource is active there
	reason = "node's guest is stopped";
	probe_then_start(guest, top);
	goto no_probe;

	} else if (guest_resource_will_stop(node)) {
	reason = "node's guest will stop";

	// Order resource start after guest stop (in case it's restarting)
	pcmk__new_ordering(guest,
	pcmk__op_key(guest->id, PCMK_ACTION_STOP, 0),
	NULL, top,
	pcmk__op_key(top->id, PCMK_ACTION_START, 0),
	NULL, pcmk__ar_ordered, rsc->private->scheduler);
	goto no_probe;
	}
	}

	// We've eliminated all cases where a probe is not needed, so now it is
	probe = probe_action(rsc, node);

	/* Below, we will order the probe relative to start or reload. If this is a
	* clone instance, the start or reload is for the entire clone rather than
	* just the instance. Otherwise, the start or reload is for the resource
	* itself.
	*/
	if (!pcmk__is_clone(top)) {
	top = rsc;
	}

	/* Prevent a start if the resource can't be probed, but don't cause the
	* resource or entire clone to stop if already active.
	*/
	if (!pcmk_is_set(probe->flags, pcmk_action_runnable)
	&& (top->private->active_nodes == NULL)) {
	pcmk__set_relation_flags(flags, pcmk__ar_unrunnable_first_blocks);
	}

	// Start or reload after probing the resource
	pcmk__new_ordering(rsc, NULL, probe,
	top, pcmk__op_key(top->id, PCMK_ACTION_START, 0), NULL,
	flags, rsc->private->scheduler);
	pcmk__new_ordering(rsc, NULL, probe, top, reload_key(rsc), NULL,
	pcmk__ar_ordered, rsc->private->scheduler);

	return true;

	no_probe:
	pcmk__rsc_trace(rsc,
	"Skipping probe for %s on %s because %s",
	rsc->id, node->private->id, reason);
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a probe should be ordered before another action
	*
	* \param[in] probe Probe action to check
	* \param[in] then Other action to check
	*
	* \return true if \p probe should be ordered before \p then, otherwise false
	*/
	static bool
	probe_needed_before_action(const pcmk_action_t *probe,
	const pcmk_action_t *then)
	{
	// Probes on a node are performed after unfencing it, not before
	if (pcmk__str_eq(then->task, PCMK_ACTION_STONITH, pcmk__str_none)
	&& pcmk__same_node(probe->node, then->node)) {
	const char *op = g_hash_table_lookup(then->meta,
	PCMK__META_STONITH_ACTION);

	if (pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	return false;
	}
	}

	// Probes should be done on a node before shutting it down
	if (pcmk__str_eq(then->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)
	&& (probe->node != NULL) && (then->node != NULL)
	&& !pcmk__same_node(probe->node, then->node)) {
	return false;
	}

	// Otherwise probes should always be done before any other action
	return true;
	}

	/*!
	* \internal
	* \brief Add implicit "probe then X" orderings for "stop then X" orderings
	*
	* If the state of a resource is not known yet, a probe will be scheduled,
	* expecting a "not running" result. If the probe fails, a stop will not be
	* scheduled until the next transition. Thus, if there are ordering constraints
	* like "stop this resource then do something else that's not for the same
	* resource", add implicit "probe this resource then do something" equivalents
	* so the relation is upheld until we know whether a stop is needed.
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	add_probe_orderings_for_stops(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->ordering_constraints; iter != NULL;
	iter = iter->next) {

	pcmk__action_relation_t *order = iter->data;
	uint32_t order_flags = pcmk__ar_ordered;
	GList *probes = NULL;
	GList *then_actions = NULL;
	pcmk_action_t *first = NULL;
	pcmk_action_t *then = NULL;

	// Skip disabled orderings
	if (order->flags == pcmk__ar_none) {
	continue;
	}

	// Skip non-resource orderings, and orderings for the same resource
	if ((order->rsc1 == NULL) \|\| (order->rsc1 == order->rsc2)) {
	continue;
	}

	// Skip invalid orderings (shouldn't be possible)
	first = order->action1;
	then = order->action2;
	if (((first == NULL) && (order->task1 == NULL))
	\|\| ((then == NULL) && (order->task2 == NULL))) {
	continue;
	}

	// Skip orderings for first actions other than stop
	if ((first != NULL) && !pcmk__str_eq(first->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((first == NULL)
	&& !pcmk__ends_with(order->task1,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}

	/* Do not imply a probe ordering for a resource inside of a stopping
	* launcher. Otherwise, it might introduce a transition loop, since a
	* probe could be scheduled after the launcher starts again.
	*/
	if ((order->rsc2 != NULL)
	&& (order->rsc1->private->launcher == order->rsc2)) {

	if ((then != NULL) && pcmk__str_eq(then->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((then == NULL)
	&& pcmk__ends_with(order->task2,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}
	}

	// Preserve certain order options for future filtering
	if (pcmk_is_set(order->flags, pcmk__ar_if_first_unmigratable)) {
	pcmk__set_relation_flags(order_flags,
	pcmk__ar_if_first_unmigratable);
	}
	if (pcmk_is_set(order->flags, pcmk__ar_if_on_same_node)) {
	pcmk__set_relation_flags(order_flags, pcmk__ar_if_on_same_node);
	}

	// Preserve certain order types for future filtering
	if ((order->flags == pcmk__ar_if_required_on_same_node)
	\|\| (order->flags == pcmk__ar_if_on_same_node_or_target)) {
	order_flags = order->flags;
	}

	// List all scheduled probes for the first resource
	probes = pe__resource_actions(order->rsc1, NULL, PCMK_ACTION_MONITOR,
	FALSE);
	if (probes == NULL) { // There aren't any
	continue;
	}

	// List all relevant "then" actions
	if (then != NULL) {
	then_actions = g_list_prepend(NULL, then);

	} else if (order->rsc2 != NULL) {
	then_actions = find_actions(order->rsc2->private->actions,
	order->task2, NULL);
	if (then_actions == NULL) { // There aren't any
	g_list_free(probes);
	continue;
	}
	}

	crm_trace("Implying 'probe then' orderings for '%s then %s' "
	"(id=%d, type=%.6x)",
	((first == NULL)? order->task1 : first->uuid),
	((then == NULL)? order->task2 : then->uuid),
	order->id, order->flags);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	for (GList *then_iter = then_actions; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk_action_t then = (pcmk_action_t ) then_iter->data;

	if (probe_needed_before_action(probe, then)) {
	order_actions(probe, then, order_flags);
	}
	}
	}

	g_list_free(then_actions);
	g_list_free(probes);
	}
	}

	/*!
	* \internal
	* \brief Add necessary orderings between probe and starts of clone instances
	*
	* , in additon to the ordering with the parent resource added upon creating
	* the probe.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action wrapper in the ordering
	*/
	static void
	add_start_orderings_for_probe(pcmk_action_t *probe,
	pcmk__related_action_t *after)
	{
	uint32_t flags = pcmk__ar_ordered\|pcmk__ar_unrunnable_first_blocks;

	/* Although the ordering between the probe of the clone instance and the
	* start of its parent has been added in pcmk__probe_rsc_on_node(), we
	* avoided enforcing `pcmk__ar_unrunnable_first_blocks` order type for that
	* as long as any of the clone instances are running to prevent them from
	* being unexpectedly stopped.
	*
	* On the other hand, we still need to prevent any inactive instances from
	* starting unless the probe is runnable so that we don't risk starting too
	* many instances before we know the state on all nodes.
	*/
	if ((after->action->rsc->private->variant <= pcmk__rsc_variant_group)
	\|\| pcmk_is_set(probe->flags, pcmk_action_runnable)
	// The order type is already enforced for its parent.
	\|\| pcmk_is_set(after->type, pcmk__ar_unrunnable_first_blocks)
	\|\| (pe__const_top_resource(probe->rsc, false) != after->action->rsc)
	\|\| !pcmk__str_eq(after->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	return;
	}

	crm_trace("Adding probe start orderings for 'unrunnable %s@%s "
	"then instances of %s@%s'",
	probe->uuid, pcmk__node_name(probe->node),
	after->action->uuid, pcmk__node_name(after->action->node));

	for (GList *then_iter = after->action->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	if ((then->action->rsc->private->active_nodes != NULL)
	\|\| (pe__const_top_resource(then->action->rsc, false)
	!= after->action->rsc)
	\|\| !pcmk__str_eq(then->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	continue;
	}

	crm_trace("Adding probe start ordering for 'unrunnable %s@%s "
	"then %s@%s' (type=%#.6x)",
	probe->uuid, pcmk__node_name(probe->node),
	then->action->uuid, pcmk__node_name(then->action->node),
	flags);

	/* Prevent the instance from starting if the instance can't, but don't
	* cause any other intances to stop if already active.
	*/
	order_actions(probe, then->action, flags);
	}

	return;
	}

	/*!
	* \internal
	* \brief Order probes before restarts and re-promotes
	*
	* If a given ordering is a "probe then start" or "probe then promote" ordering,
	* add an implicit "probe then stop/demote" ordering in case the action is part
	* of a restart/re-promote, and do the same recursively for all actions ordered
	* after the "then" action.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action in the ordering
	*/
	static void
	add_restart_orderings_for_probe(pcmk_action_t probe, pcmk_action_t after)
	{
	GList *iter = NULL;
	bool interleave = false;
	pcmk_resource_t *compatible_rsc = NULL;

	// Validate that this is a resource probe followed by some action
	if ((after == NULL) \|\| (probe == NULL) \|\| !pcmk__is_primitive(probe->rsc)
	\|\| !pcmk__str_eq(probe->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	return;
	}

	// Avoid running into any possible loop
	if (pcmk_is_set(after->flags, pcmk_action_detect_loop)) {
	return;
	}
	pcmk__set_action_flags(after, pcmk_action_detect_loop);

	crm_trace("Adding probe restart orderings for '%s@%s then %s@%s'",
	probe->uuid, pcmk__node_name(probe->node),
	after->uuid, pcmk__node_name(after->node));

	/* Add restart orderings if "then" is for a different primitive.
	* Orderings for collective resources will be added later.
	*/
	if (pcmk__is_primitive(after->rsc) && (probe->rsc != after->rsc)) {

	GList *then_actions = NULL;

	if (pcmk__str_eq(after->task, PCMK_ACTION_START, pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_STOP, FALSE);

	} else if (pcmk__str_eq(after->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_DEMOTE, FALSE);
	}

	for (iter = then_actions; iter != NULL; iter = iter->next) {
	pcmk_action_t then = (pcmk_action_t ) iter->data;

	// Skip pseudo-actions (for example, those implied by fencing)
	if (!pcmk_is_set(then->flags, pcmk_action_pseudo)) {
	order_actions(probe, then, pcmk__ar_ordered);
	}
	}
	g_list_free(then_actions);
	}

	/* Detect whether "then" is an interleaved clone action. For these, we want
	* to add orderings only for the relevant instance.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->private->variant > pcmk__rsc_variant_group)) {

	interleave = crm_is_true(g_hash_table_lookup(after->rsc->private->meta,
	PCMK_META_INTERLEAVE));
	if (interleave) {
	compatible_rsc = pcmk__find_compatible_instance(probe->rsc,
	after->rsc,
	pcmk_role_unknown,
	false);
	}
	}

	/* Now recursively do the same for all actions ordered after "then". This
	* also handles collective resources since the collective action will be
	* ordered before its individual instances' actions.
	*/
	for (iter = after->actions_after; iter != NULL; iter = iter->next) {
	pcmk__related_action_t *after_wrapper = iter->data;
	const pcmk_resource_t *chained_rsc = NULL;

	/* pcmk__ar_first_implies_then is the reason why a required A.start
	* implies/enforces B.start to be required too, which is the cause of
	* B.restart/re-promote.
	*
	* Not sure about pcmk__ar_first_implies_same_node_then though. It's now
	* only used for unfencing case, which tends to introduce transition
	* loops...
	*/
	if (!pcmk_is_set(after_wrapper->type, pcmk__ar_first_implies_then)) {
	/* The order type between a group/clone and its child such as
	* B.start-> B_child.start is:
	* pcmk__ar_then_implies_first_graphed
	* \|pcmk__ar_unrunnable_first_blocks
	*
	* Proceed through the ordering chain and build dependencies with
	* its children.
	*/
	if ((after->rsc == NULL)
	\|\| (after->rsc->private->variant < pcmk__rsc_variant_group)
	\|\| (probe->rsc->private->parent == after->rsc)
	\|\| (after_wrapper->action->rsc == NULL)) {
	continue;
	}
	chained_rsc = after_wrapper->action->rsc;

	if ((chained_rsc->private->variant > pcmk__rsc_variant_group)
	\|\| (after->rsc != chained_rsc->private->parent)) {
	continue;
	}

	/* Proceed to the children of a group or a non-interleaved clone.
	* For an interleaved clone, proceed only to the relevant child.
	*/
	if ((after->rsc->private->variant > pcmk__rsc_variant_group)
	&& interleave
	&& ((compatible_rsc == NULL)
	\|\| (compatible_rsc != chained_rsc))) {
	continue;
	}
	}

	crm_trace("Recursively adding probe restart orderings for "
	"'%s@%s then %s@%s' (type=%#.6x)",
	after->uuid, pcmk__node_name(after->node),
	after_wrapper->action->uuid,
	pcmk__node_name(after_wrapper->action->node),
	after_wrapper->type);

	add_restart_orderings_for_probe(probe, after_wrapper->action);
	}
	}

	/*!
	* \internal
	* \brief Clear the tracking flag on all scheduled actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	clear_actions_tracking_flag(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t *action = iter->data;

	pcmk__clear_action_flags(action, pcmk_action_detect_loop);
	}
	}

	/*!
	* \internal
	* \brief Add start and restart orderings for probes scheduled for a resource
	*
	* \param[in,out] data Resource whose probes should be ordered
	* \param[in] user_data Unused
	*/
	static void
	add_start_restart_orderings_for_rsc(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	GList *probes = NULL;

	// For collective resources, order each instance recursively
	if (!pcmk__is_primitive(rsc)) {
	g_list_foreach(rsc->private->children,
	add_start_restart_orderings_for_rsc, NULL);
	return;
	}

	// Find all probes for given resource
	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	// Add probe restart orderings for each probe found
	for (GList *iter = probes; iter != NULL; iter = iter->next) {
	pcmk_action_t probe = (pcmk_action_t ) iter->data;

	for (GList *then_iter = probe->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	add_start_orderings_for_probe(probe, then);
	add_restart_orderings_for_probe(probe, then->action);
	clear_actions_tracking_flag(rsc->private->scheduler);
	}
	}

	g_list_free(probes);
	}

	/*!
	* \internal
	* \brief Add "A then probe B" orderings for "A then B" orderings
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This function is currently disabled (see next comment).
	*/
	static void
	order_then_probes(pcmk_scheduler_t *scheduler)
	{
	#if 0
	/* Given an ordering "A then B", we would prefer to wait for A to be started
	* before probing B.
	*
	* For example, if A is a filesystem which B can't even run without, it
	* would be helpful if the author of B's agent could assume that A is
	* running before B.monitor will be called.
	*
	* However, we can't _only_ probe after A is running, otherwise we wouldn't
	* detect the state of B if A could not be started. We can't even do an
	* opportunistic version of this, because B may be moving:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> B.start
	*
	* and if we add B.stop -> A.stop here, we get a loop:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> A.stop
	*
	* We could kill the "B.probe -> B.stop" dependency, but that could mean
	* stopping B "too" soon, because B.start must wait for the probe, and
	* we don't want to stop B if we can't start it.
	*
	* We could add the ordering only if A is an anonymous clone with
	* clone-max == node-max (since we'll never be moving it). However, we could
	* still be stopping one instance at the same time as starting another.
	*
	* The complexity of checking for allowed conditions combined with the ever
	* narrowing use case suggests that this code should remain disabled until
	* someone gets smarter.
	*/
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	pcmk_action_t *start = NULL;
	GList *actions = NULL;
	GList *probes = NULL;

	actions = pe__resource_actions(rsc, NULL, PCMK_ACTION_START, FALSE);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if (start == NULL) {
	crm_debug("No start action for %s", rsc->id);
	continue;
	}

	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	for (actions = start->actions_before; actions != NULL;
	actions = actions->next) {

	pcmk__related_action_t *before = actions->data;

	pcmk_action_t *first = before->action;
	pcmk_resource_t *first_rsc = first->rsc;

	if (first->required_runnable_before) {
	for (GList *clone_actions = first->actions_before;
	clone_actions != NULL;
	clone_actions = clone_actions->next) {

	before = clone_actions->data;

	crm_trace("Testing '%s then %s' for %s",
	first->uuid, before->action->uuid, start->uuid);

	CRM_ASSERT(before->action->rsc != NULL);
	first_rsc = before->action->rsc;
	break;
	}

	} else if (!pcmk__str_eq(first->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if (first_rsc == NULL) {
	continue;

	} else if (pe__const_top_resource(first_rsc, false)
	== pe__const_top_resource(start->rsc, false)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if (!pcmk__is_clone(pe__const_top_resource(first_rsc,
	false))) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	crm_debug("Applying %s before %s", first->uuid, start->uuid);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	crm_debug("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pcmk__ar_ordered);
	}
	}
	}
	#endif
	}

	void
	pcmk__order_probes(pcmk_scheduler_t *scheduler)
	{
	// Add orderings for "probe then X"
	g_list_foreach(scheduler->resources, add_start_restart_orderings_for_rsc,
	NULL);
	add_probe_orderings_for_stops(scheduler);

	order_then_probes(scheduler);
	}

	/*!
	* \internal
	* \brief Schedule any probes needed
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This may also schedule fencing of failed remote nodes.
	*/
	void
	pcmk__schedule_probes(pcmk_scheduler_t *scheduler)
	{
	// Schedule probes on each node in the cluster as needed
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	if (!node->details->online) { // Don't probe offline nodes
	if (pcmk__is_failed_remote_node(node)) {
	pe_fence_node(scheduler, node,
	"the connection is unrecoverable", FALSE);
	}
	continue;
	}

	if (node->details->unclean) { // Don't probe nodes that need fencing
	continue;
	}

	if (!pcmk_is_set(node->private->flags, pcmk__node_probes_allowed)) {
	// The user requested that probes not be done on this node
	continue;
	}

	// Probe each resource in the cluster on this node, as needed
	pcmk__probe_resource_list(scheduler->resources, node);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_promotable.c b/lib/pacemaker/pcmk_sched_promotable.c
	index afe5b14b04..5cd6d1cef0 100644
	--- a/lib/pacemaker/pcmk_sched_promotable.c
	+++ b/lib/pacemaker/pcmk_sched_promotable.c
	@@ -1,1376 +1,1378 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add implicit promotion ordering for a promotable instance
	*
	* \param[in,out] clone Clone resource
	* \param[in,out] child Instance of \p clone being ordered
	* \param[in,out] last Previous instance ordered (NULL if \p child is first)
	*/
	static void
	order_instance_promotion(pcmk_resource_t clone, pcmk_resource_t child,
	pcmk_resource_t *last)
	{
	// "Promote clone" -> promote instance -> "clone promoted"
	pcmk__order_resource_actions(clone, PCMK_ACTION_PROMOTE,
	child, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(child, PCMK_ACTION_PROMOTE,
	clone, PCMK_ACTION_PROMOTED,
	pcmk__ar_ordered);

	// If clone is ordered, order this instance relative to last
	if ((last != NULL) && pe__clone_is_ordered(clone)) {
	pcmk__order_resource_actions(last, PCMK_ACTION_PROMOTE,
	child, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Add implicit demotion ordering for a promotable instance
	*
	* \param[in,out] clone Clone resource
	* \param[in,out] child Instance of \p clone being ordered
	* \param[in] last Previous instance ordered (NULL if \p child is first)
	*/
	static void
	order_instance_demotion(pcmk_resource_t clone, pcmk_resource_t child,
	pcmk_resource_t *last)
	{
	// "Demote clone" -> demote instance -> "clone demoted"
	pcmk__order_resource_actions(clone, PCMK_ACTION_DEMOTE, child,
	PCMK_ACTION_DEMOTE,
	pcmk__ar_then_implies_first_graphed);
	pcmk__order_resource_actions(child, PCMK_ACTION_DEMOTE,
	clone, PCMK_ACTION_DEMOTED,
	pcmk__ar_first_implies_then_graphed);

	// If clone is ordered, order this instance relative to last
	if ((last != NULL) && pe__clone_is_ordered(clone)) {
	pcmk__order_resource_actions(child, PCMK_ACTION_DEMOTE, last,
	PCMK_ACTION_DEMOTE, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Check whether an instance will be promoted or demoted
	*
	* \param[in] rsc Instance to check
	* \param[out] demoting If \p rsc will be demoted, this will be set to true
	* \param[out] promoting If \p rsc will be promoted, this will be set to true
	*/
	static void
	check_for_role_change(const pcmk_resource_t rsc, bool demoting,
	bool *promoting)
	{
	const GList *iter = NULL;

	// If this is a cloned group, check group members recursively
	if (rsc->private->children != NULL) {
	for (iter = rsc->private->children; iter != NULL; iter = iter->next) {
	check_for_role_change((const pcmk_resource_t *) iter->data,
	demoting, promoting);
	}
	return;
	}

	for (iter = rsc->private->actions; iter != NULL; iter = iter->next) {
	const pcmk_action_t action = (const pcmk_action_t ) iter->data;

	if (promoting && demoting) {
	return;

	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	continue;

	} else if (pcmk__str_eq(PCMK_ACTION_DEMOTE, action->task,
	pcmk__str_none)) {
	*demoting = true;

	} else if (pcmk__str_eq(PCMK_ACTION_PROMOTE, action->task,
	pcmk__str_none)) {
	*promoting = true;
	}
	}
	}

	/*!
	* \internal
	* \brief Add promoted-role location constraint scores to an instance's priority
	*
	* Adjust a promotable clone instance's promotion priority by the scores of any
	* location constraints in a list that are both limited to the promoted role and
	* for the node where the instance will be placed.
	*
	* \param[in,out] child Promotable clone instance
	* \param[in] location_constraints List of location constraints to apply
	* \param[in] chosen Node where \p child will be placed
	*/
	static void
	apply_promoted_locations(pcmk_resource_t *child,
	const GList *location_constraints,
	const pcmk_node_t *chosen)
	{
	for (const GList *iter = location_constraints; iter; iter = iter->next) {
	const pcmk__location_t *location = iter->data;
	const pcmk_node_t *constraint_node = NULL;

	if (location->role_filter == pcmk_role_promoted) {
	constraint_node = pe_find_node_id(location->nodes,
	chosen->private->id);
	}
	if (constraint_node != NULL) {
	int new_priority = pcmk__add_scores(child->private->priority,
	- constraint_node->weight);
	+ constraint_node->assign->score);

	pcmk__rsc_trace(child,
	"Applying location %s to %s promotion priority on "
	"%s: %s + %s = %s",
	location->id, child->id,
	pcmk__node_name(constraint_node),
	pcmk_readable_score(child->private->priority),
	- pcmk_readable_score(constraint_node->weight),
	+ pcmk_readable_score(constraint_node->assign->score),
	pcmk_readable_score(new_priority));
	child->private->priority = new_priority;
	}
	}
	}

	/*!
	* \internal
	* \brief Get the node that an instance will be promoted on
	*
	* \param[in] rsc Promotable clone instance to check
	*
	* \return Node that \p rsc will be promoted on, or NULL if none
	*/
	static pcmk_node_t *
	node_to_be_promoted_on(const pcmk_resource_t *rsc)
	{
	pcmk_node_t *node = NULL;
	pcmk_node_t *local_node = NULL;
	const pcmk_resource_t *parent = NULL;

	// If this is a cloned group, bail if any group member can't be promoted
	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	if (node_to_be_promoted_on(child) == NULL) {
	pcmk__rsc_trace(rsc,
	"%s can't be promoted because member %s can't",
	rsc->id, child->id);
	return NULL;
	}
	}

	node = rsc->private->fns->location(rsc, NULL, FALSE);
	if (node == NULL) {
	pcmk__rsc_trace(rsc, "%s can't be promoted because it won't be active",
	rsc->id);
	return NULL;

	} else if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	if (rsc->private->fns->state(rsc, TRUE) == pcmk_role_promoted) {
	crm_notice("Unmanaged instance %s will be left promoted on %s",
	rsc->id, pcmk__node_name(node));
	} else {
	pcmk__rsc_trace(rsc, "%s can't be promoted because it is unmanaged",
	rsc->id);
	return NULL;
	}

	} else if (rsc->private->priority < 0) {
	pcmk__rsc_trace(rsc,
	"%s can't be promoted because its promotion priority "
	"%d is negative",
	rsc->id, rsc->private->priority);
	return NULL;

	} else if (!pcmk__node_available(node, false, true)) {
	pcmk__rsc_trace(rsc,
	"%s can't be promoted because %s can't run resources",
	rsc->id, pcmk__node_name(node));
	return NULL;
	}

	parent = pe__const_top_resource(rsc, false);
	local_node = g_hash_table_lookup(parent->private->allowed_nodes,
	node->private->id);

	if (local_node == NULL) {
	/* It should not be possible for the scheduler to have assigned the
	* instance to a node where its parent is not allowed, but it's good to
	* have a fail-safe.
	*/
	if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	pcmk__sched_err("%s can't be promoted because %s is not allowed "
	"on %s (scheduler bug?)",
	rsc->id, parent->id, pcmk__node_name(node));
	} // else the instance is unmanaged and already promoted
	return NULL;

	- } else if ((local_node->count >= pe__clone_promoted_node_max(parent))
	+ } else if ((local_node->assign->count >= pe__clone_promoted_node_max(parent))
	&& pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	pcmk__rsc_trace(rsc,
	"%s can't be promoted because %s has "
	"maximum promoted instances already",
	rsc->id, pcmk__node_name(node));
	return NULL;
	}

	return local_node;
	}

	/*!
	* \internal
	* \brief Compare two promotable clone instances by promotion priority
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a has higher promotion priority,
	* a positive number if \p b has higher promotion priority,
	* or 0 if promotion priorities are equal
	*/
	static gint
	cmp_promotable_instance(gconstpointer a, gconstpointer b)
	{
	const pcmk_resource_t rsc1 = (const pcmk_resource_t ) a;
	const pcmk_resource_t rsc2 = (const pcmk_resource_t ) b;

	enum rsc_role_e role1 = pcmk_role_unknown;
	enum rsc_role_e role2 = pcmk_role_unknown;

	CRM_ASSERT((rsc1 != NULL) && (rsc2 != NULL));

	// Check promotion priority set by pcmk__set_instance_roles()
	if (rsc1->private->promotion_priority > rsc2->private->promotion_priority) {
	pcmk__rsc_trace(rsc1,
	"%s has higher promotion priority (%s) than %s (%d)",
	rsc1->id,
	pcmk_readable_score(rsc1->private->promotion_priority),
	rsc2->id, rsc2->private->promotion_priority);
	return -1;
	}

	if (rsc1->private->promotion_priority < rsc2->private->promotion_priority) {
	pcmk__rsc_trace(rsc1,
	"%s has lower promotion priority (%s) than %s (%d)",
	rsc1->id,
	pcmk_readable_score(rsc1->private->promotion_priority),
	rsc2->id, rsc2->private->promotion_priority);
	return 1;
	}

	// If those are the same, prefer instance whose current role is higher
	role1 = rsc1->private->fns->state(rsc1, TRUE);
	role2 = rsc2->private->fns->state(rsc2, TRUE);
	if (role1 > role2) {
	pcmk__rsc_trace(rsc1,
	"%s has higher promotion priority than %s "
	"(higher current role)",
	rsc1->id, rsc2->id);
	return -1;
	} else if (role1 < role2) {
	pcmk__rsc_trace(rsc1,
	"%s has lower promotion priority than %s "
	"(lower current role)",
	rsc1->id, rsc2->id);
	return 1;
	}

	// Finally, do normal clone instance sorting
	return pcmk__cmp_instance(a, b);
	}

	/*!
	* \internal
	* \brief Add promotable clone instance's promotion priority to its node's score
	*
	* Add a promotable clone instance's promotion priority (which sums its
	* promotion preferences and scores of relevant location constraints for the
	* promoted role) to the node score of the instance's assigned node.
	*
	* \param[in] data Promotable clone instance
	* \param[in,out] user_data Clone parent of \p data
	*/
	static void
	add_promotion_priority_to_node_score(gpointer data, gpointer user_data)
	{
	const pcmk_resource_t child = (const pcmk_resource_t ) data;
	pcmk_resource_t clone = (pcmk_resource_t ) user_data;

	pcmk_node_t *node = NULL;
	const pcmk_node_t *chosen = NULL;
	const int promotion_priority = child->private->promotion_priority;

	if (promotion_priority < 0) {
	pcmk__rsc_trace(clone,
	"Not adding promotion priority of %s: negative (%s)",
	child->id, pcmk_readable_score(promotion_priority));
	return;
	}

	chosen = child->private->fns->location(child, NULL, FALSE);
	if (chosen == NULL) {
	pcmk__rsc_trace(clone, "Not adding promotion priority of %s: inactive",
	child->id);
	return;
	}

	node = g_hash_table_lookup(clone->private->allowed_nodes,
	chosen->private->id);
	CRM_ASSERT(node != NULL);

	- node->weight = pcmk__add_scores(promotion_priority, node->weight);
	+ node->assign->score = pcmk__add_scores(promotion_priority,
	+ node->assign->score);
	pcmk__rsc_trace(clone,
	"Added cumulative priority of %s (%s) to score on %s "
	"(now %d)",
	child->id, pcmk_readable_score(promotion_priority),
	- pcmk__node_name(node), node->weight);
	+ pcmk__node_name(node), node->assign->score);
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent's node scores if for promoted role
	*
	* \param[in,out] data Colocation constraint to apply
	* \param[in,out] user_data Promotable clone that is constraint's dependent
	*/
	static void
	apply_coloc_to_dependent(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t *colocation = data;
	pcmk_resource_t *clone = user_data;
	pcmk_resource_t *primary = colocation->primary;
	uint32_t flags = pcmk__coloc_select_default;
	float factor = colocation->score / (float) PCMK_SCORE_INFINITY;

	if (colocation->dependent_role != pcmk_role_promoted) {
	return;
	}
	if (colocation->score < PCMK_SCORE_INFINITY) {
	flags = pcmk__coloc_select_active;
	}
	pcmk__rsc_trace(clone, "Applying colocation %s (promoted %s with %s) @%s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id,
	pcmk_readable_score(colocation->score));
	primary->private->cmds->add_colocated_node_scores(primary, clone, clone->id,
	&(clone->private->allowed_nodes),
	colocation, factor,
	flags);
	}

	/*!
	* \internal
	* \brief Apply colocation to primary's node scores if for promoted role
	*
	* \param[in,out] data Colocation constraint to apply
	* \param[in,out] user_data Promotable clone that is constraint's primary
	*/
	static void
	apply_coloc_to_primary(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t *colocation = data;
	pcmk_resource_t *clone = user_data;
	pcmk_resource_t *dependent = colocation->dependent;
	const float factor = colocation->score / (float) PCMK_SCORE_INFINITY;
	const uint32_t flags = pcmk__coloc_select_active
	\|pcmk__coloc_select_nonnegative;

	if ((colocation->primary_role != pcmk_role_promoted)
	\|\| !pcmk__colocation_has_influence(colocation, NULL)) {
	return;
	}

	pcmk__rsc_trace(clone, "Applying colocation %s (%s with promoted %s) @%s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id,
	pcmk_readable_score(colocation->score));
	dependent->private->cmds->add_colocated_node_scores(dependent, clone,
	clone->id,
	&(clone->private->allowed_nodes),
	colocation, factor,
	flags);
	}

	/*!
	* \internal
	* \brief Set clone instance's promotion priority to its node's score
	*
	* \param[in,out] data Promotable clone instance
	* \param[in] user_data Parent clone of \p data
	*/
	static void
	set_promotion_priority_to_node_score(gpointer data, gpointer user_data)
	{
	pcmk_resource_t child = (pcmk_resource_t ) data;
	const pcmk_resource_t clone = (const pcmk_resource_t ) user_data;

	pcmk_node_t *chosen = child->private->fns->location(child, NULL, FALSE);

	if (!pcmk_is_set(child->flags, pcmk__rsc_managed)
	&& (child->private->next_role == pcmk_role_promoted)) {
	child->private->promotion_priority = PCMK_SCORE_INFINITY;
	pcmk__rsc_trace(clone,
	"Final promotion priority for %s is %s "
	"(unmanaged promoted)",
	child->id, pcmk_readable_score(PCMK_SCORE_INFINITY));

	} else if (chosen == NULL) {
	child->private->promotion_priority = -PCMK_SCORE_INFINITY;
	pcmk__rsc_trace(clone,
	"Final promotion priority for %s is %s "
	"(will not be active)",
	child->id, pcmk_readable_score(-PCMK_SCORE_INFINITY));

	} else if (child->private->promotion_priority < 0) {
	pcmk__rsc_trace(clone,
	"Final promotion priority for %s is %s "
	"(ignoring node score)",
	child->id,
	pcmk_readable_score(child->private->promotion_priority));

	} else {
	const pcmk_node_t *node = NULL;

	node = g_hash_table_lookup(clone->private->allowed_nodes,
	chosen->private->id);

	CRM_ASSERT(node != NULL);
	- child->private->promotion_priority = node->weight;
	+ child->private->promotion_priority = node->assign->score;
	pcmk__rsc_trace(clone,
	"Adding scores for %s: "
	"final promotion priority for %s is %s",
	clone->id, child->id,
	pcmk_readable_score(child->private->promotion_priority));
	}
	}

	/*!
	* \internal
	* \brief Sort a promotable clone's instances by descending promotion priority
	*
	* \param[in,out] clone Promotable clone to sort
	*/
	static void
	sort_promotable_instances(pcmk_resource_t *clone)
	{
	GList *colocations = NULL;

	if (pe__set_clone_flag(clone, pcmk__clone_promotion_constrained)
	== pcmk_rc_already) {
	return;
	}
	pcmk__set_rsc_flags(clone, pcmk__rsc_updating_nodes);

	for (GList *iter = clone->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	pcmk__rsc_trace(clone,
	"Adding scores for %s: "
	"initial promotion priority for %s is %s",
	clone->id, child->id,
	pcmk_readable_score(child->private->promotion_priority));
	}
	pe__show_node_scores(true, clone, "Before", clone->private->allowed_nodes,
	clone->private->scheduler);

	g_list_foreach(clone->private->children,
	add_promotion_priority_to_node_score, clone);

	colocations = pcmk__this_with_colocations(clone);
	g_list_foreach(colocations, apply_coloc_to_dependent, clone);
	g_list_free(colocations);

	colocations = pcmk__with_this_colocations(clone);
	g_list_foreach(colocations, apply_coloc_to_primary, clone);
	g_list_free(colocations);

	// Ban resource from all nodes if it needs a ticket but doesn't have it
	pcmk__require_promotion_tickets(clone);

	pe__show_node_scores(true, clone, "After", clone->private->allowed_nodes,
	clone->private->scheduler);

	// Reset promotion priorities to final node scores
	g_list_foreach(clone->private->children,
	set_promotion_priority_to_node_score, clone);

	// Finally, sort instances in descending order of promotion priority
	clone->private->children = g_list_sort(clone->private->children,
	cmp_promotable_instance);
	pcmk__clear_rsc_flags(clone, pcmk__rsc_updating_nodes);
	}

	/*!
	* \internal
	* \brief Find the active instance (if any) of an anonymous clone on a node
	*
	* \param[in] clone Anonymous clone to check
	* \param[in] id Instance ID (without instance number) to check
	* \param[in] node Node to check
	*
	* \return
	*/
	static pcmk_resource_t *
	find_active_anon_instance(const pcmk_resource_t clone, const char id,
	const pcmk_node_t *node)
	{
	for (GList *iter = clone->private->children; iter; iter = iter->next) {
	pcmk_resource_t *child = iter->data;
	pcmk_resource_t *active = NULL;

	// Use ->find_rsc() in case this is a cloned group
	active = clone->private->fns->find_rsc(child, id, node,
	pcmk_rsc_match_clone_only
	\|pcmk_rsc_match_current_node);
	if (active != NULL) {
	return active;
	}
	}
	return NULL;
	}

	/*
	* \brief Check whether an anonymous clone instance is known on a node
	*
	* \param[in] clone Anonymous clone to check
	* \param[in] id Instance ID (without instance number) to check
	* \param[in] node Node to check
	*
	* \return true if \p id instance of \p clone is known on \p node,
	* otherwise false
	*/
	static bool
	anonymous_known_on(const pcmk_resource_t clone, const char id,
	const pcmk_node_t *node)
	{
	for (GList *iter = clone->private->children; iter; iter = iter->next) {
	pcmk_resource_t *child = iter->data;

	/* Use ->find_rsc() because this might be a cloned group, and knowing
	* that other members of the group are known here implies nothing.
	*/
	child = clone->private->fns->find_rsc(child, id, NULL,
	pcmk_rsc_match_clone_only);
	CRM_LOG_ASSERT(child != NULL);
	if (child != NULL) {
	if (g_hash_table_lookup(child->private->probed_nodes,
	node->private->id)) {
	return true;
	}
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node is allowed to run a resource
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is allowed to run \p rsc, otherwise false
	*/
	static bool
	is_allowed(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk_node_t *allowed = g_hash_table_lookup(rsc->private->allowed_nodes,
	node->private->id);

	- return (allowed != NULL) && (allowed->weight >= 0);
	+ return (allowed != NULL) && (allowed->assign->score >= 0);
	}

	/*!
	* \brief Check whether a clone instance's promotion score should be considered
	*
	* \param[in] rsc Promotable clone instance to check
	* \param[in] node Node where score would be applied
	*
	* \return true if \p rsc's promotion score should be considered on \p node,
	* otherwise false
	*/
	static bool
	promotion_score_applies(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	char *id = clone_strip(rsc->id);
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
	pcmk_resource_t *active = NULL;
	const char *reason = "allowed";

	// Some checks apply only to anonymous clone instances
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {

	// If instance is active on the node, its score definitely applies
	active = find_active_anon_instance(parent, id, node);
	if (active == rsc) {
	reason = "active";
	goto check_allowed;
	}

	/* If no instance is active on this node, this instance's score will
	* count if it has been probed on this node.
	*/
	if ((active == NULL) && anonymous_known_on(parent, id, node)) {
	reason = "probed";
	goto check_allowed;
	}
	}

	/* If this clone's status is unknown on all nodes (e.g. cluster startup),
	* take all instances' scores into account, to make sure we use any
	* permanent promotion scores.
	*/
	if ((rsc->private->active_nodes == NULL)
	&& (g_hash_table_size(rsc->private->probed_nodes) == 0)) {
	reason = "none probed";
	goto check_allowed;
	}

	/* Otherwise, we've probed and/or started the resource somewhere, so
	* consider promotion scores on nodes where we know the status.
	*/
	if ((g_hash_table_lookup(rsc->private->probed_nodes,
	node->private->id) != NULL)
	\|\| (pe_find_node_id(rsc->private->active_nodes,
	node->private->id) != NULL)) {
	reason = "known";
	} else {
	pcmk__rsc_trace(rsc,
	"Ignoring %s promotion score (for %s) on %s: "
	"not probed",
	rsc->id, id, pcmk__node_name(node));
	free(id);
	return false;
	}

	check_allowed:
	if (is_allowed(rsc, node)) {
	pcmk__rsc_trace(rsc, "Counting %s promotion score (for %s) on %s: %s",
	rsc->id, id, pcmk__node_name(node), reason);
	free(id);
	return true;
	}

	pcmk__rsc_trace(rsc,
	"Ignoring %s promotion score (for %s) on %s: not allowed",
	rsc->id, id, pcmk__node_name(node));
	free(id);
	return false;
	}

	/*!
	* \internal
	* \brief Get the value of a promotion score node attribute
	*
	* \param[in] rsc Promotable clone instance to get promotion score for
	* \param[in] node Node to get promotion score for
	* \param[in] name Resource name to use in promotion score attribute name
	*
	* \return Value of promotion score node attribute for \p rsc on \p node
	*/
	static const char *
	promotion_attr_value(const pcmk_resource_t rsc, const pcmk_node_t node,
	const char *name)
	{
	char *attr_name = NULL;
	const char *attr_value = NULL;
	const char *target = NULL;
	enum pcmk__rsc_node node_type = pcmk__rsc_node_assigned;

	if (pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
	// Not assigned yet
	node_type = pcmk__rsc_node_current;
	}
	target = g_hash_table_lookup(rsc->private->meta,
	PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
	attr_name = pcmk_promotion_score_name(name);
	attr_value = pcmk__node_attr(node, attr_name, target, node_type);
	free(attr_name);
	return attr_value;
	}

	/*!
	* \internal
	* \brief Get the promotion score for a clone instance on a node
	*
	* \param[in] rsc Promotable clone instance to get score for
	* \param[in] node Node to get score for
	* \param[out] is_default If non-NULL, will be set true if no score available
	*
	* \return Promotion score for \p rsc on \p node (or 0 if none)
	*/
	static int
	promotion_score(const pcmk_resource_t rsc, const pcmk_node_t node,
	bool *is_default)
	{
	const char *name = NULL;
	const char *attr_value = NULL;

	if (is_default != NULL) {
	*is_default = true;
	}

	CRM_CHECK((rsc != NULL) && (node != NULL), return 0);

	/* If this is an instance of a cloned group, the promotion score is the sum
	* of all members' promotion scores.
	*/
	if (rsc->private->children != NULL) {
	int score = 0;

	for (const GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;
	bool child_default = false;
	int child_score = promotion_score(child, node, &child_default);

	if (!child_default && (is_default != NULL)) {
	*is_default = false;
	}
	score += child_score;
	}
	return score;
	}

	if (!promotion_score_applies(rsc, node)) {
	return 0;
	}

	/* For the promotion score attribute name, use the name the resource is
	* known as in resource history, since that's what crm_attribute --promotion
	* would have used.
	*/
	name = pcmk__s(rsc->private->history_id, rsc->id);

	attr_value = promotion_attr_value(rsc, node, name);
	if (attr_value != NULL) {
	pcmk__rsc_trace(rsc, "Promotion score for %s on %s = %s",
	name, pcmk__node_name(node),
	pcmk__s(attr_value, "(unset)"));
	} else if (!pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
	/* If we don't have any resource history yet, we won't have history_id.
	* In that case, for anonymous clones, try the resource name without
	* any instance number.
	*/
	char *rsc_name = clone_strip(rsc->id);

	if (strcmp(rsc->id, rsc_name) != 0) {
	attr_value = promotion_attr_value(rsc, node, rsc_name);
	pcmk__rsc_trace(rsc, "Promotion score for %s on %s (for %s) = %s",
	rsc_name, pcmk__node_name(node), rsc->id,
	pcmk__s(attr_value, "(unset)"));
	}
	free(rsc_name);
	}

	if (attr_value == NULL) {
	return 0;
	}

	if (is_default != NULL) {
	*is_default = false;
	}
	return char2score(attr_value);
	}

	/*!
	* \internal
	* \brief Include promotion scores in instances' node scores and priorities
	*
	* \param[in,out] rsc Promotable clone resource to update
	*/
	void
	pcmk__add_promotion_scores(pcmk_resource_t *rsc)
	{
	if (pe__set_clone_flag(rsc,
	pcmk__clone_promotion_added) == pcmk_rc_already) {
	return;
	}

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t child_rsc = (pcmk_resource_t ) iter->data;

	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	int score, new_score;

	g_hash_table_iter_init(&iter, child_rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (!pcmk__node_available(node, false, false)) {
	/* This node will never be promoted, so don't apply the
	* promotion score, as that may lead to clone shuffling.
	*/
	continue;
	}

	score = promotion_score(child_rsc, node, NULL);
	if (score > 0) {
	- new_score = pcmk__add_scores(node->weight, score);
	- if (new_score != node->weight) { // Could remain INFINITY
	- node->weight = new_score;
	+ new_score = pcmk__add_scores(node->assign->score, score);
	+ if (new_score != node->assign->score) { // Could remain INFINITY
	+ node->assign->score = new_score;
	pcmk__rsc_trace(rsc,
	"Added %s promotion priority (%s) to score "
	"on %s (now %s)",
	child_rsc->id, pcmk_readable_score(score),
	pcmk__node_name(node),
	pcmk_readable_score(new_score));
	}
	}

	if (score > child_rsc->private->priority) {
	pcmk__rsc_trace(rsc,
	"Updating %s priority to promotion score "
	"(%d->%d)",
	child_rsc->id, child_rsc->private->priority,
	score);
	child_rsc->private->priority = score;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief If a resource's current role is started, change it to unpromoted
	*
	* \param[in,out] data Resource to update
	* \param[in] user_data Ignored
	*/
	static void
	set_current_role_unpromoted(void data, void user_data)
	{
	pcmk_resource_t rsc = (pcmk_resource_t ) data;

	if (rsc->private->orig_role == pcmk_role_started) {
	// Promotable clones should use unpromoted role instead of started
	rsc->private->orig_role = pcmk_role_unpromoted;
	}
	g_list_foreach(rsc->private->children, set_current_role_unpromoted, NULL);
	}

	/*!
	* \internal
	* \brief Set a resource's next role to unpromoted (or stopped if unassigned)
	*
	* \param[in,out] data Resource to update
	* \param[in] user_data Ignored
	*/
	static void
	set_next_role_unpromoted(void data, void user_data)
	{
	pcmk_resource_t rsc = (pcmk_resource_t ) data;
	GList *assigned = NULL;

	rsc->private->fns->location(rsc, &assigned, FALSE);
	if (assigned == NULL) {
	pe__set_next_role(rsc, pcmk_role_stopped, "stopped instance");
	} else {
	pe__set_next_role(rsc, pcmk_role_unpromoted, "unpromoted instance");
	g_list_free(assigned);
	}
	g_list_foreach(rsc->private->children, set_next_role_unpromoted, NULL);
	}

	/*!
	* \internal
	* \brief Set a resource's next role to promoted if not already set
	*
	* \param[in,out] data Resource to update
	* \param[in] user_data Ignored
	*/
	static void
	set_next_role_promoted(void *data, gpointer user_data)
	{
	pcmk_resource_t rsc = (pcmk_resource_t ) data;

	if (rsc->private->next_role == pcmk_role_unknown) {
	pe__set_next_role(rsc, pcmk_role_promoted, "promoted instance");
	}
	g_list_foreach(rsc->private->children, set_next_role_promoted, NULL);
	}

	/*!
	* \internal
	* \brief Show instance's promotion score on node where it will be active
	*
	* \param[in,out] instance Promotable clone instance to show
	*/
	static void
	show_promotion_score(pcmk_resource_t *instance)
	{
	pcmk_node_t *chosen = instance->private->fns->location(instance, NULL,
	FALSE);
	const char *score_s = NULL;

	score_s = pcmk_readable_score(instance->private->promotion_priority);
	if (pcmk_is_set(instance->private->scheduler->flags,
	pcmk_sched_output_scores)
	&& !pcmk__is_daemon && (instance->private->scheduler->priv != NULL)) {

	pcmk__output_t *out = instance->private->scheduler->priv;

	out->message(out, "promotion-score", instance, chosen, score_s);

	} else if (chosen == NULL) {
	pcmk__rsc_debug(pe__const_top_resource(instance, false),
	"%s promotion score (inactive): %s (priority=%d)",
	instance->id, score_s, instance->private->priority);

	} else {
	pcmk__rsc_debug(pe__const_top_resource(instance, false),
	"%s promotion score on %s: %s (priority=%d)",
	instance->id, pcmk__node_name(chosen),
	score_s, instance->private->priority);
	}
	}

	/*!
	* \internal
	* \brief Set a clone instance's promotion priority
	*
	* \param[in,out] data Promotable clone instance to update
	* \param[in] user_data Instance's parent clone
	*/
	static void
	set_instance_priority(gpointer data, gpointer user_data)
	{
	pcmk_resource_t instance = (pcmk_resource_t ) data;
	const pcmk_resource_t clone = (const pcmk_resource_t ) user_data;

	const pcmk_node_t *chosen = NULL;
	enum rsc_role_e next_role = pcmk_role_unknown;
	GList *list = NULL;

	pcmk__rsc_trace(clone, "Assigning priority for %s: %s", instance->id,
	pcmk_role_text(instance->private->next_role));

	if (instance->private->fns->state(instance, TRUE) == pcmk_role_started) {
	set_current_role_unpromoted(instance, NULL);
	}

	// Only an instance that will be active can be promoted
	chosen = instance->private->fns->location(instance, &list, FALSE);
	if (pcmk__list_of_multiple(list)) {
	pcmk__config_err("Cannot promote non-colocated child %s",
	instance->id);
	}
	g_list_free(list);
	if (chosen == NULL) {
	return;
	}

	next_role = instance->private->fns->state(instance, FALSE);
	switch (next_role) {
	case pcmk_role_started:
	case pcmk_role_unknown:
	// Set instance priority to its promotion score (or -1 if none)
	{
	bool is_default = false;

	instance->private->priority = promotion_score(instance, chosen,
	&is_default);
	if (is_default) {
	/* Default to -1 if no value is set. This allows instances
	* eligible for promotion to be specified based solely on
	* PCMK_XE_RSC_LOCATION constraints, but prevents any
	* instance from being promoted if neither a constraint nor
	* a promotion score is present.
	*/
	instance->private->priority = -1;
	}
	}
	break;

	case pcmk_role_unpromoted:
	case pcmk_role_stopped:
	// Instance can't be promoted
	instance->private->priority = -PCMK_SCORE_INFINITY;
	break;

	case pcmk_role_promoted:
	// Nothing needed (re-creating actions after scheduling fencing)
	break;

	default:
	CRM_CHECK(FALSE, crm_err("Unknown resource role %d for %s",
	next_role, instance->id));
	}

	// Add relevant location constraint scores for promoted role
	apply_promoted_locations(instance, instance->private->location_constraints,
	chosen);
	apply_promoted_locations(instance, clone->private->location_constraints,
	chosen);

	// Consider instance's role-based colocations with other resources
	list = pcmk__this_with_colocations(instance);
	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) iter->data;

	instance->private->cmds->apply_coloc_score(instance, cons->primary,
	cons, true);
	}
	g_list_free(list);

	instance->private->promotion_priority = instance->private->priority;
	if (next_role == pcmk_role_promoted) {
	instance->private->promotion_priority = PCMK_SCORE_INFINITY;
	}
	pcmk__rsc_trace(clone, "Assigning %s priority = %d",
	instance->id, instance->private->priority);
	}

	/*!
	* \internal
	* \brief Set a promotable clone instance's role
	*
	* \param[in,out] data Promotable clone instance to update
	* \param[in,out] user_data Pointer to count of instances chosen for promotion
	*/
	static void
	set_instance_role(gpointer data, gpointer user_data)
	{
	pcmk_resource_t instance = (pcmk_resource_t ) data;
	int count = (int ) user_data;

	const pcmk_resource_t *clone = pe__const_top_resource(instance, false);
	const pcmk_scheduler_t *scheduler = instance->private->scheduler;
	pcmk_node_t *chosen = NULL;

	show_promotion_score(instance);

	if (instance->private->promotion_priority < 0) {
	pcmk__rsc_trace(clone, "Not supposed to promote instance %s",
	instance->id);

	} else if ((*count < pe__clone_promoted_max(instance))
	\|\| !pcmk_is_set(clone->flags, pcmk__rsc_managed)) {
	chosen = node_to_be_promoted_on(instance);
	}

	if (chosen == NULL) {
	set_next_role_unpromoted(instance, NULL);
	return;
	}

	if ((instance->private->orig_role < pcmk_role_promoted)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	&& (scheduler->no_quorum_policy == pcmk_no_quorum_freeze)) {
	crm_notice("Clone instance %s cannot be promoted without quorum",
	instance->id);
	set_next_role_unpromoted(instance, NULL);
	return;
	}

	- chosen->count++;
	+ chosen->assign->count++;
	pcmk__rsc_info(clone, "Choosing %s (%s) on %s for promotion",
	instance->id, pcmk_role_text(instance->private->orig_role),
	pcmk__node_name(chosen));
	set_next_role_promoted(instance, NULL);
	(*count)++;
	}

	/*!
	* \internal
	* \brief Set roles for all instances of a promotable clone
	*
	* \param[in,out] rsc Promotable clone resource to update
	*/
	void
	pcmk__set_instance_roles(pcmk_resource_t *rsc)
	{
	int promoted = 0;
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	// Repurpose count to track the number of promoted instances assigned
	g_hash_table_iter_init(&iter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	- node->count = 0;
	+ node->assign->count = 0;
	}

	// Set instances' promotion priorities and sort by highest priority first
	g_list_foreach(rsc->private->children, set_instance_priority, rsc);
	sort_promotable_instances(rsc);

	// Choose the first N eligible instances to be promoted
	g_list_foreach(rsc->private->children, set_instance_role, &promoted);
	pcmk__rsc_info(rsc, "%s: Promoted %d instances of a possible %d",
	rsc->id, promoted, pe__clone_promoted_max(rsc));
	}

	/*!
	*
	* \internal
	* \brief Create actions for promotable clone instances
	*
	* \param[in,out] clone Promotable clone to create actions for
	* \param[out] any_promoting Will be set true if any instance is promoting
	* \param[out] any_demoting Will be set true if any instance is demoting
	*/
	static void
	create_promotable_instance_actions(pcmk_resource_t *clone,
	bool any_promoting, bool any_demoting)
	{
	for (GList *iter = clone->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->private->cmds->create_actions(instance);
	check_for_role_change(instance, any_demoting, any_promoting);
	}
	}

	/*!
	* \internal
	* \brief Reset each promotable instance's resource priority
	*
	* Reset the priority of each instance of a promotable clone to the clone's
	* priority (after promotion actions are scheduled, when instance priorities
	* were repurposed as promotion scores).
	*
	* \param[in,out] clone Promotable clone to reset
	*/
	static void
	reset_instance_priorities(pcmk_resource_t *clone)
	{
	for (GList *iter = clone->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->private->priority = clone->private->priority;
	}
	}

	/*!
	* \internal
	* \brief Create actions specific to promotable clones
	*
	* \param[in,out] clone Promotable clone to create actions for
	*/
	void
	pcmk__create_promotable_actions(pcmk_resource_t *clone)
	{
	bool any_promoting = false;
	bool any_demoting = false;

	// Create actions for each clone instance individually
	create_promotable_instance_actions(clone, &any_promoting, &any_demoting);

	// Create pseudo-actions for clone as a whole
	pe__create_promotable_pseudo_ops(clone, any_promoting, any_demoting);

	// Undo our temporary repurposing of resource priority for instances
	reset_instance_priorities(clone);
	}

	/*!
	* \internal
	* \brief Create internal orderings for a promotable clone's instances
	*
	* \param[in,out] clone Promotable clone instance to order
	*/
	void
	pcmk__order_promotable_instances(pcmk_resource_t *clone)
	{
	pcmk_resource_t *previous = NULL; // Needed for ordered clones

	pcmk__promotable_restart_ordering(clone);

	for (GList *iter = clone->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	// Demote before promote
	pcmk__order_resource_actions(instance, PCMK_ACTION_DEMOTE,
	instance, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	order_instance_promotion(clone, instance, previous);
	order_instance_demotion(clone, instance, previous);
	previous = instance;
	}
	}

	/*!
	* \internal
	* \brief Update dependent's allowed nodes for colocation with promotable
	*
	* \param[in,out] dependent Dependent resource to update
	* \param[in] primary Primary resource
	* \param[in] primary_node Node where an instance of the primary will be
	* \param[in] colocation Colocation constraint to apply
	*/
	static void
	update_dependent_allowed_nodes(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk_node_t *primary_node,
	const pcmk__colocation_t *colocation)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	const char *primary_value = NULL;
	const char *attr = colocation->node_attribute;

	if (colocation->score >= PCMK_SCORE_INFINITY) {
	return; // Colocation is mandatory, so allowed node scores don't matter
	}

	primary_value = pcmk__colocation_node_attr(primary_node, attr, primary);

	pcmk__rsc_trace(colocation->primary,
	"Applying %s (%s with %s on %s by %s @%d) to %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, pcmk__node_name(primary_node),
	attr, colocation->score, dependent->id);

	g_hash_table_iter_init(&iter, dependent->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	const char *dependent_value = pcmk__colocation_node_attr(node, attr,
	dependent);

	if (pcmk__str_eq(primary_value, dependent_value, pcmk__str_casei)) {
	- node->weight = pcmk__add_scores(node->weight, colocation->score);
	+ node->assign->score = pcmk__add_scores(node->assign->score,
	+ colocation->score);
	pcmk__rsc_trace(colocation->primary,
	"Added %s score (%s) to %s (now %s)",
	colocation->id,
	pcmk_readable_score(colocation->score),
	pcmk__node_name(node),
	- pcmk_readable_score(node->weight));
	+ pcmk_readable_score(node->assign->score));
	}
	}
	}

	/*!
	* \brief Update dependent for a colocation with a promotable clone
	*
	* \param[in] primary Primary resource in the colocation
	* \param[in,out] dependent Dependent resource in the colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	void
	pcmk__update_dependent_with_promotable(const pcmk_resource_t *primary,
	pcmk_resource_t *dependent,
	const pcmk__colocation_t *colocation)
	{
	GList *affected_nodes = NULL;

	/* Build a list of all nodes where an instance of the primary will be, and
	* (for optional colocations) update the dependent's allowed node scores for
	* each one.
	*/
	for (GList *iter = primary->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;
	pcmk_node_t *node = instance->private->fns->location(instance, NULL,
	FALSE);

	if (node == NULL) {
	continue;
	}
	if (instance->private->fns->state(instance,
	FALSE) == colocation->primary_role) {
	update_dependent_allowed_nodes(dependent, primary, node,
	colocation);
	affected_nodes = g_list_prepend(affected_nodes, node);
	}
	}

	/* For mandatory colocations, add the primary's node score to the
	* dependent's node score for each affected node, and ban the dependent
	* from all other nodes.
	*
	* However, skip this for promoted-with-promoted colocations, otherwise
	* inactive dependent instances can't start (in the unpromoted role).
	*/
	if ((colocation->score >= PCMK_SCORE_INFINITY)
	&& ((colocation->dependent_role != pcmk_role_promoted)
	\|\| (colocation->primary_role != pcmk_role_promoted))) {

	pcmk__rsc_trace(colocation->primary,
	"Applying %s (mandatory %s with %s) to %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, dependent->id);
	pcmk__colocation_intersect_nodes(dependent, primary, colocation,
	affected_nodes, true);
	}
	g_list_free(affected_nodes);
	}

	/*!
	* \internal
	* \brief Update dependent priority for colocation with promotable
	*
	* \param[in] primary Primary resource in the colocation
	* \param[in,out] dependent Dependent resource in the colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	void
	pcmk__update_promotable_dependent_priority(const pcmk_resource_t *primary,
	pcmk_resource_t *dependent,
	const pcmk__colocation_t *colocation)
	{
	pcmk_resource_t *primary_instance = NULL;

	// Look for a primary instance where dependent will be
	primary_instance = pcmk__find_compatible_instance(dependent, primary,
	colocation->primary_role,
	false);

	if (primary_instance != NULL) {
	// Add primary instance's priority to dependent's
	int new_priority = pcmk__add_scores(dependent->private->priority,
	colocation->score);

	pcmk__rsc_trace(colocation->primary,
	"Applying %s (%s with %s) to %s priority "
	"(%s + %s = %s)",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, dependent->id,
	pcmk_readable_score(dependent->private->priority),
	pcmk_readable_score(colocation->score),
	pcmk_readable_score(new_priority));
	dependent->private->priority = new_priority;

	} else if (colocation->score >= PCMK_SCORE_INFINITY) {
	// Mandatory colocation, but primary won't be here
	pcmk__rsc_trace(colocation->primary,
	"Applying %s (%s with %s) to %s: can't be promoted",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, dependent->id);
	dependent->private->priority = -PCMK_SCORE_INFINITY;
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c
	index a45d197f78..68c83264e8 100644
	--- a/lib/pacemaker/pcmk_sched_resource.c
	+++ b/lib/pacemaker/pcmk_sched_resource.c
	@@ -1,792 +1,795 @@
	/*
	* Copyright 2014-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <string.h>
	#include <crm/common/xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Resource assignment methods by resource variant
	static pcmk__assignment_methods_t assignment_methods[] = {
	{
	pcmk__primitive_assign,
	pcmk__primitive_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__primitive_internal_constraints,
	pcmk__primitive_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_primitive_colocations,
	pcmk__primitive_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__apply_location,
	pcmk__primitive_action_flags,
	pcmk__update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__primitive_add_graph_meta,
	pcmk__primitive_add_utilization,
	pcmk__primitive_shutdown_lock,
	},
	{
	pcmk__group_assign,
	pcmk__group_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__group_internal_constraints,
	pcmk__group_apply_coloc_score,
	pcmk__group_colocated_resources,
	pcmk__with_group_colocations,
	pcmk__group_with_colocations,
	pcmk__group_add_colocated_node_scores,
	pcmk__group_apply_location,
	pcmk__group_action_flags,
	pcmk__group_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__group_add_utilization,
	pcmk__group_shutdown_lock,
	},
	{
	pcmk__clone_assign,
	pcmk__clone_create_actions,
	pcmk__clone_create_probe,
	pcmk__clone_internal_constraints,
	pcmk__clone_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_clone_colocations,
	pcmk__clone_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__clone_apply_location,
	pcmk__clone_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__clone_add_actions_to_graph,
	pcmk__clone_add_graph_meta,
	pcmk__clone_add_utilization,
	pcmk__clone_shutdown_lock,
	},
	{
	pcmk__bundle_assign,
	pcmk__bundle_create_actions,
	pcmk__bundle_create_probe,
	pcmk__bundle_internal_constraints,
	pcmk__bundle_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_bundle_colocations,
	pcmk__bundle_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__bundle_apply_location,
	pcmk__bundle_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_bundle_actions,
	pcmk__bundle_add_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__bundle_add_utilization,
	pcmk__bundle_shutdown_lock,
	}
	};

	/*!
	* \internal
	* \brief Check whether a resource's agent standard, provider, or type changed
	*
	* \param[in,out] rsc Resource to check
	* \param[in,out] node Node needing unfencing if agent changed
	* \param[in] rsc_entry XML with previously known agent information
	* \param[in] active_on_node Whether \p rsc is active on \p node
	*
	* \return true if agent for \p rsc changed, otherwise false
	*/
	bool
	pcmk__rsc_agent_changed(pcmk_resource_t rsc, pcmk_node_t node,
	const xmlNode *rsc_entry, bool active_on_node)
	{
	bool changed = false;
	const char *attr_list[] = {
	PCMK_XA_TYPE,
	PCMK_XA_CLASS,
	PCMK_XA_PROVIDER,
	};

	for (int i = 0; i < PCMK__NELEM(attr_list); i++) {
	const char *value = crm_element_value(rsc->private->xml, attr_list[i]);
	const char *old_value = crm_element_value(rsc_entry, attr_list[i]);

	if (!pcmk__str_eq(value, old_value, pcmk__str_none)) {
	changed = true;
	trigger_unfencing(rsc, node, "Device definition changed", NULL,
	rsc->private->scheduler);
	if (active_on_node) {
	crm_notice("Forcing restart of %s on %s "
	"because %s changed from '%s' to '%s'",
	rsc->id, pcmk__node_name(node), attr_list[i],
	pcmk__s(old_value, ""), pcmk__s(value, ""));
	}
	}
	}
	if (changed && active_on_node) {
	// Make sure the resource is restarted
	custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE,
	rsc->private->scheduler);
	pcmk__set_rsc_flags(rsc, pcmk__rsc_start_pending);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Add resource (and any matching children) to list if it matches ID
	*
	* \param[in] result List to add resource to
	* \param[in] rsc Resource to check
	* \param[in] id ID to match
	*
	* \return (Possibly new) head of list
	*/
	static GList *
	add_rsc_if_matching(GList result, pcmk_resource_t rsc, const char *id)
	{
	if (pcmk__str_eq(id, rsc->id, pcmk__str_none)
	\|\| pcmk__str_eq(id, rsc->private->history_id, pcmk__str_none)) {
	result = g_list_prepend(result, rsc);
	}

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	result = add_rsc_if_matching(result, child, id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Find all resources matching a given ID by either ID or clone name
	*
	* \param[in] id Resource ID to check
	* \param[in] scheduler Scheduler data
	*
	* \return List of all resources that match \p id
	* \note The caller is responsible for freeing the return value with
	* g_list_free().
	*/
	GList *
	pcmk__rscs_matching_id(const char id, const pcmk_scheduler_t scheduler)
	{
	GList *result = NULL;

	CRM_CHECK((id != NULL) && (scheduler != NULL), return NULL);
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	result = add_rsc_if_matching(result, (pcmk_resource_t *) iter->data,
	id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for a resource
	*
	* \param[in,out] data Resource to set assignment methods for
	* \param[in] user_data Ignored
	*/
	static void
	set_assignment_methods_for_rsc(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;

	rsc->private->cmds = &assignment_methods[rsc->private->variant];
	g_list_foreach(rsc->private->children, set_assignment_methods_for_rsc,
	NULL);
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for all resources
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__set_assignment_methods(pcmk_scheduler_t *scheduler)
	{
	g_list_foreach(scheduler->resources, set_assignment_methods_for_rsc, NULL);
	}

	/*!
	* \internal
	* \brief Wrapper for colocated_resources() method for readability
	*
	* \param[in] rsc Resource to add to colocated list
	* \param[in] orig_rsc Resource originally requested
	* \param[in,out] list Pointer to list to add to
	*
	* \return (Possibly new) head of list
	*/
	static inline void
	add_colocated_resources(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	list = rsc->private->cmds->colocated_resources(rsc, orig_rsc, list);
	}

	// Shared implementation of pcmk__assignment_methods_t:colocated_resources()
	GList *
	pcmk__colocated_resources(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList *colocated_rscs)
	{
	const GList *iter = NULL;
	GList *colocations = NULL;

	if (orig_rsc == NULL) {
	orig_rsc = rsc;
	}

	if ((rsc == NULL) \|\| (g_list_find(colocated_rscs, rsc) != NULL)) {
	return colocated_rscs;
	}

	pcmk__rsc_trace(orig_rsc, "%s is in colocation chain with %s",
	rsc->id, orig_rsc->id);
	colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);

	// Follow colocations where this resource is the dependent resource
	colocations = pcmk__this_with_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pcmk_resource_t *primary = constraint->primary;

	if (primary == orig_rsc) {
	continue; // Break colocation loop
	}

	if ((constraint->score == PCMK_SCORE_INFINITY) &&
	(pcmk__colocation_affects(rsc, primary, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(primary, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	// Follow colocations where this resource is the primary resource
	colocations = pcmk__with_this_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pcmk_resource_t *dependent = constraint->dependent;

	if (dependent == orig_rsc) {
	continue; // Break colocation loop
	}

	if (pcmk__is_clone(rsc) && !pcmk__is_clone(dependent)) {
	continue; // We can't be sure whether dependent will be colocated
	}

	if ((constraint->score == PCMK_SCORE_INFINITY) &&
	(pcmk__colocation_affects(dependent, rsc, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(dependent, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	return colocated_rscs;
	}

	// No-op function for variants that don't need to implement add_graph_meta()
	void
	pcmk__noop_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml)
	{
	}

	/*!
	* \internal
	* \brief Output a summary of scheduled actions for a resource
	*
	* \param[in,out] rsc Resource to output actions for
	*/
	void
	pcmk__output_resource_actions(pcmk_resource_t *rsc)
	{
	pcmk_node_t *next = NULL;
	pcmk_node_t *current = NULL;
	pcmk__output_t *out = NULL;

	CRM_ASSERT(rsc != NULL);

	out = rsc->private->scheduler->priv;
	if (rsc->private->children != NULL) {

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	child->private->cmds->output_actions(child);
	}
	return;
	}

	next = rsc->private->assigned_node;
	if (rsc->private->active_nodes != NULL) {
	current = pcmk__current_node(rsc);
	if (rsc->private->orig_role == pcmk_role_stopped) {
	/* This can occur when resources are being recovered because
	* the current role can change in pcmk__primitive_create_actions()
	*/
	rsc->private->orig_role = pcmk_role_started;
	}
	}

	if ((current == NULL) && pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
	/* Don't log stopped orphans */
	return;
	}

	out->message(out, "rsc-action", rsc, current, next);
	}

	/*!
	* \internal
	* \brief Add a resource to a node's list of assigned resources
	*
	* \param[in,out] node Node to add resource to
	* \param[in] rsc Resource to add
	*/
	static inline void
	add_assigned_resource(pcmk_node_t node, pcmk_resource_t rsc)
	{
	node->private->assigned_resources =
	g_list_prepend(node->private->assigned_resources, rsc);
	}

	/*!
	* \internal
	* \brief Assign a specified resource (of any variant) to a node
	*
	* Assign a specified resource and its children (if any) to a specified node, if
	* the node can run the resource (or unconditionally, if \p force is true). Mark
	* the resources as no longer provisional.
	*
	* If a resource can't be assigned (or \p node is \c NULL), unassign any
	* previous assignment. If \p stop_if_fail is \c true, set next role to stopped
	* and update any existing actions scheduled for the resource.
	*
	* \param[in,out] rsc Resource to assign
	* \param[in,out] node Node to assign \p rsc to
	* \param[in] force If true, assign to \p node even if unavailable
	* \param[in] stop_if_fail If \c true and either \p rsc can't be assigned
	* or \p chosen is \c NULL, set next role to
	* stopped and update existing actions (if \p rsc
	* is not a primitive, this applies to its
	* primitive descendants instead)
	*
	* \return \c true if the assignment of \p rsc changed, or \c false otherwise
	*
	* \note Assigning a resource to the NULL node using this function is different
	* from calling pcmk__unassign_resource(), in that it may also update any
	* actions created for the resource.
	* \note The \c pcmk__assignment_methods_t:assign() method is preferred, unless
	* a resource should be assigned to the \c NULL node or every resource in
	* a tree should be assigned to the same node.
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	bool
	pcmk__assign_resource(pcmk_resource_t rsc, pcmk_node_t node, bool force,
	bool stop_if_fail)
	{
	bool changed = false;
	pcmk_scheduler_t *scheduler = NULL;

	CRM_ASSERT(rsc != NULL);
	scheduler = rsc->private->scheduler;

	if (rsc->private->children != NULL) {

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t *child_rsc = iter->data;

	changed \|= pcmk__assign_resource(child_rsc, node, force,
	stop_if_fail);
	}
	return changed;
	}

	// Assigning a primitive

	if (!force && (node != NULL)
	- && ((node->weight < 0)
	+ && ((node->assign->score < 0)
	// Allow graph to assume that guest node connections will come up
	\|\| (!pcmk__node_available(node, true, false)
	&& !pcmk__is_guest_or_bundle_node(node)))) {

	pcmk__rsc_debug(rsc,
	"All nodes for resource %s are unavailable, unclean or "
	"shutting down (%s can%s run resources, with score %s)",
	rsc->id, pcmk__node_name(node),
	(pcmk__node_available(node, true, false)? "" : "not"),
	- pcmk_readable_score(node->weight));
	+ pcmk_readable_score(node->assign->score));

	if (stop_if_fail) {
	pe__set_next_role(rsc, pcmk_role_stopped, "node availability");
	}
	node = NULL;
	}

	if (rsc->private->assigned_node != NULL) {
	changed = !pcmk__same_node(rsc->private->assigned_node, node);
	} else {
	changed = (node != NULL);
	}
	pcmk__unassign_resource(rsc);
	pcmk__clear_rsc_flags(rsc, pcmk__rsc_unassigned);

	if (node == NULL) {
	char *rc_stopped = NULL;

	pcmk__rsc_debug(rsc, "Could not assign %s to a node", rsc->id);

	if (!stop_if_fail) {
	return changed;
	}
	pe__set_next_role(rsc, pcmk_role_stopped, "unable to assign");

	for (GList *iter = rsc->private->actions;
	iter != NULL; iter = iter->next) {

	pcmk_action_t op = (pcmk_action_t ) iter->data;

	pcmk__rsc_debug(rsc, "Updating %s for %s assignment failure",
	op->uuid, rsc->id);

	if (pcmk__str_eq(op->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	pcmk__clear_action_flags(op, pcmk_action_optional);

	} else if (pcmk__str_eq(op->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	pcmk__clear_action_flags(op, pcmk_action_runnable);

	} else {
	// Cancel recurring actions, unless for stopped state
	const char *interval_ms_s = NULL;
	const char *target_rc_s = NULL;

	interval_ms_s = g_hash_table_lookup(op->meta,
	PCMK_META_INTERVAL);
	target_rc_s = g_hash_table_lookup(op->meta,
	PCMK__META_OP_TARGET_RC);
	if (rc_stopped == NULL) {
	rc_stopped = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
	}

	if (!pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)
	&& !pcmk__str_eq(rc_stopped, target_rc_s, pcmk__str_none)) {

	pcmk__clear_action_flags(op, pcmk_action_runnable);
	}
	}
	}
	free(rc_stopped);
	return changed;
	}

	pcmk__rsc_debug(rsc, "Assigning %s to %s", rsc->id, pcmk__node_name(node));
	rsc->private->assigned_node = pe__copy_node(node);

	add_assigned_resource(node, rsc);
	node->private->num_resources++;
	- node->count++;
	+ node->assign->count++;
	pcmk__consume_node_capacity(node->private->utilization, rsc);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	pcmk__output_t *out = scheduler->priv;

	out->message(out, "resource-util", rsc, node, __func__);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Remove any node assignment from a specified resource and its children
	*
	* If a specified resource has been assigned to a node, remove that assignment
	* and mark the resource as provisional again.
	*
	* \param[in,out] rsc Resource to unassign
	*
	* \note This function is called recursively on \p rsc and its children.
	*/
	void
	pcmk__unassign_resource(pcmk_resource_t *rsc)
	{
	pcmk_node_t *old = rsc->private->assigned_node;

	if (old == NULL) {
	crm_info("Unassigning %s", rsc->id);
	} else {
	crm_info("Unassigning %s from %s", rsc->id, pcmk__node_name(old));
	}

	pcmk__set_rsc_flags(rsc, pcmk__rsc_unassigned);

	if (rsc->private->children == NULL) {
	if (old == NULL) {
	return;
	}
	rsc->private->assigned_node = NULL;

	/* We're going to free the pcmk_node_t, but its details member is shared
	* and will remain, so update that appropriately first.
	*/
	old->private->assigned_resources =
	g_list_remove(old->private->assigned_resources, rsc);
	old->private->num_resources--;
	pcmk__release_node_capacity(old->private->utilization, rsc);
	free(old);
	return;
	}

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk__unassign_resource((pcmk_resource_t *) iter->data);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has reached its migration threshold on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node Node to check
	* \param[out] failed If threshold has been reached, this will be set to
	* resource that failed (possibly a parent of \p rsc)
	*
	* \return true if the migration threshold has been reached, false otherwise
	*/
	bool
	pcmk__threshold_reached(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_resource_t **failed)
	{
	int fail_count, remaining_tries;
	pcmk_resource_t *rsc_to_ban = rsc;

	// Migration threshold of 0 means never force away
	if (rsc->private->ban_after_failures == 0) {
	return false;
	}

	// If we're ignoring failures, also ignore the migration threshold
	if (pcmk_is_set(rsc->flags, pcmk__rsc_ignore_failure)) {
	return false;
	}

	// If there are no failures, there's no need to force away
	fail_count = pe_get_failcount(node, rsc, NULL,
	pcmk__fc_effective\|pcmk__fc_launched, NULL);
	if (fail_count <= 0) {
	return false;
	}

	// If failed resource is anonymous clone instance, we'll force clone away
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
	rsc_to_ban = uber_parent(rsc);
	}

	// How many more times recovery will be tried on this node
	remaining_tries = rsc->private->ban_after_failures - fail_count;

	if (remaining_tries <= 0) {
	pcmk__sched_warn("%s cannot run on %s due to reaching migration "
	"threshold (clean up resource to allow again)"
	QB_XS " failures=%d "
	PCMK_META_MIGRATION_THRESHOLD "=%d",
	rsc_to_ban->id, pcmk__node_name(node), fail_count,
	rsc->private->ban_after_failures);
	if (failed != NULL) {
	*failed = rsc_to_ban;
	}
	return true;
	}

	crm_info("%s can fail %d more time%s on "
	"%s before reaching migration threshold (%d)",
	rsc_to_ban->id, remaining_tries, pcmk__plural_s(remaining_tries),
	pcmk__node_name(node), rsc->private->ban_after_failures);
	return false;
	}

	/*!
	* \internal
	* \brief Get a node's score
	*
	* \param[in] node Node with ID to check
	* \param[in] nodes List of nodes to look for \p node score in
	*
	* \return Node's score, or -INFINITY if not found
	*/
	static int
	get_node_score(const pcmk_node_t node, GHashTable nodes)
	{
	pcmk_node_t *found_node = NULL;

	if ((node != NULL) && (nodes != NULL)) {
	found_node = g_hash_table_lookup(nodes, node->private->id);
	}
	- return (found_node == NULL)? -PCMK_SCORE_INFINITY : found_node->weight;
	+ if (found_node == NULL) {
	+ return -PCMK_SCORE_INFINITY;
	+ }
	+ return found_node->assign->score;
	}

	/*!
	* \internal
	* \brief Compare two resources according to which should be assigned first
	*
	* \param[in] a First resource to compare
	* \param[in] b Second resource to compare
	* \param[in] data Sorted list of all nodes in cluster
	*
	* \return -1 if \p a should be assigned before \b, 0 if they are equal,
	* or +1 if \p a should be assigned after \b
	*/
	static gint
	cmp_resources(gconstpointer a, gconstpointer b, gpointer data)
	{
	/* GLib insists that this function require gconstpointer arguments, but we
	* make a small, temporary change to each argument (setting the
	* pe_rsc_merging flag) during comparison
	*/
	pcmk_resource_t resource1 = (pcmk_resource_t ) a;
	pcmk_resource_t resource2 = (pcmk_resource_t ) b;
	const GList *nodes = data;

	int rc = 0;
	int r1_score = -PCMK_SCORE_INFINITY;
	int r2_score = -PCMK_SCORE_INFINITY;
	pcmk_node_t *r1_node = NULL;
	pcmk_node_t *r2_node = NULL;
	GHashTable *r1_nodes = NULL;
	GHashTable *r2_nodes = NULL;
	const char *reason = NULL;

	// Resources with highest priority should be assigned first
	reason = "priority";
	r1_score = resource1->private->priority;
	r2_score = resource2->private->priority;
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// We need nodes to make any other useful comparisons
	reason = "no node list";
	if (nodes == NULL) {
	goto done;
	}

	// Calculate and log node scores
	resource1->private->cmds->add_colocated_node_scores(resource1, NULL,
	resource1->id,
	&r1_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	resource2->private->cmds->add_colocated_node_scores(resource2, NULL,
	resource2->id,
	&r2_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	pe__show_node_scores(true, NULL, resource1->id, r1_nodes,
	resource1->private->scheduler);
	pe__show_node_scores(true, NULL, resource2->id, r2_nodes,
	resource2->private->scheduler);

	// The resource with highest score on its current node goes first
	reason = "current location";
	if (resource1->private->active_nodes != NULL) {
	r1_node = pcmk__current_node(resource1);
	}
	if (resource2->private->active_nodes != NULL) {
	r2_node = pcmk__current_node(resource2);
	}
	r1_score = get_node_score(r1_node, r1_nodes);
	r2_score = get_node_score(r2_node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// Otherwise a higher score on any node will do
	reason = "score";
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	const pcmk_node_t node = (const pcmk_node_t ) iter->data;

	r1_score = get_node_score(node, r1_nodes);
	r2_score = get_node_score(node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}
	}

	done:
	crm_trace("%s (%d)%s%s %c %s (%d)%s%s: %s",
	resource1->id, r1_score,
	((r1_node == NULL)? "" : " on "),
	((r1_node == NULL)? "" : r1_node->private->id),
	((rc < 0)? '>' : ((rc > 0)? '<' : '=')),
	resource2->id, r2_score,
	((r2_node == NULL)? "" : " on "),
	((r2_node == NULL)? "" : r2_node->private->id),
	reason);
	if (r1_nodes != NULL) {
	g_hash_table_destroy(r1_nodes);
	}
	if (r2_nodes != NULL) {
	g_hash_table_destroy(r2_nodes);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Sort resources in the order they should be assigned to nodes
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__sort_resources(pcmk_scheduler_t *scheduler)
	{
	GList *nodes = g_list_copy(scheduler->nodes);

	nodes = pcmk__sort_nodes(nodes, NULL);
	scheduler->resources = g_list_sort_with_data(scheduler->resources,
	cmp_resources, nodes);
	g_list_free(nodes);
	}
	diff --git a/lib/pacemaker/pcmk_scheduler.c b/lib/pacemaker/pcmk_scheduler.c
	index 9d94af30f5..edea10542c 100644
	--- a/lib/pacemaker/pcmk_scheduler.c
	+++ b/lib/pacemaker/pcmk_scheduler.c
	@@ -1,906 +1,907 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	CRM_TRACE_INIT_DATA(pacemaker);

	/*!
	* \internal
	* \brief Do deferred action checks after assignment
	*
	* When unpacking the resource history, the scheduler checks for resource
	* configurations that have changed since an action was run. However, at that
	* time, bundles using the REMOTE_CONTAINER_HACK don't have their final
	* parameter information, so instead they add a deferred check to a list. This
	* function processes one entry in that list.
	*
	* \param[in,out] rsc Resource that action history is for
	* \param[in,out] node Node that action history is for
	* \param[in] rsc_op Action history entry
	* \param[in] check Type of deferred check to do
	*/
	static void
	check_params(pcmk_resource_t rsc, pcmk_node_t node, const xmlNode *rsc_op,
	enum pcmk__check_parameters check)
	{
	const char *reason = NULL;
	pcmk__op_digest_t *digest_data = NULL;

	switch (check) {
	case pcmk__check_active:
	if (pcmk__check_action_config(rsc, node, rsc_op)
	&& pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
	NULL)) {
	reason = "action definition changed";
	}
	break;

	case pcmk__check_last_failure:
	digest_data = rsc_action_digest_cmp(rsc, rsc_op, node,
	rsc->private->scheduler);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s has "
	"no digest to compare",
	rsc->id, pcmk__xe_id(rsc_op), node->private->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	reason = "resource parameters have changed";
	break;
	}
	break;
	}
	if (reason != NULL) {
	pe__clear_failcount(rsc, node, reason, rsc->private->scheduler);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has failcount clearing scheduled on a node
	*
	* \param[in] node Node to check
	* \param[in] rsc Resource to check
	*
	* \return true if \p rsc has failcount clearing scheduled on \p node,
	* otherwise false
	*/
	static bool
	failcount_clear_action_exists(const pcmk_node_t *node,
	const pcmk_resource_t *rsc)
	{
	GList *list = pe__resource_actions(rsc, node, PCMK_ACTION_CLEAR_FAILCOUNT,
	TRUE);

	if (list != NULL) {
	g_list_free(list);
	return true;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it reached its failure threshold there
	*
	* \param[in,out] data Resource to check failure threshold for
	* \param[in] user_data Node to check resource on
	*/
	static void
	check_failure_threshold(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;

	// If this is a collective resource, apply recursively to children instead
	if (rsc->private->children != NULL) {
	g_list_foreach(rsc->private->children, check_failure_threshold,
	user_data);
	return;
	}

	if (!failcount_clear_action_exists(node, rsc)) {
	/* Don't force the resource away from this node due to a failcount
	* that's going to be cleared.
	*
	* @TODO Failcount clearing can be scheduled in
	* pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
	* schedule_resource_actions() via check_params(). This runs well before
	* then, so it cannot detect those, meaning we might check the migration
	* threshold when we shouldn't. Worst case, we stop or move the
	* resource, then move it back in the next transition.
	*/
	pcmk_resource_t *failed = NULL;

	if (pcmk__threshold_reached(rsc, node, &failed)) {
	resource_location(failed, node, -PCMK_SCORE_INFINITY,
	"__fail_limit__", rsc->private->scheduler);
	}
	}
	}

	/*!
	* \internal
	* \brief If resource has exclusive discovery, ban node if not allowed
	*
	* Location constraints have a PCMK_XA_RESOURCE_DISCOVERY option that allows
	* users to specify where probes are done for the affected resource. If this is
	* set to \c exclusive, probes will only be done on nodes listed in exclusive
	* constraints. This function bans the resource from the node if the node is not
	* listed.
	*
	* \param[in,out] data Resource to check
	* \param[in] user_data Node to check resource on
	*/
	static void
	apply_exclusive_discovery(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;

	if (pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)
	\|\| pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk__rsc_exclusive_probes)) {
	pcmk_node_t *match = NULL;

	// If this is a collective resource, apply recursively to children
	g_list_foreach(rsc->private->children, apply_exclusive_discovery,
	user_data);

	match = g_hash_table_lookup(rsc->private->allowed_nodes,
	node->private->id);
	if ((match != NULL)
	- && (match->rsc_discover_mode != pcmk_probe_exclusive)) {
	- match->weight = -PCMK_SCORE_INFINITY;
	+ && (match->assign->probe_mode != pcmk__probe_exclusive)) {
	+ match->assign->score = -PCMK_SCORE_INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Apply stickiness to a resource if appropriate
	*
	* \param[in,out] data Resource to check for stickiness
	* \param[in] user_data Ignored
	*/
	static void
	apply_stickiness(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	pcmk_node_t *node = NULL;

	// If this is a collective resource, apply recursively to children instead
	if (rsc->private->children != NULL) {
	g_list_foreach(rsc->private->children, apply_stickiness, NULL);
	return;
	}

	/* A resource is sticky if it is managed, has stickiness configured, and is
	* active on a single node.
	*/
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)
	\|\| (rsc->private->stickiness < 1)
	\|\| !pcmk__list_of_1(rsc->private->active_nodes)) {
	return;
	}

	node = rsc->private->active_nodes->data;

	/* In a symmetric cluster, stickiness can always be used. In an
	* asymmetric cluster, we have to check whether the resource is still
	* allowed on the node, so we don't keep the resource somewhere it is no
	* longer explicitly enabled.
	*/
	if (!pcmk_is_set(rsc->private->scheduler->flags,
	pcmk_sched_symmetric_cluster)
	&& (g_hash_table_lookup(rsc->private->allowed_nodes,
	node->private->id) == NULL)) {
	pcmk__rsc_debug(rsc,
	"Ignoring %s stickiness because the cluster is "
	"asymmetric and %s is not explicitly allowed",
	rsc->id, pcmk__node_name(node));
	return;
	}

	pcmk__rsc_debug(rsc, "Resource %s has %d stickiness on %s",
	rsc->id, rsc->private->stickiness, pcmk__node_name(node));
	resource_location(rsc, node, rsc->private->stickiness, "stickiness",
	rsc->private->scheduler);
	}

	/*!
	* \internal
	* \brief Apply shutdown locks for all resources as appropriate
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	apply_shutdown_locks(pcmk_scheduler_t *scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	return;
	}
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->private->cmds->shutdown_lock(rsc);
	}
	}

	/*!
	* \internal
	* \brief Calculate the number of available nodes in the cluster
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	count_available_nodes(pcmk_scheduler_t *scheduler)
	{
	if (pcmk_is_set(scheduler->flags, pcmk_sched_no_compat)) {
	return;
	}

	// @COMPAT for API backward compatibility only (cluster does not use value)
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	- if ((node != NULL) && (node->weight >= 0) && node->details->online
	+ if ((node != NULL) && (node->assign->score >= 0)
	+ && node->details->online
	&& (node->private->variant != pcmk__node_variant_ping)) {
	scheduler->max_valid_nodes++;
	}
	}
	crm_trace("Online node count: %d", scheduler->max_valid_nodes);
	}

	/*
	* \internal
	* \brief Apply node-specific scheduling criteria
	*
	* After the CIB has been unpacked, process node-specific scheduling criteria
	* including shutdown locks, location constraints, resource stickiness,
	* migration thresholds, and exclusive resource discovery.
	*/
	static void
	apply_node_criteria(pcmk_scheduler_t *scheduler)
	{
	crm_trace("Applying node-specific scheduling criteria");
	apply_shutdown_locks(scheduler);
	count_available_nodes(scheduler);
	pcmk__apply_locations(scheduler);
	g_list_foreach(scheduler->resources, apply_stickiness, NULL);

	for (GList *node_iter = scheduler->nodes; node_iter != NULL;
	node_iter = node_iter->next) {
	for (GList *rsc_iter = scheduler->resources; rsc_iter != NULL;
	rsc_iter = rsc_iter->next) {
	check_failure_threshold(rsc_iter->data, node_iter->data);
	apply_exclusive_discovery(rsc_iter->data, node_iter->data);
	}
	}
	}

	/*!
	* \internal
	* \brief Assign resources to nodes
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	assign_resources(pcmk_scheduler_t *scheduler)
	{
	GList *iter = NULL;

	crm_trace("Assigning resources to nodes");

	if (!pcmk__str_eq(scheduler->placement_strategy, PCMK_VALUE_DEFAULT,
	pcmk__str_casei)) {
	pcmk__sort_resources(scheduler);
	}
	pcmk__show_node_capacities("Original", scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {
	/* Assign remote connection resources first (which will also assign any
	* colocation dependencies). If the connection is migrating, always
	* prefer the partial migration target.
	*/
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;
	const pcmk_node_t *target = rsc->private->partial_migration_target;

	if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
	pcmk__rsc_trace(rsc, "Assigning remote connection resource '%s'",
	rsc->id);
	rsc->private->cmds->assign(rsc, target, true);
	}
	}
	}

	/* now do the rest of the resources */
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
	pcmk__rsc_trace(rsc, "Assigning %s resource '%s'",
	rsc->private->xml->name, rsc->id);
	rsc->private->cmds->assign(rsc, NULL, true);
	}
	}

	pcmk__show_node_capacities("Remaining", scheduler);
	}

	/*!
	* \internal
	* \brief Schedule fail count clearing on online nodes if resource is orphaned
	*
	* \param[in,out] data Resource to check
	* \param[in] user_data Ignored
	*/
	static void
	clear_failcounts_if_orphaned(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
	return;
	}
	crm_trace("Clear fail counts for orphaned resource %s", rsc->id);

	/* There's no need to recurse into rsc->private->children because those
	* should just be unassigned clone instances.
	*/

	for (GList *iter = rsc->private->scheduler->nodes;
	iter != NULL; iter = iter->next) {

	pcmk_node_t node = (pcmk_node_t ) iter->data;
	pcmk_action_t *clear_op = NULL;

	if (!node->details->online) {
	continue;
	}
	if (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective, NULL) == 0) {
	continue;
	}

	clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
	rsc->private->scheduler);

	/* We can't use order_action_then_stop() here because its
	* pcmk__ar_guest_allowed breaks things
	*/
	pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc),
	NULL, pcmk__ar_ordered, rsc->private->scheduler);
	}
	}

	/*!
	* \internal
	* \brief Schedule any resource actions needed
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	schedule_resource_actions(pcmk_scheduler_t *scheduler)
	{
	// Process deferred action checks
	pe__foreach_param_check(scheduler, check_params);
	pe__free_param_checks(scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_trace("Scheduling probes");
	pcmk__schedule_probes(scheduler);
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	g_list_foreach(scheduler->resources, clear_failcounts_if_orphaned,
	NULL);
	}

	crm_trace("Scheduling resource actions");
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->private->cmds->create_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource or any of its descendants are managed
	*
	* \param[in] rsc Resource to check
	*
	* \return true if resource or any descendant is managed, otherwise false
	*/
	static bool
	is_managed(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	return true;
	}
	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	if (is_managed((pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether any resources in the cluster are managed
	*
	* \param[in] scheduler Scheduler data
	*
	* \return true if any resource is managed, otherwise false
	*/
	static bool
	any_managed_resources(const pcmk_scheduler_t *scheduler)
	{
	for (const GList *iter = scheduler->resources;
	iter != NULL; iter = iter->next) {
	if (is_managed((const pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node requires fencing
	*
	* \param[in] node Node to check
	* \param[in] have_managed Whether any resource in cluster is managed
	*
	* \return true if \p node should be fenced, otherwise false
	*/
	static bool
	needs_fencing(const pcmk_node_t *node, bool have_managed)
	{
	return have_managed && node->details->unclean
	&& pe_can_fence(node->private->scheduler, node);
	}

	/*!
	* \internal
	* \brief Check whether a node requires shutdown
	*
	* \param[in] node Node to check
	*
	* \return true if \p node should be shut down, otherwise false
	*/
	static bool
	needs_shutdown(const pcmk_node_t *node)
	{
	if (pcmk__is_pacemaker_remote_node(node)) {
	/* Do not send shutdown actions for Pacemaker Remote nodes.
	* @TODO We might come up with a good use for this in the future.
	*/
	return false;
	}
	return node->details->online && node->details->shutdown;
	}

	/*!
	* \internal
	* \brief Track and order non-DC fencing
	*
	* \param[in,out] list List of existing non-DC fencing actions
	* \param[in,out] action Fencing action to prepend to \p list
	* \param[in] scheduler Scheduler data
	*
	* \return (Possibly new) head of \p list
	*/
	static GList *
	add_nondc_fencing(GList list, pcmk_action_t action,
	const pcmk_scheduler_t *scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)
	&& (list != NULL)) {
	/* Concurrent fencing is disabled, so order each non-DC
	* fencing in a chain. If there is any DC fencing or
	* shutdown, it will be ordered after the last action in the
	* chain later.
	*/
	order_actions((pcmk_action_t *) list->data, action, pcmk__ar_ordered);
	}
	return g_list_prepend(list, action);
	}

	/*!
	* \internal
	* \brief Schedule a node for fencing
	*
	* \param[in,out] node Node that requires fencing
	*/
	static pcmk_action_t *
	schedule_fencing(pcmk_node_t *node)
	{
	pcmk_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean",
	FALSE, node->private->scheduler);

	pcmk__sched_warn("Scheduling node %s for fencing", pcmk__node_name(node));
	pcmk__order_vs_fence(fencing, node->private->scheduler);
	return fencing;
	}

	/*!
	* \internal
	* \brief Create and order node fencing and shutdown actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	schedule_fencing_and_shutdowns(pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *dc_down = NULL;
	bool integrity_lost = false;
	bool have_managed = any_managed_resources(scheduler);
	GList *fencing_ops = NULL;
	GList *shutdown_ops = NULL;

	crm_trace("Scheduling fencing and shutdowns as needed");
	if (!have_managed) {
	crm_notice("No fencing will be done until there are resources "
	"to manage");
	}

	// Check each node for whether it needs fencing or shutdown
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	pcmk_action_t *fencing = NULL;
	const bool is_dc = pcmk__same_node(node, scheduler->dc_node);

	/* Guest nodes are "fenced" by recovering their container resource,
	* so handle them separately.
	*/
	if (pcmk__is_guest_or_bundle_node(node)) {
	if (pcmk_is_set(node->private->flags, pcmk__node_remote_reset)
	&& have_managed && pe_can_fence(scheduler, node)) {
	pcmk__fence_guest(node);
	}
	continue;
	}

	if (needs_fencing(node, have_managed)) {
	fencing = schedule_fencing(node);

	// Track DC and non-DC fence actions separately
	if (is_dc) {
	dc_down = fencing;
	} else {
	fencing_ops = add_nondc_fencing(fencing_ops, fencing,
	scheduler);
	}

	} else if (needs_shutdown(node)) {
	pcmk_action_t *down_op = pcmk__new_shutdown_action(node);

	// Track DC and non-DC shutdown actions separately
	if (is_dc) {
	dc_down = down_op;
	} else {
	shutdown_ops = g_list_prepend(shutdown_ops, down_op);
	}
	}

	if ((fencing == NULL) && node->details->unclean) {
	integrity_lost = true;
	pcmk__config_warn("Node %s is unclean but cannot be fenced",
	pcmk__node_name(node));
	}
	}

	if (integrity_lost) {
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Resource functionality and data integrity "
	"cannot be guaranteed (configure, enable, "
	"and test fencing to correct this)");

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	crm_notice("Unclean nodes will not be fenced until quorum is "
	"attained or " PCMK_OPT_NO_QUORUM_POLICY " is set to "
	PCMK_VALUE_IGNORE);
	}
	}

	if (dc_down != NULL) {
	/* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
	* DC elections. However, we don't want to order non-DC shutdowns before
	* a DC fencing, because even though we don't want a node that's
	* shutting down to become DC, the DC fencing could be ordered before a
	* clone stop that's also ordered before the shutdowns, thus leading to
	* a graph loop.
	*/
	if (pcmk__str_eq(dc_down->task, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_none)) {
	pcmk__order_after_each(dc_down, shutdown_ops);
	}

	// Order any non-DC fencing before any DC fencing or shutdown

	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	/* With concurrent fencing, order each non-DC fencing action
	* separately before any DC fencing or shutdown.
	*/
	pcmk__order_after_each(dc_down, fencing_ops);
	} else if (fencing_ops != NULL) {
	/* Without concurrent fencing, the non-DC fencing actions are
	* already ordered relative to each other, so we just need to order
	* the DC fencing after the last action in the chain (which is the
	* first item in the list).
	*/
	order_actions((pcmk_action_t *) fencing_ops->data, dc_down,
	pcmk__ar_ordered);
	}
	}
	g_list_free(fencing_ops);
	g_list_free(shutdown_ops);
	}

	static void
	log_resource_details(pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	GList *all = NULL;

	/* Due to the `crm_mon --node=` feature, out->message() for all the
	* resource-related messages expects a list of nodes that we are allowed to
	* output information for. Here, we create a wildcard to match all nodes.
	*/
	all = g_list_prepend(all, (gpointer) "*");

	for (GList *item = scheduler->resources; item != NULL; item = item->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) item->data;

	// Log all resources except inactive orphans
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)
	\|\| (rsc->private->orig_role != pcmk_role_stopped)) {
	out->message(out, pcmk__map_element_name(rsc->private->xml), 0UL,
	rsc, all, all);
	}
	}

	g_list_free(all);
	}

	static void
	log_all_actions(pcmk_scheduler_t *scheduler)
	{
	/* This only ever outputs to the log, so ignore whatever output object was
	* previously set and just log instead.
	*/
	pcmk__output_t *prev_out = scheduler->priv;
	pcmk__output_t *out = NULL;

	if (pcmk__log_output_new(&out) != pcmk_rc_ok) {
	return;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);
	pcmk__output_set_log_level(out, LOG_NOTICE);
	scheduler->priv = out;

	out->begin_list(out, NULL, NULL, "Actions");
	pcmk__output_actions(scheduler);
	out->end_list(out);
	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);

	scheduler->priv = prev_out;
	}

	/*!
	* \internal
	* \brief Log all required but unrunnable actions at trace level
	*
	* \param[in] scheduler Scheduler data
	*/
	static void
	log_unrunnable_actions(const pcmk_scheduler_t *scheduler)
	{
	const uint64_t flags = pcmk_action_optional
	\|pcmk_action_runnable
	\|pcmk_action_pseudo;

	crm_trace("Required but unrunnable actions:");
	for (const GList *iter = scheduler->actions;
	iter != NULL; iter = iter->next) {

	const pcmk_action_t action = (const pcmk_action_t ) iter->data;

	if (!pcmk_any_flags_set(action->flags, flags)) {
	pcmk__log_action("\t", action, true);
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack the CIB for scheduling
	*
	* \param[in,out] cib CIB XML to unpack (may be NULL if already unpacked)
	* \param[in] flags Scheduler flags to set in addition to defaults
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_cib(xmlNode cib, unsigned long long flags, pcmk_scheduler_t scheduler)
	{
	const char* localhost_save = NULL;

	if (pcmk_is_set(scheduler->flags, pcmk_sched_have_status)) {
	crm_trace("Reusing previously calculated cluster status");
	pcmk__set_scheduler_flags(scheduler, flags);
	return;
	}

	if (scheduler->localhost) {
	localhost_save = scheduler->localhost;
	}

	CRM_ASSERT(cib != NULL);
	crm_trace("Calculating cluster status");

	/* This will zero the entire struct without freeing anything first, so
	* callers should never call pcmk__schedule_actions() with a populated data
	* set unless pcmk_sched_have_status is set (i.e. cluster_status() was
	* previously called, whether directly or via pcmk__schedule_actions()).
	*/
	set_working_set_defaults(scheduler);

	if (localhost_save) {
	scheduler->localhost = localhost_save;
	}

	pcmk__set_scheduler_flags(scheduler, flags);
	scheduler->input = cib;
	cluster_status(scheduler); // Sets pcmk_sched_have_status
	}

	/*!
	* \internal
	* \brief Run the scheduler for a given CIB
	*
	* \param[in,out] cib CIB XML to use as scheduler input
	* \param[in] flags Scheduler flags to set in addition to defaults
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__schedule_actions(xmlNode *cib, unsigned long long flags,
	pcmk_scheduler_t *scheduler)
	{
	unpack_cib(cib, flags, scheduler);
	pcmk__set_assignment_methods(scheduler);
	pcmk__apply_node_health(scheduler);
	pcmk__unpack_constraints(scheduler);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_validate_only)) {
	return;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)
	&& pcmk__is_daemon) {
	log_resource_details(scheduler);
	}

	apply_node_criteria(scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	return;
	}

	pcmk__create_internal_constraints(scheduler);
	pcmk__handle_rsc_config_changes(scheduler);
	assign_resources(scheduler);
	schedule_resource_actions(scheduler);

	/* Remote ordering constraints need to happen prior to calculating fencing
	* because it is one more place we can mark nodes as needing fencing.
	*/
	pcmk__order_remote_connection_actions(scheduler);

	schedule_fencing_and_shutdowns(scheduler);
	pcmk__apply_orderings(scheduler);
	log_all_actions(scheduler);
	pcmk__create_graph(scheduler);

	if (get_crm_log_level() == LOG_TRACE) {
	log_unrunnable_actions(scheduler);
	}
	}

	/*!
	* \internal
	* \brief Initialize scheduler data
	*
	* Make our own copies of the CIB XML and date/time object, if they're not
	* \c NULL. This way we don't have to take ownership of the objects passed via
	* the API.
	*
	* This function is most useful for public API functions that want the caller
	* to retain ownership of the CIB object
	*
	* \param[in,out] out Output object
	* \param[in] input The CIB XML to check (if \c NULL, use current CIB)
	* \param[in] date Date and time to use in the scheduler (if \c NULL,
	* use current date and time). This can be used for
	* checking whether a rule is in effect at a certa
	* date and time.
	* \param[out] scheduler Where to store initialized scheduler data
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__init_scheduler(pcmk__output_t out, xmlNodePtr input, const crm_time_t date,
	pcmk_scheduler_t **scheduler)
	{
	// Allows for cleaner syntax than dereferencing the scheduler argument
	pcmk_scheduler_t *new_scheduler = NULL;

	new_scheduler = pe_new_working_set();
	if (new_scheduler == NULL) {
	return ENOMEM;
	}

	pcmk__set_scheduler_flags(new_scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);

	// Populate the scheduler data

	// Make our own copy of the given input or fetch the CIB and use that
	if (input != NULL) {
	new_scheduler->input = pcmk__xml_copy(NULL, input);
	if (new_scheduler->input == NULL) {
	out->err(out, "Failed to copy input XML");
	pe_free_working_set(new_scheduler);
	return ENOMEM;
	}

	} else {
	int rc = cib__signon_query(out, NULL, &(new_scheduler->input));

	if (rc != pcmk_rc_ok) {
	pe_free_working_set(new_scheduler);
	return rc;
	}
	}

	// Make our own copy of the given crm_time_t object; otherwise
	// cluster_status() populates with the current time
	if (date != NULL) {
	// pcmk_copy_time() guarantees non-NULL
	new_scheduler->now = pcmk_copy_time(date);
	}

	// Unpack everything
	cluster_status(new_scheduler);
	*scheduler = new_scheduler;

	return pcmk_rc_ok;
	}
	diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
	index 555a8d8ba8..7eff081eec 100644
	--- a/lib/pengine/bundle.c
	+++ b/lib/pengine/bundle.c
	@@ -1,2149 +1,2149 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <ctype.h>
	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	enum pe__bundle_mount_flags {
	pe__bundle_mount_none = 0x00,

	// mount instance-specific subdirectory rather than source directly
	pe__bundle_mount_subdir = 0x01
	};

	typedef struct {
	char *source;
	char *target;
	char *options;
	uint32_t flags; // bitmask of pe__bundle_mount_flags
	} pe__bundle_mount_t;

	typedef struct {
	char *source;
	char *target;
	} pe__bundle_port_t;

	enum pe__container_agent {
	PE__CONTAINER_AGENT_UNKNOWN,
	PE__CONTAINER_AGENT_DOCKER,
	PE__CONTAINER_AGENT_RKT,
	PE__CONTAINER_AGENT_PODMAN,
	};

	#define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
	#define PE__CONTAINER_AGENT_DOCKER_S "docker"
	#define PE__CONTAINER_AGENT_RKT_S "rkt"
	#define PE__CONTAINER_AGENT_PODMAN_S "podman"

	typedef struct pe__bundle_variant_data_s {
	int promoted_max;
	int nreplicas;
	int nreplicas_per_host;
	char *prefix;
	char *image;
	const char *ip_last;
	char *host_network;
	char *host_netmask;
	char *control_port;
	char *container_network;
	char *ip_range_start;
	gboolean add_host;
	gchar *container_host_options;
	char *container_command;
	char *launcher_options;
	const char *attribute_target;

	pcmk_resource_t *child;

	GList replicas; // pcmk__bundle_replica_t
	GList ports; // pe__bundle_port_t
	GList mounts; // pe__bundle_mount_t

	enum pe__container_agent agent_type;
	} pe__bundle_variant_data_t;

	#define get_bundle_variant_data(data, rsc) do { \
	CRM_ASSERT(pcmk__is_bundle(rsc)); \
	data = rsc->private->variant_opaque; \
	} while (0)

	/*!
	* \internal
	* \brief Get maximum number of bundle replicas allowed to run
	*
	* \param[in] rsc Bundle or bundled resource to check
	*
	* \return Maximum replicas for bundle corresponding to \p rsc
	*/
	int
	pe__bundle_max(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->nreplicas;
	}

	/*!
	* \internal
	* \brief Get the resource inside a bundle
	*
	* \param[in] bundle Bundle to check
	*
	* \return Resource inside \p bundle if any, otherwise NULL
	*/
	pcmk_resource_t *
	pe__bundled_resource(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->child;
	}

	/*!
	* \internal
	* \brief Get containerized resource corresponding to a given bundle container
	*
	* \param[in] instance Collective instance that might be a bundle container
	*
	* \return Bundled resource instance inside \p instance if it is a bundle
	* container instance, otherwise NULL
	*/
	const pcmk_resource_t *
	pe__get_rsc_in_container(const pcmk_resource_t *instance)
	{
	const pe__bundle_variant_data_t *data = NULL;
	const pcmk_resource_t *top = pe__const_top_resource(instance, true);

	if (!pcmk__is_bundle(top)) {
	return NULL;
	}
	get_bundle_variant_data(data, top);

	for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	const pcmk__bundle_replica_t *replica = iter->data;

	if (instance == replica->container) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a given node is created by a bundle
	*
	* \param[in] bundle Bundle resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is an instance of \p bundle, otherwise false
	*/
	bool
	pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (pcmk__same_node(node, replica->node)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Get the container of a bundle's first replica
	*
	* \param[in] bundle Bundle resource to get container for
	*
	* \return Container resource from first replica of \p bundle if any,
	* otherwise NULL
	*/
	pcmk_resource_t *
	pe__first_container(const pcmk_resource_t *bundle)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;
	const pcmk__bundle_replica_t *replica = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	if (bundle_data->replicas == NULL) {
	return NULL;
	}
	replica = bundle_data->replicas->data;
	return replica->container;
	}

	/*!
	* \internal
	* \brief Iterate over bundle replicas
	*
	* \param[in,out] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Iterate over const bundle replicas
	*
	* \param[in] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (const GList *iter = bundle_data->replicas; iter != NULL;
	iter = iter->next) {

	if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	static char *
	next_ip(const char *last_ip)
	{
	unsigned int oct1 = 0;
	unsigned int oct2 = 0;
	unsigned int oct3 = 0;
	unsigned int oct4 = 0;
	int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);

	if (rc != 4) {
	/@ TODO check for IPv6 /
	return NULL;

	} else if (oct3 > 253) {
	return NULL;

	} else if (oct4 > 253) {
	++oct3;
	oct4 = 1;

	} else {
	++oct4;
	}

	return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
	}

	static void
	allocate_ip(pe__bundle_variant_data_t data, pcmk__bundle_replica_t replica,
	GString *buffer)
	{
	if(data->ip_range_start == NULL) {
	return;

	} else if(data->ip_last) {
	replica->ipaddr = next_ip(data->ip_last);

	} else {
	replica->ipaddr = strdup(data->ip_range_start);
	}

	data->ip_last = replica->ipaddr;
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (data->add_host) {
	g_string_append_printf(buffer, " --add-host=%s-%d:%s",
	data->prefix, replica->offset,
	replica->ipaddr);
	} else {
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	}
	break;

	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	break;

	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	}

	static xmlNode *
	create_resource(const char name, const char provider, const char *kind)
	{
	xmlNode *rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);

	crm_xml_add(rsc, PCMK_XA_ID, name);
	crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
	crm_xml_add(rsc, PCMK_XA_TYPE, kind);

	return rsc;
	}

	/*!
	* \internal
	* \brief Check whether cluster can manage resource inside container
	*
	* \param[in,out] data Container variant data
	*
	* \return TRUE if networking configuration is acceptable, FALSE otherwise
	*
	* \note The resource is manageable if an IP range or control port has been
	* specified. If a control port is used without an IP range, replicas per
	* host must be 1.
	*/
	static bool
	valid_network(pe__bundle_variant_data_t *data)
	{
	if(data->ip_range_start) {
	return TRUE;
	}
	if(data->control_port) {
	if(data->nreplicas_per_host > 1) {
	pcmk__config_err("Specifying the '" PCMK_XA_CONTROL_PORT "' for %s "
	"requires '" PCMK_XA_REPLICAS_PER_HOST "=1'",
	data->prefix);
	data->nreplicas_per_host = 1;
	// @TODO to be sure:
	// pcmk__clear_rsc_flags(rsc, pcmk__rsc_unique);
	}
	return TRUE;
	}
	return FALSE;
	}

	static int
	create_ip_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if(data->ip_range_start) {
	char *id = NULL;
	xmlNode *xml_ip = NULL;
	xmlNode *xml_obj = NULL;

	id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
	pcmk__xml_sanitize_id(id);
	xml_ip = create_resource(id, "heartbeat", "IPaddr2");
	free(id);

	xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_INSTANCE_ATTRIBUTES);
	pcmk__xe_set_id(xml_obj, "%s-attributes-%d",
	data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
	if(data->host_network) {
	crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
	}

	if(data->host_netmask) {
	crm_create_nvpair_xml(xml_obj, NULL,
	"cidr_netmask", data->host_netmask);

	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
	}

	xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_OPERATIONS);
	crm_create_op_xml(xml_obj, pcmk__xe_id(xml_ip), PCMK_ACTION_MONITOR,
	"60s", NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?

	if (pe__unpack_resource(xml_ip, &replica->ip, parent,
	parent->private->scheduler) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	parent->private->children = g_list_append(parent->private->children,
	replica->ip);
	}
	return pcmk_rc_ok;
	}

	static const char*
	container_agent_str(enum pe__container_agent t)
	{
	switch (t) {
	case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
	case PE__CONTAINER_AGENT_RKT: return PE__CONTAINER_AGENT_RKT_S;
	case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	return PE__CONTAINER_AGENT_UNKNOWN_S;
	}

	static int
	create_container_resource(pcmk_resource_t *parent,
	const pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	char *id = NULL;
	xmlNode *xml_container = NULL;
	xmlNode *xml_obj = NULL;

	// Agent-specific
	const char *hostname_opt = NULL;
	const char *env_opt = NULL;
	const char *agent_str = NULL;
	int volid = 0; // rkt-only

	GString *buffer = NULL;
	GString *dbuffer = NULL;

	// Where syntax differences are drop-in replacements, set them now
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	hostname_opt = "-h ";
	env_opt = "-e ";
	break;
	case PE__CONTAINER_AGENT_RKT:
	hostname_opt = "--hostname=";
	env_opt = "--environment=";
	break;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	return pcmk_rc_unpack_error;
	}
	agent_str = container_agent_str(data->agent_type);

	buffer = g_string_sized_new(4096);

	id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
	replica->offset);
	pcmk__xml_sanitize_id(id);
	xml_container = create_resource(id, "heartbeat", agent_str);
	free(id);

	xml_obj = pcmk__xe_create(xml_container, PCMK_XE_INSTANCE_ATTRIBUTES);
	pcmk__xe_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
	crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
	crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
	crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);

	if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
	g_string_append(buffer, " --restart=no");
	}

	/* Set a container hostname only if we have an IP to map it to. The user can
	* set -h or --uts=host themselves if they want a nicer name for logs, but
	* this makes applications happy who need their hostname to match the IP
	* they bind to.
	*/
	if (data->ip_range_start != NULL) {
	g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
	replica->offset);
	}
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);

	if (data->container_network != NULL) {
	pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
	}

	if (data->control_port != NULL) {
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
	data->control_port, NULL);
	} else {
	g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
	env_opt, DEFAULT_REMOTE_PORT);
	}

	for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
	pe__bundle_mount_t mount = (pe__bundle_mount_t ) iter->data;
	char *source = NULL;

	if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
	source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
	replica->offset);
	pcmk__add_separated_word(&dbuffer, 1024, source, ",");
	}

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	pcmk__g_strcat(buffer,
	" -v ", pcmk__s(source, mount->source),
	":", mount->target, NULL);

	if (mount->options != NULL) {
	pcmk__g_strcat(buffer, ":", mount->options, NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer,
	" --volume vol%d,kind=host,"
	"source=%s%s%s "
	"--mount volume=vol%d,target=%s",
	volid, pcmk__s(source, mount->source),
	(mount->options != NULL)? "," : "",
	pcmk__s(mount->options, ""),
	volid, mount->target);
	volid++;
	break;
	default:
	break;
	}
	free(source);
	}

	for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
	pe__bundle_port_t port = (pe__bundle_port_t ) iter->data;

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" -p ", replica->ipaddr, ":", port->source,
	":", port->target, NULL);

	} else if (!pcmk__str_eq(data->container_network,
	PCMK_VALUE_HOST, pcmk__str_none)) {
	// No need to do port mapping if net == host
	pcmk__g_strcat(buffer,
	" -p ", port->source, ":", port->target,
	NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" --port=", port->target,
	":", replica->ipaddr, ":", port->source,
	NULL);
	} else {
	pcmk__g_strcat(buffer,
	" --port=", port->target, ":", port->source,
	NULL);
	}
	break;
	default:
	break;
	}
	}

	/* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
	* it would cause restarts during rolling upgrades.
	*
	* In a previous version of the container resource creation logic, if
	* data->launcher_options is not NULL, we append
	* (" %s", data->launcher_options) even if data->launcher_options is an
	* empty string. Likewise for data->container_host_options. Using
	*
	* pcmk__add_word(buffer, 0, data->launcher_options)
	*
	* removes that extra trailing space, causing a resource definition change.
	*/
	if (data->launcher_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
	}

	if (data->container_host_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
	}

	crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
	(const char *) buffer->str);
	g_string_free(buffer, TRUE);

	crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
	(dbuffer != NULL)? (const char *) dbuffer->str : "");
	if (dbuffer != NULL) {
	g_string_free(dbuffer, TRUE);
	}

	if (replica->child != NULL) {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	SBIN_DIR "/pacemaker-remoted");
	}

	/* TODO: Allow users to specify their own?
	*
	* We just want to know if the container is alive; we'll monitor the
	* child independently.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	#if 0
	/* @TODO Consider supporting the use case where we can start and stop
	* resources, but not proxy local commands (such as setting node
	* attributes), by running the local executor in stand-alone mode.
	* However, this would probably be better done via ACLs as with other
	* Pacemaker Remote nodes.
	*/
	} else if ((child != NULL) && data->untrusted) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	CRM_DAEMON_DIR "/pacemaker-execd");
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
	CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
	#endif
	} else {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	}

	/* TODO: Allow users to specify their own?
	*
	* We don't know what's in the container, so we just want to know if it
	* is alive.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	}

	xml_obj = pcmk__xe_create(xml_container, PCMK_XE_OPERATIONS);
	crm_create_op_xml(xml_obj, pcmk__xe_id(xml_container), PCMK_ACTION_MONITOR,
	"60s", NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?
	if (pe__unpack_resource(xml_container, &replica->container, parent,
	parent->private->scheduler) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}
	pcmk__set_rsc_flags(replica->container, pcmk__rsc_replica_container);
	parent->private->children = g_list_append(parent->private->children,
	replica->container);

	return pcmk_rc_ok;
	}

	/*!
	* \brief Ban a node from a resource's (and its children's) allowed nodes list
	*
	* \param[in,out] rsc Resource to modify
	* \param[in] uname Name of node to ban
	*/
	static void
	disallow_node(pcmk_resource_t rsc, const char uname)
	{
	gpointer match = g_hash_table_lookup(rsc->private->allowed_nodes, uname);

	if (match) {
	- ((pcmk_node_t *) match)->weight = -PCMK_SCORE_INFINITY;
	- ((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
	+ ((pcmk_node_t *) match)->assign->score = -PCMK_SCORE_INFINITY;
	+ ((pcmk_node_t *) match)->assign->probe_mode = pcmk__probe_never;
	}
	g_list_foreach(rsc->private->children, (GFunc) disallow_node,
	(gpointer) uname);
	}

	static int
	create_remote_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if (replica->child && valid_network(data)) {
	GHashTableIter gIter;
	pcmk_node_t *node = NULL;
	xmlNode *xml_remote = NULL;
	char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
	char *port_s = NULL;
	const char *uname = NULL;
	const char *connect_name = NULL;

	if (pe_find_resource(parent->private->scheduler->resources,
	id) != NULL) {
	free(id);
	// The biggest hammer we have
	id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
	replica->child->id, replica->offset);
	//@TODO return error instead of asserting?
	CRM_ASSERT(pe_find_resource(parent->private->scheduler->resources,
	id) == NULL);
	}

	/* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
	* connection does not have its own IP is a magic string that we use to
	* support nested remotes (i.e. a bundle running on a remote node).
	*/
	connect_name = (replica->ipaddr? replica->ipaddr : "#uname");

	if (data->control_port == NULL) {
	port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}

	/* This sets replica->container as replica->remote's container, which is
	* similar to what happens with guest nodes. This is how the scheduler
	* knows that the bundle node is fenced by recovering the container, and
	* that remote should be ordered relative to the container.
	*/
	xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
	NULL, NULL, NULL,
	connect_name, (data->control_port?
	data->control_port : port_s));
	free(port_s);

	/* Abandon our created ID, and pull the copy from the XML, because we
	* need something that will get freed during scheduler data cleanup to
	* use as the node ID and uname.
	*/
	free(id);
	id = NULL;
	uname = pcmk__xe_id(xml_remote);

	/* Ensure a node has been created for the guest (it may have already
	* been, if it has a permanent node attribute), and ensure its weight is
	* -INFINITY so no other resources can run on it.
	*/
	node = pcmk_find_node(parent->private->scheduler, uname);
	if (node == NULL) {
	node = pe_create_node(uname, uname, PCMK_VALUE_REMOTE,
	PCMK_VALUE_MINUS_INFINITY,
	parent->private->scheduler);
	} else {
	- node->weight = -PCMK_SCORE_INFINITY;
	+ node->assign->score = -PCMK_SCORE_INFINITY;
	}
	- node->rsc_discover_mode = pcmk_probe_never;
	+ node->assign->probe_mode = pcmk__probe_never;

	/* unpack_remote_nodes() ensures that each remote node and guest node
	* has a pcmk_node_t entry. Ideally, it would do the same for bundle
	* nodes. Unfortunately, a bundle has to be mostly unpacked before it's
	* obvious what nodes will be needed, so we do it just above.
	*
	* Worse, that means that the node may have been utilized while
	* unpacking other resources, without our weight correction. The most
	* likely place for this to happen is when pe__unpack_resource() calls
	* resource_location() to set a default score in symmetric clusters.
	* This adds a node copy to each resource's allowed nodes, and these
	* copies will have the wrong weight.
	*
	* As a hacky workaround, fix those copies here.
	*
	* @TODO Possible alternative: ensure bundles are unpacked before other
	* resources, so the weight is correct before any copies are made.
	*/
	g_list_foreach(parent->private->scheduler->resources,
	(GFunc) disallow_node, (gpointer) uname);

	replica->node = pe__copy_node(node);
	- replica->node->weight = 500;
	- replica->node->rsc_discover_mode = pcmk_probe_exclusive;
	+ replica->node->assign->score = 500;
	+ replica->node->assign->probe_mode = pcmk__probe_exclusive;

	/* Ensure the node shows up as allowed and with the correct discovery set */
	if (replica->child->private->allowed_nodes != NULL) {
	g_hash_table_destroy(replica->child->private->allowed_nodes);
	}
	replica->child->private->allowed_nodes = pcmk__strkey_table(NULL, free);
	g_hash_table_insert(replica->child->private->allowed_nodes,
	(gpointer) replica->node->private->id,
	pe__copy_node(replica->node));

	{
	const pcmk_resource_t *parent = replica->child->private->parent;
	pcmk_node_t *copy = pe__copy_node(replica->node);

	- copy->weight = -PCMK_SCORE_INFINITY;
	+ copy->assign->score = -PCMK_SCORE_INFINITY;
	g_hash_table_insert(parent->private->allowed_nodes,
	(gpointer) replica->node->private->id, copy);
	}
	if (pe__unpack_resource(xml_remote, &replica->remote, parent,
	parent->private->scheduler) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	g_hash_table_iter_init(&gIter, replica->remote->private->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
	if (pcmk__is_pacemaker_remote_node(node)) {
	/* Remote resources can only run on 'normal' cluster node */
	- node->weight = -PCMK_SCORE_INFINITY;
	+ node->assign->score = -PCMK_SCORE_INFINITY;
	}
	}

	replica->node->private->remote = replica->remote;

	// Ensure pcmk__is_guest_or_bundle_node() functions correctly
	replica->remote->private->launcher = replica->container;

	/* A bundle's #kind is closer to "container" (guest node) than the
	* "remote" set by pe_create_node().
	*/
	pcmk__insert_dup(replica->node->private->attrs,
	CRM_ATTR_KIND, "container");

	/* One effect of this is that unpack_launcher() will add
	* replica->remote to replica->container's launched resources, which
	* will make pe__resource_contains_guest_node() true for
	* replica->container.
	*
	* replica->child does NOT get added to replica->container's launched
	* resources. The only noticeable effect if it did would be for its
	* fail count to be taken into account when checking
	* replica->container's migration threshold.
	*/
	parent->private->children = g_list_append(parent->private->children,
	replica->remote);
	}
	return pcmk_rc_ok;
	}

	static int
	create_replica_resources(pcmk_resource_t *parent,
	pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	int rc = pcmk_rc_ok;

	rc = create_container_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_ip_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_remote_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
	pcmk__insert_meta(replica->child->private, "external-ip",
	replica->ipaddr);
	}

	if (replica->remote != NULL) {
	/*
	* Allow the remote connection resource to be allocated to a
	* different node than the one on which the container is active.
	*
	* This makes it possible to have Pacemaker Remote nodes running
	* containers with pacemaker-remoted inside in order to start
	* services inside those containers.
	*/
	pcmk__set_rsc_flags(replica->remote, pcmk__rsc_remote_nesting_allowed);
	}
	return rc;
	}

	static void
	mount_add(pe__bundle_variant_data_t bundle_data, const char source,
	const char target, const char options, uint32_t flags)
	{
	pe__bundle_mount_t *mount = pcmk__assert_alloc(1,
	sizeof(pe__bundle_mount_t));

	mount->source = pcmk__str_copy(source);
	mount->target = pcmk__str_copy(target);
	mount->options = pcmk__str_copy(options);
	mount->flags = flags;
	bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
	}

	static void
	mount_free(pe__bundle_mount_t *mount)
	{
	free(mount->source);
	free(mount->target);
	free(mount->options);
	free(mount);
	}

	static void
	port_free(pe__bundle_port_t *port)
	{
	free(port->source);
	free(port->target);
	free(port);
	}

	static pcmk__bundle_replica_t *
	replica_for_remote(pcmk_resource_t *remote)
	{
	pcmk_resource_t *top = remote;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (top == NULL) {
	return NULL;
	}
	while (top->private->parent != NULL) {
	top = top->private->parent;
	}

	get_bundle_variant_data(bundle_data, top);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->remote == remote) {
	return replica;
	}
	}
	CRM_LOG_ASSERT(FALSE);
	return NULL;
	}

	bool
	pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
	{
	const char *value;
	GHashTable *params = NULL;

	if (rsc == NULL) {
	return false;
	}

	// Use NULL node since pcmk__bundle_expand() uses that to set value
	params = pe_rsc_params(rsc, NULL, rsc->private->scheduler);
	value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);

	return pcmk__str_eq(value, "#uname", pcmk__str_casei)
	&& xml_contains_remote_node(rsc->private->xml);
	}

	const char *
	pe__add_bundle_remote_name(pcmk_resource_t rsc, xmlNode xml,
	const char *field)
	{
	// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside

	pcmk_node_t *node = NULL;
	pcmk__bundle_replica_t *replica = NULL;

	if (!pe__bundle_needs_remote_name(rsc)) {
	return NULL;
	}

	replica = replica_for_remote(rsc);
	if (replica == NULL) {
	return NULL;
	}

	node = replica->container->private->assigned_node;
	if (node == NULL) {
	/* If it won't be running anywhere after the
	* transition, go with where it's running now.
	*/
	node = pcmk__current_node(replica->container);
	}

	if(node == NULL) {
	crm_trace("Cannot determine address for bundle connection %s", rsc->id);
	return NULL;
	}

	crm_trace("Setting address for bundle connection %s to bundle host %s",
	rsc->id, pcmk__node_name(node));
	if(xml != NULL && field != NULL) {
	crm_xml_add(xml, field, node->private->name);
	}

	return node->private->name;
	}

	#define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do { \
	flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Bundle mount", pcmk__xe_id(mount_xml), \
	flags, (flags_to_set), #flags_to_set); \
	} while (0)

	gboolean
	pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	xmlNode *xml_obj = NULL;
	const xmlNode *xml_child = NULL;
	xmlNode *xml_resource = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	bool need_log_mount = TRUE;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	bundle_data = pcmk__assert_alloc(1, sizeof(pe__bundle_variant_data_t));
	rsc->private->variant_opaque = bundle_data;
	bundle_data->prefix = strdup(rsc->id);

	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_DOCKER, NULL,
	NULL);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
	} else {
	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK__XE_RKT, NULL,
	NULL);
	if (xml_obj != NULL) {
	pcmk__warn_once(pcmk__wo_rkt,
	"Support for " PCMK__XE_RKT " in bundles "
	"(such as %s) is deprecated and will be "
	"removed in a future release", rsc->id);
	bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
	} else {
	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_PODMAN,
	NULL, NULL);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
	} else {
	return FALSE;
	}
	}
	}

	// Use 0 for default, minimum, and invalid PCMK_XA_PROMOTED_MAX
	value = crm_element_value(xml_obj, PCMK_XA_PROMOTED_MAX);
	if (value == NULL) {
	// @COMPAT deprecated since 2.0.0
	value = crm_element_value(xml_obj, PCMK__XA_PROMOTED_MAX_LEGACY);

	if (value != NULL) {
	pcmk__warn_once(pcmk__wo_bundle_master,
	"Support for the " PCMK__XA_PROMOTED_MAX_LEGACY
	" attribute (such as in %s) is deprecated and "
	"will be removed in a future release. Use "
	PCMK_XA_PROMOTED_MAX " instead.",
	rsc->id);
	}
	}
	pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);

	/* Default replicas to PCMK_XA_PROMOTED_MAX if it was specified and 1
	* otherwise
	*/
	value = crm_element_value(xml_obj, PCMK_XA_REPLICAS);
	if ((value == NULL) && (bundle_data->promoted_max > 0)) {
	bundle_data->nreplicas = bundle_data->promoted_max;
	} else {
	pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
	}

	/*
	* Communication between containers on the same host via the
	* floating IPs only works if the container is started with:
	* --userland-proxy=false --ip-masq=false
	*/
	value = crm_element_value(xml_obj, PCMK_XA_REPLICAS_PER_HOST);
	pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
	if (bundle_data->nreplicas_per_host == 1) {
	pcmk__clear_rsc_flags(rsc, pcmk__rsc_unique);
	}

	bundle_data->container_command =
	crm_element_value_copy(xml_obj, PCMK_XA_RUN_COMMAND);
	bundle_data->launcher_options = crm_element_value_copy(xml_obj,
	PCMK_XA_OPTIONS);
	bundle_data->image = crm_element_value_copy(xml_obj, PCMK_XA_IMAGE);
	bundle_data->container_network = crm_element_value_copy(xml_obj,
	PCMK_XA_NETWORK);

	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_NETWORK, NULL,
	NULL);
	if(xml_obj) {
	bundle_data->ip_range_start =
	crm_element_value_copy(xml_obj, PCMK_XA_IP_RANGE_START);
	bundle_data->host_netmask =
	crm_element_value_copy(xml_obj, PCMK_XA_HOST_NETMASK);
	bundle_data->host_network =
	crm_element_value_copy(xml_obj, PCMK_XA_HOST_INTERFACE);
	bundle_data->control_port =
	crm_element_value_copy(xml_obj, PCMK_XA_CONTROL_PORT);
	value = crm_element_value(xml_obj, PCMK_XA_ADD_HOST);
	if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
	bundle_data->add_host = TRUE;
	}

	for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_PORT_MAPPING,
	NULL, NULL);
	xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {

	pe__bundle_port_t *port =
	pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));

	port->source = crm_element_value_copy(xml_child, PCMK_XA_PORT);

	if(port->source == NULL) {
	port->source = crm_element_value_copy(xml_child, PCMK_XA_RANGE);
	} else {
	port->target = crm_element_value_copy(xml_child,
	PCMK_XA_INTERNAL_PORT);
	}

	if(port->source != NULL && strlen(port->source) > 0) {
	if(port->target == NULL) {
	port->target = strdup(port->source);
	}
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	} else {
	pcmk__config_err("Invalid " PCMK_XA_PORT " directive %s",
	pcmk__xe_id(xml_child));
	port_free(port);
	}
	}
	}

	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_STORAGE, NULL,
	NULL);
	for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_STORAGE_MAPPING,
	NULL, NULL);
	xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {

	const char *source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR);
	const char *target = crm_element_value(xml_child, PCMK_XA_TARGET_DIR);
	const char *options = crm_element_value(xml_child, PCMK_XA_OPTIONS);
	int flags = pe__bundle_mount_none;

	if (source == NULL) {
	source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR_ROOT);
	pe__set_bundle_mount_flags(xml_child, flags,
	pe__bundle_mount_subdir);
	}

	if (source && target) {
	mount_add(bundle_data, source, target, options, flags);
	if (strcmp(target, "/var/log") == 0) {
	need_log_mount = FALSE;
	}
	} else {
	pcmk__config_err("Invalid mount directive %s",
	pcmk__xe_id(xml_child));
	}
	}

	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_PRIMITIVE, NULL,
	NULL);
	if (xml_obj && valid_network(bundle_data)) {
	const char *suffix = NULL;
	char *value = NULL;
	xmlNode *xml_set = NULL;

	xml_resource = pcmk__xe_create(NULL, PCMK_XE_CLONE);

	/* @COMPAT We no longer use the <master> tag, but we need to keep it as
	* part of the resource name, so that bundles don't restart in a rolling
	* upgrade. (It also avoids needing to change regression tests.)
	*/
	suffix = (const char *) xml_resource->name;
	if (bundle_data->promoted_max > 0) {
	suffix = "master";
	}

	pcmk__xe_set_id(xml_resource, "%s-%s", bundle_data->prefix, suffix);

	xml_set = pcmk__xe_create(xml_resource, PCMK_XE_META_ATTRIBUTES);
	pcmk__xe_set_id(xml_set, "%s-%s-meta",
	bundle_data->prefix, xml_resource->name);

	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_ORDERED, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->nreplicas);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
	free(value);

	value = pcmk__itoa(bundle_data->nreplicas_per_host);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
	free(value);

	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
	pcmk__btoa(bundle_data->nreplicas_per_host > 1));

	if (bundle_data->promoted_max) {
	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->promoted_max);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
	free(value);
	}

	//crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
	pcmk__xml_copy(xml_resource, xml_obj);

	} else if(xml_obj) {
	pcmk__config_err("Cannot control %s inside %s without either "
	PCMK_XA_IP_RANGE_START " or " PCMK_XA_CONTROL_PORT,
	rsc->id, pcmk__xe_id(xml_obj));
	return FALSE;
	}

	if(xml_resource) {
	int lpc = 0;
	GList *childIter = NULL;
	pe__bundle_port_t *port = NULL;
	GString *buffer = NULL;

	if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
	scheduler) != pcmk_rc_ok) {
	return FALSE;
	}

	/* Currently, we always map the default authentication key location
	* into the same location inside the container.
	*
	* Ideally, we would respect the host's PCMK_authkey_location, but:
	* - it may be different on different nodes;
	* - the actual connection will do extra checking to make sure the key
	* file exists and is readable, that we can't do here on the DC
	* - tools such as crm_resource and crm_simulate may not have the same
	* environment variables as the cluster, causing operation digests to
	* differ
	*
	* Always using the default location inside the container is fine,
	* because we control the pacemaker_remote environment, and it avoids
	* having to pass another environment variable to the container.
	*
	* @TODO A better solution may be to have only pacemaker_remote use the
	* environment variable, and have the cluster nodes use a new
	* cluster option for key location. This would introduce the limitation
	* of the location being the same on all cluster nodes, but that's
	* reasonable.
	*/
	mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
	DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);

	if (need_log_mount) {
	mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
	pe__bundle_mount_subdir);
	}

	port = pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
	if(bundle_data->control_port) {
	port->source = strdup(bundle_data->control_port);
	} else {
	/* If we wanted to respect PCMK_remote_port, we could use
	* crm_default_remote_port() here and elsewhere in this file instead
	* of DEFAULT_REMOTE_PORT.
	*
	* However, it gains nothing, since we control both the container
	* environment and the connection resource parameters, and the user
	* can use a different port if desired by setting
	* PCMK_XA_CONTROL_PORT.
	*/
	port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}
	port->target = strdup(port->source);
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	buffer = g_string_sized_new(1024);
	for (childIter = bundle_data->child->private->children;
	childIter != NULL; childIter = childIter->next) {

	pcmk__bundle_replica_t *replica = NULL;

	replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
	replica->child = childIter->data;
	pcmk__set_rsc_flags(replica->child, pcmk__rsc_exclusive_probes);
	replica->offset = lpc++;

	// Ensure the child's notify gets set based on the underlying primitive's value
	if (pcmk_is_set(replica->child->flags, pcmk__rsc_notify)) {
	pcmk__set_rsc_flags(bundle_data->child, pcmk__rsc_notify);
	}

	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	bundle_data->attribute_target =
	g_hash_table_lookup(replica->child->private->meta,
	PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);

	if (bundle_data->attribute_target) {
	pcmk__insert_dup(rsc->private->meta,
	PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
	bundle_data->attribute_target);
	pcmk__insert_dup(bundle_data->child->private->meta,
	PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
	bundle_data->attribute_target);
	}

	} else {
	// Just a naked container, no pacemaker-remote
	GString *buffer = g_string_sized_new(1024);

	for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
	pcmk__bundle_replica_t *replica = NULL;

	replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
	replica->offset = lpc;
	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
	pcmk__config_err("Failed unpacking resource %s", rsc->id);
	rsc->private->fns->free(rsc);
	return FALSE;
	}

	/* Utilization needs special handling for bundles. It makes no sense for
	* the inner primitive to have utilization, because it is tied
	* one-to-one to the guest node created by the container resource -- and
	* there's no way to set capacities for that guest node anyway.
	*
	* What the user really wants is to configure utilization for the
	* container. However, the schema only allows utilization for
	* primitives, and the container resource is implicit anyway, so the
	* user can only configure utilization for the inner primitive. If
	* they do, move the primitive's utilization values to the container.
	*
	* @TODO This means that bundles without an inner primitive can't have
	* utilization. An alternative might be to allow utilization values in
	* the top-level bundle XML in the schema, and copy those to each
	* container.
	*/
	if (replica->child != NULL) {
	GHashTable *empty = replica->container->private->utilization;

	replica->container->private->utilization =
	replica->child->private->utilization;

	replica->child->private->utilization = empty;
	}
	}

	if (bundle_data->child) {
	rsc->private->children = g_list_append(rsc->private->children,
	bundle_data->child);
	}
	return TRUE;
	}

	static int
	replica_resource_active(pcmk_resource_t *rsc, gboolean all)
	{
	if (rsc) {
	gboolean child_active = rsc->private->fns->active(rsc, all);

	if (child_active && !all) {
	return TRUE;
	} else if (!child_active && all) {
	return FALSE;
	}
	}
	return -1;
	}

	gboolean
	pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	GList *iter = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;
	int rsc_active;

	rsc_active = replica_resource_active(replica->ip, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->child, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->container, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->remote, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}
	}

	/* If "all" is TRUE, we've already checked that no resources were inactive,
	* so return TRUE; if "all" is FALSE, we didn't find any active resources,
	* so return FALSE.
	*/
	return all;
	}

	/*!
	* \internal
	* \brief Find the bundle replica corresponding to a given node
	*
	* \param[in] bundle Top-level bundle resource
	* \param[in] node Node to search for
	*
	* \return Bundle replica if found, NULL otherwise
	*/
	pcmk_resource_t *
	pe__find_bundle_replica(const pcmk_resource_t bundle, const pcmk_node_t node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_ASSERT(bundle && node);

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica && replica->node);
	if (pcmk__same_node(replica->node, node)) {
	return replica->child;
	}
	}
	return NULL;
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	const char *desc = NULL;

	CRM_ASSERT(rsc != NULL);
	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	pcmk_resource_t *ip = replica->ip;
	pcmk_resource_t *child = replica->child;
	pcmk_resource_t *container = replica->container;
	pcmk_resource_t *remote = replica->remote;
	char *id = NULL;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(container, only_node)) {
	continue;
	}

	print_ip = (ip != NULL)
	&& !ip->private->fns->is_filtered(ip, only_rsc,
	print_everything);
	print_child = (child != NULL)
	&& !child->private->fns->is_filtered(child, only_rsc,
	print_everything);
	print_ctnr = !container->private->fns->is_filtered(container, only_rsc,
	print_everything);
	print_remote = (remote != NULL)
	&& !remote->private->fns->is_filtered(remote, only_rsc,
	print_everything);

	if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
	continue;
	}

	if (!printed_header) {
	const char *type = container_agent_str(bundle_data->agent_type);
	const char *unique = pcmk__flag_text(rsc->flags, pcmk__rsc_unique);
	const char *maintenance = pcmk__flag_text(rsc->flags,
	pcmk__rsc_maintenance);
	const char *managed = pcmk__flag_text(rsc->flags,
	pcmk__rsc_managed);
	const char *failed = pcmk__flag_text(rsc->flags, pcmk__rsc_failed);

	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_BUNDLE,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_TYPE, type,
	PCMK_XA_IMAGE, bundle_data->image,
	PCMK_XA_UNIQUE, unique,
	PCMK_XA_MAINTENANCE, maintenance,
	PCMK_XA_MANAGED, managed,
	PCMK_XA_FAILED, failed,
	PCMK_XA_DESCRIPTION, desc,
	NULL);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	id = pcmk__itoa(replica->offset);
	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_REPLICA,
	PCMK_XA_ID, id,
	NULL);
	free(id);
	CRM_ASSERT(rc == pcmk_rc_ok);

	if (print_ip) {
	out->message(out, (const char *) ip->private->xml->name, show_opts,
	ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, (const char *) child->private->xml->name,
	show_opts, child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, (const char *) container->private->xml->name,
	show_opts, container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, (const char *) remote->private->xml->name,
	show_opts, remote, only_node, only_rsc);
	}

	pcmk__output_xml_pop_parent(out); // replica
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out); // bundle
	}

	return rc;
	}

	static void
	pe__bundle_replica_output_html(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_html(out, rsc, buffer, node, show_opts);
	}

	/*!
	* \internal
	* \brief Get a string describing a resource's unmanaged state or lack thereof
	*
	* \param[in] rsc Resource to describe
	*
	* \return A string indicating that a resource is in maintenance mode or
	* otherwise unmanaged, or an empty string otherwise
	*/
	static const char *
	get_unmanaged_str(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk__rsc_maintenance)) {
	return " (maintenance)";
	}
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	return " (unmanaged)";
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	CRM_ASSERT(rsc != NULL);
	get_bundle_variant_data(bundle_data, rsc);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	pcmk_resource_t *ip = replica->ip;
	pcmk_resource_t *child = replica->child;
	pcmk_resource_t *container = replica->container;
	pcmk_resource_t *remote = replica->remote;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(container, only_node)) {
	continue;
	}

	print_ip = (ip != NULL)
	&& !ip->private->fns->is_filtered(ip, only_rsc,
	print_everything);
	print_child = (child != NULL)
	&& !child->private->fns->is_filtered(child, only_rsc,
	print_everything);
	print_ctnr = !container->private->fns->is_filtered(container, only_rsc,
	print_everything);
	print_remote = (remote != NULL)
	&& !remote->private->fns->is_filtered(remote, only_rsc,
	print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
	}

	if (print_ip) {
	out->message(out, (const char *) ip->private->xml->name,
	new_show_opts, ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, (const char *) child->private->xml->name,
	new_show_opts, child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, (const char *) container->private->xml->name,
	new_show_opts, container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, (const char *) remote->private->xml->name,
	new_show_opts, remote, only_node, only_rsc);
	}

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->end_list(out);
	}
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_html(out, replica,
	pcmk__current_node(container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	pe__bundle_replica_output_text(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	const pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_text(out, rsc, buffer, node, show_opts);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	CRM_ASSERT(rsc != NULL);
	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	pcmk_resource_t *ip = replica->ip;
	pcmk_resource_t *child = replica->child;
	pcmk_resource_t *container = replica->container;
	pcmk_resource_t *remote = replica->remote;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(container, only_node)) {
	continue;
	}

	print_ip = (ip != NULL)
	&& !ip->private->fns->is_filtered(ip, only_rsc,
	print_everything);
	print_child = (child != NULL)
	&& !child->private->fns->is_filtered(child, only_rsc,
	print_everything);
	print_ctnr = !container->private->fns->is_filtered(container, only_rsc,
	print_everything);
	print_remote = (remote != NULL)
	&& !remote->private->fns->is_filtered(remote, only_rsc,
	print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->list_item(out, NULL, "Replica[%d]", replica->offset);
	}

	out->begin_list(out, NULL, NULL, NULL);

	if (print_ip) {
	out->message(out, (const char *) ip->private->xml->name,
	new_show_opts, ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, (const char *) child->private->xml->name,
	new_show_opts, child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, (const char *) container->private->xml->name,
	new_show_opts, container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, (const char *) remote->private->xml->name,
	new_show_opts, remote, only_node, only_rsc);
	}

	out->end_list(out);
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_text(out, replica,
	pcmk__current_node(container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	free_bundle_replica(pcmk__bundle_replica_t *replica)
	{
	if (replica == NULL) {
	return;
	}

	if (replica->node) {
	free(replica->node);
	replica->node = NULL;
	}

	if (replica->ip) {
	pcmk__xml_free(replica->ip->private->xml);
	replica->ip->private->xml = NULL;
	replica->ip->private->fns->free(replica->ip);
	}
	if (replica->container) {
	pcmk__xml_free(replica->container->private->xml);
	replica->container->private->xml = NULL;
	replica->container->private->fns->free(replica->container);
	}
	if (replica->remote) {
	pcmk__xml_free(replica->remote->private->xml);
	replica->remote->private->xml = NULL;
	replica->remote->private->fns->free(replica->remote);
	}
	free(replica->ipaddr);
	free(replica);
	}

	void
	pe__free_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);
	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	free(bundle_data->prefix);
	free(bundle_data->image);
	free(bundle_data->control_port);
	free(bundle_data->host_network);
	free(bundle_data->host_netmask);
	free(bundle_data->ip_range_start);
	free(bundle_data->container_network);
	free(bundle_data->launcher_options);
	free(bundle_data->container_command);
	g_free(bundle_data->container_host_options);

	g_list_free_full(bundle_data->replicas,
	(GDestroyNotify) free_bundle_replica);
	g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
	g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
	g_list_free(rsc->private->children);

	if(bundle_data->child) {
	pcmk__xml_free(bundle_data->child->private->xml);
	bundle_data->child->private->xml = NULL;
	bundle_data->child->private->fns->free(bundle_data->child);
	}
	common_free(rsc);
	}

	enum rsc_role_e
	pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
	{
	enum rsc_role_e container_role = pcmk_role_unknown;
	return container_role;
	}

	/*!
	* \brief Get the number of configured replicas in a bundle
	*
	* \param[in] rsc Bundle resource
	*
	* \return Number of configured replicas, or 0 on error
	*/
	int
	pe_bundle_replicas(const pcmk_resource_t *rsc)
	{
	if (pcmk__is_bundle(rsc)) {
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	return bundle_data->nreplicas;
	}
	return 0;
	}

	void
	pe__count_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
	pcmk__bundle_replica_t *replica = item->data;

	if (replica->ip) {
	replica->ip->private->fns->count(replica->ip);
	}
	if (replica->child) {
	replica->child->private->fns->count(replica->child);
	}
	if (replica->container) {
	replica->container->private->fns->count(replica->container);
	}
	if (replica->remote) {
	replica->remote->private->fns->count(replica->remote);
	}
	}
	}

	gboolean
	pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_bundle_variant_data(bundle_data, rsc);

	for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	pcmk_resource_t *ip = replica->ip;
	pcmk_resource_t *child = replica->child;
	pcmk_resource_t *container = replica->container;
	pcmk_resource_t *remote = replica->remote;

	if ((ip != NULL)
	&& !ip->private->fns->is_filtered(ip, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	if ((child != NULL)
	&& !child->private->fns->is_filtered(child, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	if (!container->private->fns->is_filtered(container, only_rsc,
	FALSE)) {
	passes = TRUE;
	break;
	}
	if ((remote != NULL)
	&& !remote->private->fns->is_filtered(remote, only_rsc,
	FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get a list of a bundle's containers
	*
	* \param[in] bundle Bundle resource
	*
	* \return Newly created list of \p bundle's containers
	* \note It is the caller's responsibility to free the result with
	* g_list_free().
	*/
	GList *
	pe__bundle_containers(const pcmk_resource_t *bundle)
	{
	GList *containers = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, bundle);
	for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	containers = g_list_append(containers, replica->container);
	}
	return containers;
	}

	// Bundle implementation of pcmk__rsc_methods_t:active_node()
	pcmk_node_t *
	pe__bundle_active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;
	pcmk_node_t *node = NULL;
	pcmk_resource_t *container = NULL;
	GList *containers = NULL;
	GList *iter = NULL;
	GHashTable *nodes = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}

	/* For the purposes of this method, we only care about where the bundle's
	* containers are active, so build a list of active containers.
	*/
	get_bundle_variant_data(data, rsc);
	for (iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (replica->container->private->active_nodes != NULL) {
	containers = g_list_append(containers, replica->container);
	}
	}
	if (containers == NULL) {
	return NULL;
	}

	/* If the bundle has only a single active container, just use that
	* container's method. If live migration is ever supported for bundle
	* containers, this will allow us to prefer the migration source when there
	* is only one container and it is migrating. For now, this just lets us
	* avoid creating the nodes table.
	*/
	if (pcmk__list_of_1(containers)) {
	container = containers->data;
	node = container->private->fns->active_node(container, count_all,
	count_clean);
	g_list_free(containers);
	return node;
	}

	// Add all containers' active nodes to a hash table (for uniqueness)
	nodes = g_hash_table_new(NULL, NULL);
	for (iter = containers; iter != NULL; iter = iter->next) {
	container = iter->data;
	for (GList *node_iter = container->private->active_nodes;
	node_iter != NULL; node_iter = node_iter->next) {

	node = node_iter->data;

	// If insert returns true, we haven't counted this node yet
	if (g_hash_table_insert(nodes, (gpointer) node->details,
	(gpointer) node)
	&& !pe__count_active_node(rsc, node, &active, count_all,
	count_clean)) {
	goto done;
	}
	}
	}

	done:
	g_list_free(containers);
	g_hash_table_destroy(nodes);
	return active;
	}

	/*!
	* \internal
	* \brief Get maximum bundle resource instances per node
	*
	* \param[in] rsc Bundle resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__bundle_max_per_node(const pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
	return (unsigned int) bundle_data->nreplicas_per_host;
	}
	diff --git a/lib/pengine/native.c b/lib/pengine/native.c
	index 2876a24762..8afd67f789 100644
	--- a/lib/pengine/native.c
	+++ b/lib/pengine/native.c
	@@ -1,1184 +1,1184 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/output.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/complex.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml.h>
	#include <pe_status_private.h>

	/*!
	* \internal
	* \brief Check whether a resource is active on multiple nodes
	*/
	static bool
	is_multiply_active(const pcmk_resource_t *rsc)
	{
	unsigned int count = 0;

	if (pcmk__is_primitive(rsc)) {
	pe__find_active_requires(rsc, &count);
	}
	return count > 1;
	}

	static void
	native_priority_to_node(pcmk_resource_t rsc, pcmk_node_t node,
	gboolean failed)
	{
	int priority = 0;
	const bool promoted = (rsc->private->orig_role == pcmk_role_promoted);

	if ((rsc->private->priority == 0) \|\| failed) {
	return;
	}

	if (promoted) {
	// Promoted instance takes base priority + 1
	priority = rsc->private->priority + 1;

	} else {
	priority = rsc->private->priority;
	}

	node->private->priority += priority;
	pcmk__rsc_trace(rsc, "%s now has priority %d with %s'%s' (priority: %d%s)",
	pcmk__node_name(node), node->private->priority,
	(promoted? "promoted " : ""),
	rsc->id, rsc->private->priority, (promoted? " + 1" : ""));

	/* Priority of a resource running on a guest node is added to the cluster
	* node as well. */
	if ((node->private->remote != NULL)
	&& (node->private->remote->private->launcher != NULL)) {
	const pcmk_resource_t *launcher = NULL;

	launcher = node->private->remote->private->launcher;
	for (GList *gIter = launcher->private->active_nodes;
	gIter != NULL; gIter = gIter->next) {

	pcmk_node_t *a_node = gIter->data;

	a_node->private->priority += priority;
	pcmk__rsc_trace(rsc,
	"%s now has priority %d with %s'%s' "
	"(priority: %d%s) from guest node %s",
	pcmk__node_name(a_node), a_node->private->priority,
	(promoted? "promoted " : ""), rsc->id,
	rsc->private->priority, (promoted? " + 1" : ""),
	pcmk__node_name(node));
	}
	}
	}

	void
	native_add_running(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler, gboolean failed)
	{
	pcmk_resource_t *parent = rsc->private->parent;

	CRM_CHECK(node != NULL, return);

	for (GList *gIter = rsc->private->active_nodes;
	gIter != NULL; gIter = gIter->next) {

	pcmk_node_t a_node = (pcmk_node_t ) gIter->data;

	if (pcmk__same_node(a_node, node)) {
	return;
	}
	}

	pcmk__rsc_trace(rsc, "Adding %s to %s %s", rsc->id, pcmk__node_name(node),
	pcmk_is_set(rsc->flags, pcmk__rsc_managed)? "" : "(unmanaged)");

	rsc->private->active_nodes = g_list_append(rsc->private->active_nodes,
	node);
	if (pcmk__is_primitive(rsc)) {
	node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);
	native_priority_to_node(rsc, node, failed);
	if (node->details->maintenance) {
	pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed);
	pcmk__set_rsc_flags(rsc, pcmk__rsc_maintenance);
	}
	}

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	pcmk_resource_t *p = parent;

	pcmk__rsc_info(rsc, "resource %s isn't managed", rsc->id);
	resource_location(rsc, node, PCMK_SCORE_INFINITY,
	"not_managed_default", scheduler);

	while(p && node->details->online) {
	/* add without the additional location constraint */
	p->private->active_nodes = g_list_append(p->private->active_nodes,
	node);
	p = p->private->parent;
	}
	return;
	}

	if (is_multiply_active(rsc)) {
	switch (rsc->private->multiply_active_policy) {
	case pcmk__multiply_active_stop:
	{
	GHashTableIter gIter;
	pcmk_node_t *local_node = NULL;

	/* make sure it doesn't come up again */
	if (rsc->private->allowed_nodes != NULL) {
	g_hash_table_destroy(rsc->private->allowed_nodes);
	}
	rsc->private->allowed_nodes =
	pe__node_list2table(scheduler->nodes);
	g_hash_table_iter_init(&gIter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
	- local_node->weight = -PCMK_SCORE_INFINITY;
	+ local_node->assign->score = -PCMK_SCORE_INFINITY;
	}
	}
	break;
	case pcmk__multiply_active_block:
	pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed);
	pcmk__set_rsc_flags(rsc, pcmk__rsc_blocked);

	/* If the resource belongs to a group or bundle configured with
	* PCMK_META_MULTIPLE_ACTIVE=PCMK_VALUE_BLOCK, block the entire
	* entity.
	*/
	if ((pcmk__is_group(parent) \|\| pcmk__is_bundle(parent))
	&& (parent->private->multiply_active_policy
	== pcmk__multiply_active_block)) {

	for (GList *gIter = parent->private->children;
	gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *child = gIter->data;

	pcmk__clear_rsc_flags(child, pcmk__rsc_managed);
	pcmk__set_rsc_flags(child, pcmk__rsc_blocked);
	}
	}
	break;

	// pcmk__multiply_active_restart, pcmk__multiply_active_unexpected
	default:
	/* The scheduler will do the right thing because the relevant
	* variables and flags are set when unpacking the history.
	*/
	break;
	}
	crm_debug("%s is active on multiple nodes including %s: %s",
	rsc->id, pcmk__node_name(node),
	pcmk__multiply_active_text(rsc));

	} else {
	pcmk__rsc_trace(rsc, "Resource %s is active on %s",
	rsc->id, pcmk__node_name(node));
	}

	if (parent != NULL) {
	native_add_running(parent, node, scheduler, FALSE);
	}
	}

	static void
	recursive_clear_unique(pcmk_resource_t *rsc, gpointer user_data)
	{
	pcmk__clear_rsc_flags(rsc, pcmk__rsc_unique);
	pcmk__insert_meta(rsc->private, PCMK_META_GLOBALLY_UNIQUE,
	PCMK_VALUE_FALSE);
	g_list_foreach(rsc->private->children, (GFunc) recursive_clear_unique,
	NULL);
	}

	gboolean
	native_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *parent = uber_parent(rsc);
	const char *standard = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);
	uint32_t ra_caps = pcmk_get_ra_caps(standard);

	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	// Only some agent standards support unique and promotable clones
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_unique)
	&& pcmk_is_set(rsc->flags, pcmk__rsc_unique)
	&& pcmk__is_clone(parent)) {

	/* @COMPAT We should probably reject this situation as an error (as we
	* do for promotable below) rather than warn and convert, but that would
	* be a backward-incompatible change that we should probably do with a
	* transform at a schema major version bump.
	*/
	pe__force_anon(standard, parent, rsc->id, scheduler);

	/* Clear PCMK_META_GLOBALLY_UNIQUE on the parent and all its descendants
	* unpacked so far (clearing the parent should make any future children
	* unpacking correct). We have to clear this resource explicitly because
	* it isn't hooked into the parent's children yet.
	*/
	recursive_clear_unique(parent, NULL);
	recursive_clear_unique(rsc, NULL);
	}
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_promotable)
	&& pcmk_is_set(parent->flags, pcmk__rsc_promotable)) {

	pcmk__config_err("Resource %s is of type %s and therefore "
	"cannot be used as a promotable clone resource",
	rsc->id, standard);
	return FALSE;
	}
	return TRUE;
	}

	static bool
	rsc_is_on_node(pcmk_resource_t rsc, const pcmk_node_t node, int flags)
	{
	pcmk__rsc_trace(rsc, "Checking whether %s is on %s",
	rsc->id, pcmk__node_name(node));

	if (pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->private->active_nodes != NULL)) {

	for (GList *iter = rsc->private->active_nodes;
	iter != NULL; iter = iter->next) {

	if (pcmk__same_node((pcmk_node_t *) iter->data, node)) {
	return true;
	}
	}

	} else if (pcmk_is_set(flags, pe_find_inactive) // @COMPAT deprecated
	&& (rsc->private->active_nodes == NULL)) {
	return true;

	} else if (!pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->private->assigned_node != NULL)
	&& pcmk__same_node(rsc->private->assigned_node, node)) {
	return true;
	}
	return false;
	}

	pcmk_resource_t *
	native_find_rsc(pcmk_resource_t rsc, const char id,
	const pcmk_node_t *on_node, int flags)
	{
	bool match = false;
	pcmk_resource_t *result = NULL;

	CRM_CHECK(id && rsc && rsc->id, return NULL);

	if (pcmk_is_set(flags, pcmk_rsc_match_clone_only)) {
	const char *rid = pcmk__xe_id(rsc->private->xml);

	if (!pcmk__is_clone(pe__const_top_resource(rsc, false))) {
	match = false;

	} else if (!strcmp(id, rsc->id) \|\| pcmk__str_eq(id, rid, pcmk__str_none)) {
	match = true;
	}

	} else if (!strcmp(id, rsc->id)) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_history)
	&& pcmk__str_eq(rsc->private->history_id, id, pcmk__str_none)) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_basename)
	\|\| (pcmk_is_set(flags, pcmk_rsc_match_anon_basename)
	&& !pcmk_is_set(rsc->flags, pcmk__rsc_unique))) {
	match = pe_base_name_eq(rsc, id);
	}

	if (match && on_node) {
	if (!rsc_is_on_node(rsc, on_node, flags)) {
	match = false;
	}
	}

	if (match) {
	return rsc;
	}

	for (GList *gIter = rsc->private->children;
	gIter != NULL; gIter = gIter->next) {

	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	result = rsc->private->fns->find_rsc(child, id, on_node, flags);
	if (result) {
	return result;
	}
	}
	return NULL;
	}

	// create is ignored
	char *
	native_parameter(pcmk_resource_t rsc, pcmk_node_t node, gboolean create,
	const char name, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	GHashTable *params = NULL;

	CRM_CHECK(rsc != NULL, return NULL);
	CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);

	pcmk__rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
	params = pe_rsc_params(rsc, node, scheduler);
	value = g_hash_table_lookup(params, name);
	if (value == NULL) {
	/* try meta attributes instead */
	value = g_hash_table_lookup(rsc->private->meta, name);
	}
	return pcmk__str_copy(value);
	}

	gboolean
	native_active(pcmk_resource_t * rsc, gboolean all)
	{
	for (GList *gIter = rsc->private->active_nodes;
	gIter != NULL; gIter = gIter->next) {

	pcmk_node_t a_node = (pcmk_node_t ) gIter->data;

	if (a_node->details->unclean) {
	pcmk__rsc_trace(rsc, "Resource %s: %s is unclean",
	rsc->id, pcmk__node_name(a_node));
	return TRUE;
	} else if (!a_node->details->online
	&& pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	pcmk__rsc_trace(rsc, "Resource %s: %s is offline",
	rsc->id, pcmk__node_name(a_node));
	} else {
	pcmk__rsc_trace(rsc, "Resource %s active on %s",
	rsc->id, pcmk__node_name(a_node));
	return TRUE;
	}
	}
	return FALSE;
	}

	struct print_data_s {
	long options;
	void *print_data;
	};

	static const char *
	native_pending_state(const pcmk_resource_t *rsc)
	{
	const char *pending_state = NULL;

	if (pcmk__str_eq(rsc->private->pending_action, PCMK_ACTION_START,
	pcmk__str_none)) {
	pending_state = "Starting";

	} else if (pcmk__str_eq(rsc->private->pending_action, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	pending_state = "Stopping";

	} else if (pcmk__str_eq(rsc->private->pending_action, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->private->pending_action,
	PCMK_ACTION_MIGRATE_FROM, pcmk__str_none)) {
	/* Work might be done in here. */
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->private->pending_action, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	pending_state = "Promoting";

	} else if (pcmk__str_eq(rsc->private->pending_action, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	pending_state = "Demoting";
	}

	return pending_state;
	}

	static const char *
	native_pending_action(const pcmk_resource_t *rsc)
	{
	const char *pending_action = NULL;

	if (pcmk__str_eq(rsc->private->pending_action, PCMK_ACTION_MONITOR,
	pcmk__str_none)) {
	pending_action = "Monitoring";

	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* unpack.c:unpack_rsc_op().
	*/
	#if 0
	} else if (pcmk__str_eq(rsc->private->pending_action, "probe",
	pcmk__str_none)) {
	pending_action = "Checking";
	#endif
	}

	return pending_action;
	}

	static enum rsc_role_e
	native_displayable_role(const pcmk_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->private->orig_role;

	if ((role == pcmk_role_started)
	&& pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk__rsc_promotable)) {

	role = pcmk_role_unpromoted;
	}
	return role;
	}

	static const char *
	native_displayable_state(const pcmk_resource_t *rsc, bool print_pending)
	{
	const char *rsc_state = NULL;

	if (print_pending) {
	rsc_state = native_pending_state(rsc);
	}
	if (rsc_state == NULL) {
	rsc_state = pcmk_role_text(native_displayable_role(rsc));
	}
	return rsc_state;
	}

	// Append a flag to resource description string's flags list
	static bool
	add_output_flag(GString s, const char flag_desc, bool have_flags)
	{
	g_string_append(s, (have_flags? ", " : " ("));
	g_string_append(s, flag_desc);
	return true;
	}

	// Append a node name to resource description string's node list
	static bool
	add_output_node(GString s, const char node, bool have_nodes)
	{
	g_string_append(s, (have_nodes? " " : " [ "));
	g_string_append(s, node);
	return true;
	}

	/*!
	* \internal
	* \brief Create a string description of a resource
	*
	* \param[in] rsc Resource to describe
	* \param[in] name Desired identifier for the resource
	* \param[in] node If not NULL, node that resource is "on"
	* \param[in] show_opts Bitmask of pcmk_show_opt_e.
	* \param[in] target_role Resource's target role
	* \param[in] show_nodes Whether to display nodes when multiply active
	*
	* \return Newly allocated string description of resource
	* \note Caller must free the result with g_free().
	*/
	gchar *
	pcmk__native_output_string(const pcmk_resource_t rsc, const char name,
	const pcmk_node_t *node, uint32_t show_opts,
	const char *target_role, bool show_nodes)
	{
	const char *class = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);
	const char *provider = NULL;
	const char *kind = crm_element_value(rsc->private->xml, PCMK_XA_TYPE);
	GString *outstr = NULL;
	bool have_flags = false;

	if (!pcmk__is_primitive(rsc)) {
	return NULL;
	}

	CRM_CHECK(name != NULL, name = "unknown");
	CRM_CHECK(kind != NULL, kind = "unknown");
	CRM_CHECK(class != NULL, class = "unknown");

	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	provider = crm_element_value(rsc->private->xml, PCMK_XA_PROVIDER);
	}

	if ((node == NULL) && (rsc->private->lock_node != NULL)) {
	node = rsc->private->lock_node;
	}
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only)
	\|\| pcmk__list_of_multiple(rsc->private->active_nodes)) {
	node = NULL;
	}

	outstr = g_string_sized_new(128);

	// Resource name and agent
	pcmk__g_strcat(outstr,
	name, "\t(", class, ((provider == NULL)? "" : ":"),
	pcmk__s(provider, ""), ":", kind, "):\t", NULL);

	// State on node
	if (pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
	g_string_append(outstr, " ORPHANED");
	}
	if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
	enum rsc_role_e role = native_displayable_role(rsc);

	g_string_append(outstr, " FAILED");
	if (role > pcmk_role_unpromoted) {
	pcmk__add_word(&outstr, 0, pcmk_role_text(role));
	}
	} else {
	bool show_pending = pcmk_is_set(show_opts, pcmk_show_pending);

	pcmk__add_word(&outstr, 0, native_displayable_state(rsc, show_pending));
	}
	if (node) {
	pcmk__add_word(&outstr, 0, pcmk__node_name(node));
	}

	// Failed probe operation
	if (native_displayable_role(rsc) == pcmk_role_stopped) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node ? node->private->name : NULL);
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op, PCMK__XA_RC_CODE),
	&rc, 0);
	pcmk__g_strcat(outstr, " (", services_ocf_exitcode_str(rc), ") ",
	NULL);
	}
	}

	// Flags, as: (<flag> [...])
	if (node && !(node->details->online) && node->details->unclean) {
	have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
	}
	if ((node != NULL) && pcmk__same_node(node, rsc->private->lock_node)) {
	have_flags = add_output_flag(outstr, "LOCKED", have_flags);
	}
	if (pcmk_is_set(show_opts, pcmk_show_pending)) {
	const char *pending_action = native_pending_action(rsc);

	if (pending_action != NULL) {
	have_flags = add_output_flag(outstr, pending_action, have_flags);
	}
	}
	if (target_role != NULL) {
	switch (pcmk_parse_role(target_role)) {
	case pcmk_role_unknown:
	pcmk__config_err("Invalid " PCMK_META_TARGET_ROLE
	" %s for resource %s", target_role, rsc->id);
	break;

	case pcmk_role_stopped:
	have_flags = add_output_flag(outstr, "disabled", have_flags);
	break;

	case pcmk_role_unpromoted:
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk__rsc_promotable)) {
	have_flags = add_output_flag(outstr,
	PCMK_META_TARGET_ROLE ":",
	have_flags);
	g_string_append(outstr, target_role);
	}
	break;

	default:
	/* Only show target role if it limits our abilities (i.e. ignore
	* Started, as it is the default anyways, and doesn't prevent
	* the resource from becoming promoted).
	*/
	break;
	}
	}

	// Blocked or maintenance implies unmanaged
	if (pcmk_any_flags_set(rsc->flags,
	pcmk__rsc_blocked\|pcmk__rsc_maintenance)) {
	if (pcmk_is_set(rsc->flags, pcmk__rsc_blocked)) {
	have_flags = add_output_flag(outstr, "blocked", have_flags);

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_maintenance)) {
	have_flags = add_output_flag(outstr, "maintenance", have_flags);
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	have_flags = add_output_flag(outstr, "unmanaged", have_flags);
	}

	if (pcmk_is_set(rsc->flags, pcmk__rsc_ignore_failure)) {
	have_flags = add_output_flag(outstr, "failure ignored", have_flags);
	}


	if (have_flags) {
	g_string_append_c(outstr, ')');
	}

	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)
	\|\| pcmk__list_of_multiple(rsc->private->active_nodes)) {
	const char *desc = crm_element_value(rsc->private->xml,
	PCMK_XA_DESCRIPTION);

	if (desc) {
	g_string_append(outstr, " (");
	g_string_append(outstr, desc);
	g_string_append(outstr, ")");

	}
	}

	if (show_nodes && !pcmk_is_set(show_opts, pcmk_show_rsc_only)
	&& pcmk__list_of_multiple(rsc->private->active_nodes)) {
	bool have_nodes = false;

	for (GList *iter = rsc->private->active_nodes;
	iter != NULL; iter = iter->next) {

	pcmk_node_t n = (pcmk_node_t ) iter->data;

	have_nodes = add_output_node(outstr, n->private->name, have_nodes);
	}
	if (have_nodes) {
	g_string_append(outstr, " ]");
	}
	}

	return g_string_free(outstr, FALSE);
	}

	int
	pe__common_output_html(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	uint32_t show_opts)
	{
	const char *kind = crm_element_value(rsc->private->xml, PCMK_XA_TYPE);
	const char *target_role = NULL;
	const char *cl = NULL;

	xmlNode *child = NULL;
	gchar *content = NULL;

	CRM_ASSERT((kind != NULL) && pcmk__is_primitive(rsc));

	if (crm_is_true(g_hash_table_lookup(rsc->private->meta,
	PCMK__META_INTERNAL_RSC))
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->private->meta,
	PCMK_META_TARGET_ROLE);

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	cl = PCMK__VALUE_RSC_MANAGED;

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
	cl = PCMK__VALUE_RSC_FAILED;

	} else if (pcmk__is_primitive(rsc)
	&& (rsc->private->active_nodes == NULL)) {
	cl = PCMK__VALUE_RSC_FAILED;

	} else if (pcmk__list_of_multiple(rsc->private->active_nodes)) {
	cl = PCMK__VALUE_RSC_MULTIPLE;

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_ignore_failure)) {
	cl = PCMK__VALUE_RSC_FAILURE_IGNORED;

	} else {
	cl = PCMK__VALUE_RSC_OK;
	}

	child = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
	child = pcmk__html_create(child, PCMK__XE_SPAN, NULL, cl);
	content = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);
	pcmk__xe_set_content(child, "%s", content);
	g_free(content);

	return pcmk_rc_ok;
	}

	int
	pe__common_output_text(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	uint32_t show_opts)
	{
	const char *target_role = NULL;

	CRM_ASSERT(pcmk__is_primitive(rsc));

	if (crm_is_true(g_hash_table_lookup(rsc->private->meta,
	PCMK__META_INTERNAL_RSC))
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->private->meta,
	PCMK_META_TARGET_ROLE);

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	out->list_item(out, NULL, "%s", s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	int rc = pcmk_rc_no_output;
	bool print_pending = pcmk_is_set(show_opts, pcmk_show_pending);
	const char *class = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);
	const char *prov = crm_element_value(rsc->private->xml, PCMK_XA_PROVIDER);

	char ra_name[LINE_MAX];
	const char *rsc_state = native_displayable_state(rsc, print_pending);
	const char *target_role = NULL;
	const char *active = pcmk__btoa(rsc->private->fns->active(rsc, TRUE));
	const char *orphaned = pcmk__flag_text(rsc->flags, pcmk__rsc_removed);
	const char *blocked = pcmk__flag_text(rsc->flags, pcmk__rsc_blocked);
	const char *maintenance = pcmk__flag_text(rsc->flags,
	pcmk__rsc_maintenance);
	const char *managed = pcmk__flag_text(rsc->flags, pcmk__rsc_managed);
	const char *failed = pcmk__flag_text(rsc->flags, pcmk__rsc_failed);
	const char *ignored = pcmk__flag_text(rsc->flags, pcmk__rsc_ignore_failure);
	char *nodes_running_on = NULL;
	const char *pending = print_pending? native_pending_action(rsc) : NULL;
	const char *locked_to = NULL;
	const char *desc = pe__resource_description(rsc, show_opts);

	CRM_ASSERT(pcmk__is_primitive(rsc));

	if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	// Resource information
	snprintf(ra_name, LINE_MAX, "%s%s%s:%s", class,
	((prov == NULL)? "" : ":"), ((prov == NULL)? "" : prov),
	crm_element_value(rsc->private->xml, PCMK_XA_TYPE));

	target_role = g_hash_table_lookup(rsc->private->meta,
	PCMK_META_TARGET_ROLE);

	nodes_running_on = pcmk__itoa(g_list_length(rsc->private->active_nodes));

	if (rsc->private->lock_node != NULL) {
	locked_to = rsc->private->lock_node->private->name;
	}

	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_RESOURCE,
	PCMK_XA_ID, rsc_printable_id(rsc),
	PCMK_XA_RESOURCE_AGENT, ra_name,
	PCMK_XA_ROLE, rsc_state,
	PCMK_XA_TARGET_ROLE, target_role,
	PCMK_XA_ACTIVE, active,
	PCMK_XA_ORPHANED, orphaned,
	PCMK_XA_BLOCKED, blocked,
	PCMK_XA_MAINTENANCE, maintenance,
	PCMK_XA_MANAGED, managed,
	PCMK_XA_FAILED, failed,
	PCMK_XA_FAILURE_IGNORED, ignored,
	PCMK_XA_NODES_RUNNING_ON, nodes_running_on,
	PCMK_XA_PENDING, pending,
	PCMK_XA_LOCKED_TO, locked_to,
	PCMK_XA_DESCRIPTION, desc,
	NULL);
	free(nodes_running_on);

	CRM_ASSERT(rc == pcmk_rc_ok);

	for (GList *gIter = rsc->private->active_nodes;
	gIter != NULL; gIter = gIter->next) {

	pcmk_node_t node = (pcmk_node_t ) gIter->data;
	const char *cached = pcmk__btoa(node->details->online);

	rc = pe__name_and_nvpairs_xml(out, false, PCMK_XE_NODE,
	PCMK_XA_NAME, node->private->name,
	PCMK_XA_ID, node->private->id,
	PCMK_XA_CACHED, cached,
	NULL);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	pcmk__output_xml_pop_parent(out);
	return rc;
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pcmk_node_t *node = pcmk__current_node(rsc);

	if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	CRM_ASSERT(pcmk__is_primitive(rsc));

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->private->pending_node;
	}
	return pe__common_output_html(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pcmk_node_t *node = pcmk__current_node(rsc);

	CRM_ASSERT(pcmk__is_primitive(rsc));

	if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->private->pending_node;
	}
	return pe__common_output_text(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	void
	native_free(pcmk_resource_t * rsc)
	{
	pcmk__rsc_trace(rsc, "Freeing resource action list (not the data)");
	common_free(rsc);
	}

	enum rsc_role_e
	native_resource_state(const pcmk_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e role = rsc->private->next_role;

	if (current) {
	role = rsc->private->orig_role;
	}
	pcmk__rsc_trace(rsc, "%s state: %s", rsc->id, pcmk_role_text(role));
	return role;
	}

	/*!
	* \internal
	* \brief List nodes where a resource (or any of its children) is
	*
	* \param[in] rsc Resource to check
	* \param[out] list List to add result to
	* \param[in] current 0 = where allocated, 1 = where running,
	* 2 = where running or pending
	*
	* \return If list contains only one node, that node, or NULL otherwise
	*/
	pcmk_node_t *
	native_location(const pcmk_resource_t rsc, GList *list, int current)
	{
	// @COMPAT: Accept a pcmk__rsc_node argument instead of int current
	pcmk_node_t *one = NULL;
	GList *result = NULL;

	if (rsc->private->children != NULL) {

	for (GList *gIter = rsc->private->children;
	gIter != NULL; gIter = gIter->next) {

	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	child->private->fns->location(child, &result, current);
	}

	} else if (current) {

	result = g_list_copy(rsc->private->active_nodes);
	if ((current == 2) && (rsc->private->pending_node != NULL)
	&& !pe_find_node_id(result,
	rsc->private->pending_node->private->id)) {

	result = g_list_append(result,
	(gpointer) rsc->private->pending_node);
	}

	} else if (!current && (rsc->private->assigned_node != NULL)) {
	result = g_list_append(NULL, rsc->private->assigned_node);
	}

	if (result && (result->next == NULL)) {
	one = result->data;
	}

	if (list) {
	GList *gIter = result;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if ((*list == NULL)
	\|\| (pe_find_node_id(*list, node->private->id) == NULL)) {
	list = g_list_append(list, node);
	}
	}
	}

	g_list_free(result);
	return one;
	}

	static void
	get_rscs_brief(GList rsc_list, GHashTable rsc_table, GHashTable * active_table)
	{
	GList *gIter = rsc_list;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	const char *class = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);
	const char *kind = crm_element_value(rsc->private->xml, PCMK_XA_TYPE);

	int offset = 0;
	char buffer[LINE_MAX];

	int *rsc_counter = NULL;
	int *active_counter = NULL;

	if (!pcmk__is_primitive(rsc)) {
	continue;
	}

	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	const char *prov = crm_element_value(rsc->private->xml,
	PCMK_XA_PROVIDER);

	if (prov != NULL) {
	offset += snprintf(buffer + offset, LINE_MAX - offset,
	":%s", prov);
	}
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
	CRM_LOG_ASSERT(offset > 0);

	if (rsc_table) {
	rsc_counter = g_hash_table_lookup(rsc_table, buffer);
	if (rsc_counter == NULL) {
	rsc_counter = pcmk__assert_alloc(1, sizeof(int));
	*rsc_counter = 0;
	g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
	}
	(*rsc_counter)++;
	}

	if (active_table) {
	for (GList *gIter2 = rsc->private->active_nodes;
	gIter2 != NULL; gIter2 = gIter2->next) {

	pcmk_node_t node = (pcmk_node_t ) gIter2->data;
	GHashTable *node_table = NULL;

	if (node->details->unclean == FALSE && node->details->online == FALSE &&
	pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	continue;
	}

	node_table = g_hash_table_lookup(active_table, node->private->name);
	if (node_table == NULL) {
	node_table = pcmk__strkey_table(free, free);
	g_hash_table_insert(active_table,
	strdup(node->private->name),
	node_table);
	}

	active_counter = g_hash_table_lookup(node_table, buffer);
	if (active_counter == NULL) {
	active_counter = pcmk__assert_alloc(1, sizeof(int));
	*active_counter = 0;
	g_hash_table_insert(node_table, strdup(buffer), active_counter);
	}
	(*active_counter)++;
	}
	}
	}
	}

	static void
	destroy_node_table(gpointer data)
	{
	GHashTable *node_table = data;

	if (node_table) {
	g_hash_table_destroy(node_table);
	}
	}

	int
	pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, uint32_t show_opts)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GList *sorted_rscs;
	int rc = pcmk_rc_no_output;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	/* Make a list of the rsc_table keys so that it can be sorted. This is to make sure
	* output order stays consistent between systems.
	*/
	sorted_rscs = g_hash_table_get_keys(rsc_table);
	sorted_rscs = g_list_sort(sorted_rscs, (GCompareFunc) strcmp);

	for (GList *gIter = sorted_rscs; gIter; gIter = gIter->next) {
	char type = (char ) gIter->data;
	int *rsc_counter = g_hash_table_lookup(rsc_table, type);

	GList *sorted_nodes = NULL;
	int active_counter_all = 0;

	/* Also make a list of the active_table keys so it can be sorted. If there's
	* more than one instance of a type of resource running, we need the nodes to
	* be sorted to make sure output order stays consistent between systems.
	*/
	sorted_nodes = g_hash_table_get_keys(active_table);
	sorted_nodes = g_list_sort(sorted_nodes, (GCompareFunc) pcmk__numeric_strcasecmp);

	for (GList *gIter2 = sorted_nodes; gIter2; gIter2 = gIter2->next) {
	char node_name = (char ) gIter2->data;
	GHashTable *node_table = g_hash_table_lookup(active_table, node_name);
	int *active_counter = NULL;

	if (node_table == NULL) {
	continue;
	}

	active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rsc_only)) {
	node_name = NULL;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive %s",
	*active_counter,
	rsc_counter ? *rsc_counter : 0, type,
	(*active_counter > 0) && node_name ? node_name : "");
	} else {
	out->list_item(out, NULL, "%d\t(%s):\tActive %s",
	*active_counter, type,
	(*active_counter > 0) && node_name ? node_name : "");
	}

	rc = pcmk_rc_ok;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs) && active_counter_all == 0) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);
	rc = pcmk_rc_ok;
	}

	if (sorted_nodes) {
	g_list_free(sorted_nodes);
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	if (sorted_rscs) {
	g_list_free(sorted_rscs);
	}

	return rc;
	}

	gboolean
	pe__native_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
	return FALSE;
	} else if (check_parent && (rsc->private->parent != NULL)) {
	const pcmk_resource_t *up = pe__const_top_resource(rsc, true);

	return up->private->fns->is_filtered(up, only_rsc, FALSE);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Get maximum primitive resource instances per node
	*
	* \param[in] rsc Primitive resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__primitive_max_per_node(const pcmk_resource_t *rsc)
	{
	CRM_ASSERT(pcmk__is_primitive(rsc));
	return 1U;
	}
	diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
	index 219618c5ad..333c7162b9 100644
	--- a/lib/pengine/pe_output.c
	+++ b/lib/pengine/pe_output.c
	@@ -1,3452 +1,3452 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/xml_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/cib/util.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/internal.h>

	const char *
	pe__resource_description(const pcmk_resource_t *rsc, uint32_t show_opts)
	{
	const char * desc = NULL;

	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)) {
	desc = crm_element_value(rsc->private->xml, PCMK_XA_DESCRIPTION);
	}
	return desc;
	}

	/* Never display node attributes whose name starts with one of these prefixes */
	#define FILTER_STR { PCMK__FAIL_COUNT_PREFIX, PCMK__LAST_FAILURE_PREFIX, \
	PCMK__NODE_ATTR_SHUTDOWN, PCMK_NODE_ATTR_TERMINATE, \
	PCMK_NODE_ATTR_STANDBY, "#", NULL }

	static int
	compare_attribute(gconstpointer a, gconstpointer b)
	{
	int rc;

	rc = strcmp((const char )a, (const char )b);

	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether extended information about an attribute should be added.
	*
	* \param[in] node Node that ran this resource
	* \param[in,out] rsc_list List of resources for this node
	* \param[in,out] scheduler Scheduler data
	* \param[in] attrname Attribute to find
	* \param[out] expected_score Expected value for this attribute
	*
	* \return true if extended information should be printed, false otherwise
	* \note Currently, extended information is only supported for ping/pingd
	* resources, for which a message will be printed if connectivity is lost
	* or degraded.
	*/
	static bool
	add_extra_info(const pcmk_node_t node, GList rsc_list,
	pcmk_scheduler_t scheduler, const char attrname,
	int *expected_score)
	{
	GList *gIter = NULL;

	for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;
	const char *type = g_hash_table_lookup(rsc->private->meta,
	PCMK_XA_TYPE);
	const char *name = NULL;
	GHashTable *params = NULL;

	if (rsc->private->children != NULL) {
	if (add_extra_info(node, rsc->private->children, scheduler,
	attrname, expected_score)) {
	return true;
	}
	}

	if (!pcmk__strcase_any_of(type, "ping", "pingd", NULL)) {
	continue;
	}

	params = pe_rsc_params(rsc, node, scheduler);
	name = g_hash_table_lookup(params, PCMK_XA_NAME);

	if (name == NULL) {
	name = "pingd";
	}

	/* To identify the resource with the attribute name. */
	if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
	int host_list_num = 0;
	const char *hosts = g_hash_table_lookup(params, "host_list");
	const char *multiplier = g_hash_table_lookup(params, "multiplier");
	int multiplier_i;

	if (hosts) {
	char **host_list = g_strsplit(hosts, " ", 0);
	host_list_num = g_strv_length(host_list);
	g_strfreev(host_list);
	}

	if ((multiplier == NULL)
	\|\| (pcmk__scan_min_int(multiplier, &multiplier_i,
	INT_MIN) != pcmk_rc_ok)) {
	/* The ocf:pacemaker:ping resource agent defaults multiplier to
	* 1. The agent currently does not handle invalid text, but it
	* should, and this would be a reasonable choice ...
	*/
	multiplier_i = 1;
	}
	expected_score = host_list_num multiplier_i;

	return true;
	}
	}
	return false;
	}

	static GList *
	filter_attr_list(GList attr_list, char name)
	{
	int i;
	const char *filt_str[] = FILTER_STR;

	CRM_CHECK(name != NULL, return attr_list);

	/* filtering automatic attributes */
	for (i = 0; filt_str[i] != NULL; i++) {
	if (g_str_has_prefix(name, filt_str[i])) {
	return attr_list;
	}
	}

	return g_list_insert_sorted(attr_list, name, compare_attribute);
	}

	static GList *
	get_operation_list(xmlNode *rsc_entry) {
	GList *op_list = NULL;
	xmlNode *rsc_op = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry, NULL, NULL, NULL);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	const char *task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	const char *interval_ms_s = crm_element_value(rsc_op,
	PCMK_META_INTERVAL);
	const char *op_rc = crm_element_value(rsc_op, PCMK__XA_RC_CODE);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* Ignore notifies and some probes */
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)
	\|\| (pcmk__str_eq(task, "probe", pcmk__str_none)
	&& (op_rc_i == CRM_EX_NOT_RUNNING))) {
	continue;
	}

	if (pcmk__xe_is(rsc_op, PCMK__XE_LRM_RSC_OP)) {
	op_list = g_list_append(op_list, rsc_op);
	}
	}

	op_list = g_list_sort(op_list, sort_op_by_callid);
	return op_list;
	}

	static void
	add_dump_node(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNodePtr node = user_data;

	node = pcmk__xe_create(node, (const char *) key);
	pcmk__xe_set_content(node, "%s", (const char *) value);
	}

	static void
	append_dump_text(gpointer key, gpointer value, gpointer user_data)
	{
	char **dump_text = user_data;
	char *new_text = crm_strdup_printf("%s %s=%s",
	dump_text, (char )key, (char *)value);

	free(*dump_text);
	*dump_text = new_text;
	}

	#define XPATH_STACK "//" PCMK_XE_NVPAIR \
	"[@" PCMK_XA_NAME "='" \
	PCMK_OPT_CLUSTER_INFRASTRUCTURE "']"

	static const char *
	get_cluster_stack(pcmk_scheduler_t *scheduler)
	{
	xmlNode *stack = get_xpath_object(XPATH_STACK, scheduler->input, LOG_DEBUG);

	if (stack != NULL) {
	return crm_element_value(stack, PCMK_XA_VALUE);
	}
	return PCMK_VALUE_UNKNOWN;
	}

	static char *
	last_changed_string(const char last_written, const char user,
	const char client, const char origin) {
	if (last_written != NULL \|\| user != NULL \|\| client != NULL \|\| origin != NULL) {
	return crm_strdup_printf("%s%s%s%s%s%s%s",
	last_written ? last_written : "",
	user ? " by " : "",
	user ? user : "",
	client ? " via " : "",
	client ? client : "",
	origin ? " on " : "",
	origin ? origin : "");
	} else {
	return strdup("");
	}
	}

	static char *
	op_history_string(xmlNode xml_op, const char task, const char *interval_ms_s,
	int rc, bool print_timing) {
	const char *call = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	char *interval_str = NULL;
	char *buf = NULL;

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *pair = pcmk__format_nvpair(PCMK_XA_INTERVAL, interval_ms_s, "ms");
	interval_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	if (print_timing) {
	char *last_change_str = NULL;
	char *exec_str = NULL;
	char *queue_str = NULL;

	const char *value = NULL;

	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&epoch) == pcmk_ok)
	&& (epoch > 0)) {
	char *epoch_str = pcmk__epoch2str(&epoch, 0);

	last_change_str = crm_strdup_printf(" %s=\"%s\"",
	PCMK_XA_LAST_RC_CHANGE,
	pcmk__s(epoch_str, ""));
	free(epoch_str);
	}

	value = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
	if (value) {
	char *pair = pcmk__format_nvpair(PCMK_XA_EXEC_TIME, value, "ms");
	exec_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	value = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	if (value) {
	char *pair = pcmk__format_nvpair(PCMK_XA_QUEUE_TIME, value, "ms");
	queue_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	buf = crm_strdup_printf("(%s) %s:%s%s%s%s rc=%d (%s)", call, task,
	interval_str ? interval_str : "",
	last_change_str ? last_change_str : "",
	exec_str ? exec_str : "",
	queue_str ? queue_str : "",
	rc, services_ocf_exitcode_str(rc));

	if (last_change_str) {
	free(last_change_str);
	}

	if (exec_str) {
	free(exec_str);
	}

	if (queue_str) {
	free(queue_str);
	}
	} else {
	buf = crm_strdup_printf("(%s) %s%s%s", call, task,
	interval_str ? ":" : "",
	interval_str ? interval_str : "");
	}

	if (interval_str) {
	free(interval_str);
	}

	return buf;
	}

	static char *
	resource_history_string(pcmk_resource_t rsc, const char rsc_id, bool all,
	int failcount, time_t last_failure) {
	char *buf = NULL;

	if (rsc == NULL) {
	buf = crm_strdup_printf("%s: orphan", rsc_id);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *failcount_s = NULL;
	char *lastfail_s = NULL;

	if (failcount > 0) {
	failcount_s = crm_strdup_printf(" %s=%d",
	PCMK_XA_FAIL_COUNT, failcount);
	} else {
	failcount_s = strdup("");
	}
	if (last_failure > 0) {
	buf = pcmk__epoch2str(&last_failure, 0);
	lastfail_s = crm_strdup_printf(" %s='%s'",
	PCMK_XA_LAST_FAILURE, buf);
	free(buf);
	}

	buf = crm_strdup_printf("%s: " PCMK_META_MIGRATION_THRESHOLD "=%d%s%s",
	rsc_id, rsc->private->ban_after_failures,
	failcount_s, pcmk__s(lastfail_s, ""));
	free(failcount_s);
	free(lastfail_s);
	} else {
	buf = crm_strdup_printf("%s:", rsc_id);
	}

	return buf;
	}

	/*!
	* \internal
	* \brief Get a node's feature set for status display purposes
	*
	* \param[in] node Node to check
	*
	* \return String representation of feature set if the node is fully up (using
	* "<3.15.1" for older nodes that don't set the #feature-set attribute),
	* otherwise NULL
	*/
	static const char *
	get_node_feature_set(const pcmk_node_t *node)
	{
	if (node->details->online
	&& pcmk_is_set(node->private->flags, pcmk__node_expected_up)
	&& !pcmk__is_pacemaker_remote_node(node)) {

	const char *feature_set = g_hash_table_lookup(node->private->attrs,
	CRM_ATTR_FEATURE_SET);

	/* The feature set attribute is present since 3.15.1. If it is missing,
	* then the node must be running an earlier version.
	*/
	return pcmk__s(feature_set, "<3.15.1");
	}
	return NULL;
	}

	static bool
	is_mixed_version(pcmk_scheduler_t *scheduler)
	{
	const char *feature_set = NULL;
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;
	const char *node_feature_set = get_node_feature_set(node);
	if (node_feature_set != NULL) {
	if (feature_set == NULL) {
	feature_set = node_feature_set;
	} else if (strcmp(feature_set, node_feature_set) != 0) {
	return true;
	}
	}
	}
	return false;
	}

	static void
	formatted_xml_buf(const pcmk_resource_t rsc, GString xml_buf, bool raw)
	{
	if (raw && (rsc->private->orig_xml != NULL)) {
	pcmk__xml_string(rsc->private->orig_xml, pcmk__xml_fmt_pretty, xml_buf,
	0);
	} else {
	pcmk__xml_string(rsc->private->xml, pcmk__xml_fmt_pretty, xml_buf, 0);
	}
	}

	#define XPATH_DC_VERSION "//" PCMK_XE_NVPAIR \
	"[@" PCMK_XA_NAME "='" PCMK_OPT_DC_VERSION "']"

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	if (pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_ORIGIN);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	/* Always print DC if none, even if not requested */
	if ((scheduler->dc_node == NULL)
	\|\| pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_ORIGIN);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	/* Kind of a hack - close the list we may have opened earlier in this
	* function so we can put all the options into their own list. We
	* only want to do this on HTML output, though.
	*/
	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	out->begin_list(out, NULL, NULL, "Config Options");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	char *
	pe__node_display_name(pcmk_node_t *node, bool print_detail)
	{
	char *node_name;
	const char *node_host = NULL;
	const char *node_id = NULL;
	int name_len;

	CRM_ASSERT((node != NULL) && (node->private->name != NULL));

	/* Host is displayed only if this is a guest node and detail is requested */
	if (print_detail && pcmk__is_guest_or_bundle_node(node)) {
	const pcmk_resource_t *launcher = NULL;
	const pcmk_node_t *host_node = NULL;

	launcher = node->private->remote->private->launcher;
	host_node = pcmk__current_node(launcher);

	if (host_node && host_node->details) {
	node_host = host_node->private->name;
	}
	if (node_host == NULL) {
	node_host = ""; /* so we at least get "uname@" to indicate guest */
	}
	}

	/* Node ID is displayed if different from uname and detail is requested */
	if (print_detail
	&& !pcmk__str_eq(node->private->name, node->private->id,
	pcmk__str_casei)) {
	node_id = node->private->id;
	}

	/* Determine name length */
	name_len = strlen(node->private->name) + 1;
	if (node_host) {
	name_len += strlen(node_host) + 1; /* "@node_host" */
	}
	if (node_id) {
	name_len += strlen(node_id) + 3; /* + " (node_id)" */
	}

	/* Allocate and populate display name */
	node_name = pcmk__assert_alloc(name_len, sizeof(char));
	strcpy(node_name, node->private->name);
	if (node_host) {
	strcat(node_name, "@");
	strcat(node_name, node_host);
	}
	if (node_id) {
	strcat(node_name, " (");
	strcat(node_name, node_id);
	strcat(node_name, ")");
	}
	return node_name;
	}

	int
	pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name,
	...)
	{
	xmlNodePtr xml_node = NULL;
	va_list pairs;

	CRM_ASSERT(tag_name != NULL);

	xml_node = pcmk__output_xml_peek_parent(out);
	CRM_ASSERT(xml_node != NULL);
	xml_node = pcmk__xe_create(xml_node, tag_name);

	va_start(pairs, tag_name);
	pcmk__xe_set_propv(xml_node, pairs);
	va_end(pairs);

	if (is_list) {
	pcmk__output_xml_push_parent(out, xml_node);
	}
	return pcmk_rc_ok;
	}

	static const char *
	role_desc(enum rsc_role_e role)
	{
	if (role == pcmk_role_promoted) {
	return "in " PCMK_ROLE_PROMOTED " role ";
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	char *buf = crm_strdup_printf("%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	pcmk__output_create_html_node(out, "li", NULL, NULL, buf);

	free(node_name);
	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	out->list_item(out, NULL, "%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	free(node_name);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *promoted_only = pcmk__btoa(location->role_filter == pcmk_role_promoted);
	- char *weight_s = pcmk__itoa(pe_node->weight);
	+ char *weight_s = pcmk__itoa(pe_node->assign->score);

	pcmk__output_create_xml_node(out, PCMK_XE_BAN,
	PCMK_XA_ID, location->id,
	PCMK_XA_RESOURCE, location->rsc->id,
	PCMK_XA_NODE, pe_node->private->name,
	PCMK_XA_WEIGHT, weight_s,
	PCMK_XA_PROMOTED_ONLY, promoted_only,
	/* This is a deprecated alias for
	* promoted_only. Removing it will break
	* backward compatibility of the API schema,
	* which will require an API schema major
	* version bump.
	*/
	PCMK__XA_PROMOTED_ONLY_LEGACY, promoted_only,
	NULL);

	free(weight_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban-list", "pcmk_scheduler_t ", "const char ", "GList *",
	"uint32_t", "bool")
	static int
	ban_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	const char prefix = va_arg(args, const char );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	GList gIter, gIter2;
	int rc = pcmk_rc_no_output;

	/* Print each ban */
	for (gIter = scheduler->placement_constraints;
	gIter != NULL; gIter = gIter->next) {
	pcmk__location_t *location = gIter->data;
	const pcmk_resource_t *rsc = location->rsc;

	if (prefix != NULL && !g_str_has_prefix(location->id, prefix)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc, false)),
	only_rsc, pcmk__str_star_matches)) {
	continue;
	}

	for (gIter2 = location->nodes; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter2->data;

	- if (node->weight < 0) {
	+ if (node->assign->score < 0) {
	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Negative Location Constraints");
	out->message(out, "ban", node, location, show_opts);
	}
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_html(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(nodes_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	if (ndisabled && nblocked) {
	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources), ndisabled);

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "DISABLED");

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, ", %d ", nblocked);

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "BLOCKED");

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " from further action due to failure)");

	} else if (ndisabled && !nblocked) {
	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "DISABLED");

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, ")");

	} else if (!ndisabled && nblocked) {
	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	nblocked);

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "BLOCKED");

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " from further action due to failure)");

	} else {
	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_text(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	out->list_item(out, NULL, "%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	if (ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED, %d BLOCKED from "
	"further action due to failure)",
	nresources, pcmk__plural_s(nresources), ndisabled,
	nblocked);
	} else if (ndisabled && !nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED)",
	nresources, pcmk__plural_s(nresources), ndisabled);
	} else if (!ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d BLOCKED from further action "
	"due to failure)",
	nresources, pcmk__plural_s(nresources), nblocked);
	} else {
	out->list_item(out, NULL, "%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_xml(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = NULL;
	xmlNodePtr resources_node = NULL;
	char *s = NULL;

	nodes_node = pcmk__output_create_xml_node(out, PCMK_XE_NODES_CONFIGURED,
	NULL);
	resources_node = pcmk__output_create_xml_node(out,
	PCMK_XE_RESOURCES_CONFIGURED,
	NULL);

	s = pcmk__itoa(nnodes);
	crm_xml_add(nodes_node, PCMK_XA_NUMBER, s);
	free(s);

	s = pcmk__itoa(nresources);
	crm_xml_add(resources_node, PCMK_XA_NUMBER, s);
	free(s);

	s = pcmk__itoa(ndisabled);
	crm_xml_add(resources_node, PCMK_XA_DISABLED, s);
	free(s);

	s = pcmk__itoa(nblocked);
	crm_xml_add(resources_node, PCMK_XA_BLOCKED, s);
	free(s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "Current DC: ");

	if (dc) {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s (version %s) -",
	dc_name, pcmk__s(dc_version_s, "unknown"));

	if (mixed_version) {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_WARNING);
	pcmk__xe_set_content(child, " MIXED-VERSION");
	}

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " partition");

	if (crm_is_true(quorum)) {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " with");

	} else {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_WARNING);
	pcmk__xe_set_content(child, " WITHOUT");
	}

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " quorum");

	} else {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_WARNING);
	pcmk__xe_set_content(child, "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	out->list_item(out, "Current DC",
	"%s (version %s) - %spartition %s quorum",
	dc_name, dc_version_s ? dc_version_s : "unknown",
	mixed_version ? "MIXED-VERSION " : "",
	crm_is_true(quorum) ? "with" : "WITHOUT");
	} else {
	out->list_item(out, "Current DC", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name G_GNUC_UNUSED = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	const char *with_quorum = pcmk__btoa(crm_is_true(quorum));
	const char *mixed_version_s = pcmk__btoa(mixed_version);

	pcmk__output_create_xml_node(out, PCMK_XE_CURRENT_DC,
	PCMK_XA_PRESENT, PCMK_VALUE_TRUE,
	PCMK_XA_VERSION, pcmk__s(dc_version_s, ""),
	PCMK_XA_NAME, dc->private->name,
	PCMK_XA_ID, dc->private->id,
	PCMK_XA_WITH_QUORUM, with_quorum,
	PCMK_XA_MIXED_VERSION, mixed_version_s,
	NULL);
	} else {
	pcmk__output_create_xml_node(out, PCMK_XE_CURRENT_DC,
	PCMK_XA_PRESENT, PCMK_VALUE_FALSE,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("maint-mode", "unsigned long long int")
	static int
	cluster_maint_mode_text(pcmk__output_t *out, va_list args) {
	unsigned long long flags = va_arg(args, unsigned long long);

	if (pcmk_is_set(flags, pcmk_sched_in_maintenance)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will not attempt to start, stop or recover services\n");
	return pcmk_rc_ok;
	} else if (pcmk_is_set(flags, pcmk_sched_stop_all)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will keep all resources stopped\n");
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "Resource management: ");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "DISABLED");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child,
	" (the cluster will not attempt to start, stop,"
	" or recover services)");

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "Resource management: ");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "STOPPED");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child,
	" (the cluster will keep all resources stopped)");

	} else {
	out->list_item(out, NULL, "Resource management: enabled");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_log(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	return out->info(out, "Resource management is DISABLED. The cluster will not attempt to start, stop or recover services.");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	return out->info(out, "Resource management is DISABLED. The cluster has stopped all resources.");
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_text(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Get readable string representation of a no-quorum policy
	*
	* \param[in] policy No-quorum policy
	*
	* \return String representation of \p policy
	*/
	static const char *
	no_quorum_policy_text(enum pe_quorum_policy policy)
	{
	switch (policy) {
	case pcmk_no_quorum_freeze:
	return PCMK_VALUE_FREEZE;

	case pcmk_no_quorum_stop:
	return PCMK_VALUE_STOP;

	case pcmk_no_quorum_demote:
	return PCMK_VALUE_DEMOTE;

	case pcmk_no_quorum_ignore:
	return PCMK_VALUE_IGNORE;

	case pcmk_no_quorum_fence:
	return PCMK_VALUE_FENCE_LEGACY;

	default:
	return PCMK_VALUE_UNKNOWN;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	const char *stonith_enabled = pcmk__flag_text(scheduler->flags,
	pcmk_sched_fencing_enabled);
	const char *symmetric_cluster =
	pcmk__flag_text(scheduler->flags, pcmk_sched_symmetric_cluster);
	const char *no_quorum_policy =
	no_quorum_policy_text(scheduler->no_quorum_policy);
	const char *maintenance_mode = pcmk__flag_text(scheduler->flags,
	pcmk_sched_in_maintenance);
	const char *stop_all_resources = pcmk__flag_text(scheduler->flags,
	pcmk_sched_stop_all);
	char *stonith_timeout_ms_s = pcmk__itoa(scheduler->stonith_timeout);
	char *priority_fencing_delay_ms_s =
	pcmk__itoa(scheduler->priority_fencing_delay * 1000);

	pcmk__output_create_xml_node(out, PCMK_XE_CLUSTER_OPTIONS,
	PCMK_XA_STONITH_ENABLED, stonith_enabled,
	PCMK_XA_SYMMETRIC_CLUSTER, symmetric_cluster,
	PCMK_XA_NO_QUORUM_POLICY, no_quorum_policy,
	PCMK_XA_MAINTENANCE_MODE, maintenance_mode,
	PCMK_XA_STOP_ALL_RESOURCES, stop_all_resources,
	PCMK_XA_STONITH_TIMEOUT_MS,
	stonith_timeout_ms_s,
	PCMK_XA_PRIORITY_FENCING_DELAY_MS,
	priority_fencing_delay_ms_s,
	NULL);
	free(stonith_timeout_ms_s);
	free(priority_fencing_delay_ms_s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_html(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "Stack: ");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s", stack_s);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " (");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s",
	pcmk__pcmkd_state_enum2friendly(pcmkd_state));

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, ")");
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_text(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	out->list_item(out, "Stack", "%s (%s)",
	stack_s, pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	} else {
	out->list_item(out, "Stack", "%s", stack_s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_xml(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	const char *state_s = NULL;

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(pcmkd_state);
	}

	pcmk__output_create_xml_node(out, PCMK_XE_STACK,
	PCMK_XA_TYPE, stack_s,
	PCMK_XA_PACEMAKERD_STATE, state_s,
	NULL);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_html(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	char *time_s = NULL;

	child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "Last updated: ");

	child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL, NULL);
	time_s = pcmk__epoch2str(NULL, 0);
	pcmk__xe_set_content(child, "%s", time_s);
	free(time_s);

	if (our_nodename != NULL) {
	child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " on ");

	child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s", our_nodename);
	}

	child = pcmk__html_create(changed_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "Last change: ");

	child = pcmk__html_create(changed_node, PCMK__XE_SPAN, NULL, NULL);
	time_s = last_changed_string(last_written, user, client, origin);
	pcmk__xe_set_content(child, "%s", time_s);
	free(time_s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_xml(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk__output_create_xml_node(out, PCMK_XE_LAST_UPDATE,
	PCMK_XA_TIME, time_s,
	PCMK_XA_ORIGIN, our_nodename,
	NULL);

	pcmk__output_create_xml_node(out, PCMK_XE_LAST_CHANGE,
	PCMK_XA_TIME, pcmk__s(last_written, ""),
	PCMK_XA_USER, pcmk__s(user, ""),
	PCMK_XA_CLIENT, pcmk__s(client, ""),
	PCMK_XA_ORIGIN, pcmk__s(origin, ""),
	NULL);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_text(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	out->list_item(out, "Last updated", "%s%s%s",
	time_s, (our_nodename != NULL)? " on " : "",
	pcmk__s(our_nodename, ""));

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	out->list_item(out, "Last change", " %s", time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Display a failed action in less-technical natural language
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_friendly(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	char *rsc_id = NULL;
	char *task = NULL;
	guint interval_ms = 0;
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)
	\|\| !parse_op_key(op_key, &rsc_id, &task, &interval_ms)) {

	pcmk__str_update(&rsc_id, "unknown resource");
	pcmk__str_update(&task, "unknown action");
	interval_ms = 0;
	}
	CRM_ASSERT((rsc_id != NULL) && (task != NULL));

	str = g_string_sized_new(256); // Should be sufficient for most messages

	pcmk__g_strcat(str, rsc_id, " ", NULL);

	if (interval_ms != 0) {
	pcmk__g_strcat(str, pcmk__readable_interval(interval_ms), "-interval ",
	NULL);
	}
	pcmk__g_strcat(str, pcmk__readable_action(task, interval_ms), " on ",
	node_name, NULL);

	if (status == PCMK_EXEC_DONE) {
	pcmk__g_strcat(str, " returned '", services_ocf_exitcode_str(rc), "'",
	NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, " (", exit_reason, ")", NULL);
	}

	} else {
	pcmk__g_strcat(str, " could not be executed (",
	pcmk_exec_status_str(status), NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ": ", exit_reason, NULL);
	}
	g_string_append_c(str, ')');
	}


	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *s = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str, " at ", s, NULL);
	free(s);
	}
	if (!pcmk__str_empty(exec_time)) {
	int exec_time_ms = 0;

	if ((pcmk__scan_min_int(exec_time, &exec_time_ms, 0) == pcmk_rc_ok)
	&& (exec_time_ms > 0)) {

	pcmk__g_strcat(str, " after ",
	pcmk__readable_interval(exec_time_ms), NULL);
	}
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	free(rsc_id);
	free(task);
	}

	/*!
	* \internal
	* \brief Display a failed action with technical details
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_technical(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	const char *queue_time = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	const char *exit_status = services_ocf_exitcode_str(rc);
	const char *lrm_status = pcmk_exec_status_str(status);
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)) {
	op_key = "unknown operation";
	}
	if (pcmk__str_empty(exit_status)) {
	exit_status = "unknown exit status";
	}
	if (pcmk__str_empty(call_id)) {
	call_id = "unknown";
	}

	str = g_string_sized_new(256);

	g_string_append_printf(str, "%s on %s '%s' (%d): call=%s, status='%s'",
	op_key, node_name, exit_status, rc, call_id,
	lrm_status);

	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ", exitreason='", exit_reason, "'", NULL);
	}

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *last_change_str = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str,
	", " PCMK_XA_LAST_RC_CHANGE "="
	"'", last_change_str, "'", NULL);
	free(last_change_str);
	}
	if (!pcmk__str_empty(queue_time)) {
	pcmk__g_strcat(str, ", queued=", queue_time, "ms", NULL);
	}
	if (!pcmk__str_empty(exec_time)) {
	pcmk__g_strcat(str, ", exec=", exec_time, "ms", NULL);
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNode *", "uint32_t")
	static int
	failed_action_default(pcmk__output_t *out, va_list args)
	{
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts = va_arg(args, uint32_t);

	const char *op_key = pcmk__xe_history_key(xml_op);
	const char *node_name = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *exit_reason = crm_element_value(xml_op, PCMK_XA_EXIT_REASON);
	const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);

	int rc;
	int status;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_RC_CODE), &rc, 0);

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
	0);

	if (pcmk__str_empty(node_name)) {
	node_name = "unknown node";
	}

	if (pcmk_is_set(show_opts, pcmk_show_failed_detail)) {
	failed_action_technical(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	} else {
	failed_action_friendly(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNode *", "uint32_t")
	static int
	failed_action_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *op_key = pcmk__xe_history_key(xml_op);
	const char *op_key_name = PCMK_XA_OP_KEY;
	int rc;
	int status;
	const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	const char *exitstatus = NULL;
	const char *exit_reason = pcmk__s(crm_element_value(xml_op,
	PCMK_XA_EXIT_REASON),
	"none");
	const char *status_s = NULL;

	time_t epoch = 0;
	gchar *exit_reason_esc = NULL;
	char *rc_s = NULL;
	xmlNodePtr node = NULL;

	if (pcmk__xml_needs_escape(exit_reason, pcmk__xml_escape_attr)) {
	exit_reason_esc = pcmk__xml_escape(exit_reason, pcmk__xml_escape_attr);
	exit_reason = exit_reason_esc;
	}
	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_RC_CODE), &rc, 0);
	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
	0);

	if (crm_element_value(xml_op, PCMK__XA_OPERATION_KEY) == NULL) {
	op_key_name = PCMK_XA_ID;
	}
	exitstatus = services_ocf_exitcode_str(rc);
	rc_s = pcmk__itoa(rc);
	status_s = pcmk_exec_status_str(status);
	node = pcmk__output_create_xml_node(out, PCMK_XE_FAILURE,
	op_key_name, op_key,
	PCMK_XA_NODE, uname,
	PCMK_XA_EXITSTATUS, exitstatus,
	PCMK_XA_EXITREASON, exit_reason,
	PCMK_XA_EXITCODE, rc_s,
	PCMK_XA_CALL, call_id,
	PCMK_XA_STATUS, status_s,
	NULL);
	free(rc_s);

	if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {

	const char *queue_time = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	const char *exec = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
	const char *task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	guint interval_ms = 0;
	char *interval_ms_s = NULL;
	char *rc_change = pcmk__epoch2str(&epoch,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);

	crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);

	pcmk__xe_set_props(node,
	PCMK_XA_LAST_RC_CHANGE, rc_change,
	PCMK_XA_QUEUED, queue_time,
	PCMK_XA_EXEC, exec,
	PCMK_XA_INTERVAL, interval_ms_s,
	PCMK_XA_TASK, task,
	NULL);

	free(interval_ms_s);
	free(rc_change);
	}

	g_free(exit_reason_esc);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action-list", "pcmk_scheduler_t ", "GList ",
	"GList *", "uint32_t", "bool")
	static int
	failed_action_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *xml_op = NULL;
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(scheduler->failed) == 0) {
	return rc;
	}

	for (xml_op = pcmk__xe_first_child(scheduler->failed, NULL, NULL, NULL);
	xml_op != NULL; xml_op = pcmk__xe_next(xml_op)) {

	char *rsc = NULL;

	if (!pcmk__str_in_list(crm_element_value(xml_op, PCMK_XA_UNAME),
	only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	if (pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	if (!parse_op_key(pcmk__xe_history_key(xml_op), &rsc, NULL, NULL)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc, only_rsc, pcmk__str_star_matches)) {
	free(rsc);
	continue;
	}

	free(rsc);

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Resource Actions");
	out->message(out, "failed-action", xml_op, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	status_node(pcmk_node_t *node, xmlNodePtr parent, uint32_t show_opts)
	{
	int health = pe__node_health(node);
	xmlNode *child = NULL;

	// Cluster membership
	if (node->details->online) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_ONLINE);
	pcmk__xe_set_content(child, " online");

	} else {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_OFFLINE);
	pcmk__xe_set_content(child, " OFFLINE");
	}

	// Standby mode
	if (pcmk_is_set(node->private->flags, pcmk__node_fail_standby)) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_STANDBY);
	if (node->details->running_rsc == NULL) {
	pcmk__xe_set_content(child,
	" (in standby due to " PCMK_META_ON_FAIL ")");
	} else {
	pcmk__xe_set_content(child,
	" (in standby due to " PCMK_META_ON_FAIL ","
	" with active resources)");
	}

	} else if (pcmk_is_set(node->private->flags, pcmk__node_standby)) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_STANDBY);
	if (node->details->running_rsc == NULL) {
	pcmk__xe_set_content(child, " (in standby)");
	} else {
	pcmk__xe_set_content(child, " (in standby, with active resources)");
	}
	}

	// Maintenance mode
	if (node->details->maintenance) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_MAINT);
	pcmk__xe_set_content(child, " (in maintenance mode)");
	}

	// Node health
	if (health < 0) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_HEALTH_RED);
	pcmk__xe_set_content(child, " (health is RED)");

	} else if (health == 0) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_HEALTH_YELLOW);
	pcmk__xe_set_content(child, " (health is YELLOW)");
	}

	// Feature set
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, ", feature set %s", feature_set);
	}
	}
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool",
	"GList ", "GList ")
	static int
	node_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (full) {
	xmlNode *item_node = NULL;
	xmlNode *child = NULL;

	if (pcmk_all_flags_set(show_opts, pcmk_show_brief \| pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "Status:");
	status_node(node, item_node, show_opts);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "Resources");
	pe__rscs_brief_output(out, rscs, new_show_opts);
	out->end_list(out);
	}

	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc2 = NULL;
	int rc = pcmk_rc_no_output;

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "Status:");
	status_node(node, item_node, show_opts);

	for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc2->data;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources");

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, pcmk__map_element_name(rsc->private->xml),
	show_opts, rsc, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else {
	item_node = pcmk__output_create_xml_node(out, "li", NULL);
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "%s:", node_name);
	status_node(node, item_node, show_opts);
	}
	} else {
	out->begin_list(out, NULL, NULL, "%s:", node_name);
	}

	free(node_name);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Get a human-friendly textual description of a node's status
	*
	* \param[in] node Node to check
	*
	* \return String representation of node's status
	*/
	static const char *
	node_text_status(const pcmk_node_t *node)
	{
	if (node->details->unclean) {
	if (node->details->online) {
	return "UNCLEAN (online)";

	} else if (node->details->pending) {
	return "UNCLEAN (pending)";

	} else {
	return "UNCLEAN (offline)";
	}

	} else if (node->details->pending) {
	return "pending";

	} else if (pcmk_is_set(node->private->flags, pcmk__node_fail_standby)
	&& node->details->online) {
	return "standby (" PCMK_META_ON_FAIL ")";

	} else if (pcmk_is_set(node->private->flags, pcmk__node_standby)) {
	if (!node->details->online) {
	return "OFFLINE (standby)";
	} else if (node->details->running_rsc == NULL) {
	return "standby";
	} else {
	return "standby (with active resources)";
	}

	} else if (node->details->maintenance) {
	if (node->details->online) {
	return "maintenance";
	} else {
	return "OFFLINE (maintenance)";
	}

	} else if (node->details->online) {
	return "online";
	}

	return "OFFLINE";
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	GString *str = g_string_sized_new(64);
	int health = pe__node_health(node);

	// Create a summary line with node type, name, and status
	if (pcmk__is_guest_or_bundle_node(node)) {
	g_string_append(str, "GuestNode");
	} else if (pcmk__is_remote_node(node)) {
	g_string_append(str, "RemoteNode");
	} else {
	g_string_append(str, "Node");
	}
	pcmk__g_strcat(str, " ", node_name, ": ", node_text_status(node), NULL);

	if (health < 0) {
	g_string_append(str, " (health is RED)");
	} else if (health == 0) {
	g_string_append(str, " (health is YELLOW)");
	}
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	pcmk__g_strcat(str, ", feature set ", feature_set, NULL);
	}
	}

	/* If we're grouping by node, print its resources */
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (pcmk_is_set(show_opts, pcmk_show_brief)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	pe__rscs_brief_output(out, rscs, new_show_opts);

	out->end_list(out);
	out->end_list(out);

	g_list_free(rscs);
	}

	} else {
	GList *gIter2 = NULL;

	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter2->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, pcmk__map_element_name(rsc->private->xml),
	show_opts, rsc, only_node, only_rsc);
	}

	out->end_list(out);
	out->end_list(out);
	}
	} else {
	out->list_item(out, NULL, "%s", str->str);
	}

	g_string_free(str, TRUE);
	free(node_name);
	} else {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	out->begin_list(out, NULL, NULL, "Node: %s", node_name);
	free(node_name);
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Convert an integer health value to a string representation
	*
	* \param[in] health Integer health value
	*
	* \retval \c PCMK_VALUE_RED if \p health is less than 0
	* \retval \c PCMK_VALUE_YELLOW if \p health is equal to 0
	* \retval \c PCMK_VALUE_GREEN if \p health is greater than 0
	*/
	static const char *
	health_text(int health)
	{
	if (health < 0) {
	return PCMK_VALUE_RED;
	} else if (health == 0) {
	return PCMK_VALUE_YELLOW;
	} else {
	return PCMK_VALUE_GREEN;
	}
	}

	/*!
	* \internal
	* \brief Convert a node variant to a string representation
	*
	* \param[in] variant Node variant
	*
	* \retval \c PCMK_VALUE_MEMBER if \p node_type is \c pcmk__node_variant_cluster
	* \retval \c PCMK_VALUE_REMOTE if \p node_type is \c pcmk__node_variant_remote
	* \retval \c PCMK__VALUE_PING if \p node_type is \c pcmk__node_variant_ping
	* \retval \c PCMK_VALUE_UNKNOWN otherwise
	*/
	static const char *
	node_variant_text(enum pcmk__node_variant variant)
	{
	switch (variant) {
	case pcmk__node_variant_cluster:
	return PCMK_VALUE_MEMBER;
	case pcmk__node_variant_remote:
	return PCMK_VALUE_REMOTE;
	case pcmk__node_variant_ping:
	return PCMK__VALUE_PING;
	default:
	return PCMK_VALUE_UNKNOWN;
	}
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	const char *online = pcmk__btoa(node->details->online);
	const char *standby = pcmk__flag_text(node->private->flags,
	pcmk__node_standby);
	const char *standby_onfail = pcmk__flag_text(node->private->flags,
	pcmk__node_fail_standby);
	const char *maintenance = pcmk__btoa(node->details->maintenance);
	const char *pending = pcmk__btoa(node->details->pending);
	const char *unclean = pcmk__btoa(node->details->unclean);
	const char *health = health_text(pe__node_health(node));
	const char *feature_set = get_node_feature_set(node);
	const char *shutdown = pcmk__btoa(node->details->shutdown);
	const char *expected_up = pcmk__flag_text(node->private->flags,
	pcmk__node_expected_up);
	const bool is_dc = pcmk__same_node(node,
	node->private->scheduler->dc_node);
	int length = g_list_length(node->details->running_rsc);
	char *resources_running = pcmk__itoa(length);
	const char *node_type = node_variant_text(node->private->variant);

	int rc = pcmk_rc_ok;

	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_NODE,
	PCMK_XA_NAME, node->private->name,
	PCMK_XA_ID, node->private->id,
	PCMK_XA_ONLINE, online,
	PCMK_XA_STANDBY, standby,
	PCMK_XA_STANDBY_ONFAIL, standby_onfail,
	PCMK_XA_MAINTENANCE, maintenance,
	PCMK_XA_PENDING, pending,
	PCMK_XA_UNCLEAN, unclean,
	PCMK_XA_HEALTH, health,
	PCMK_XA_FEATURE_SET, feature_set,
	PCMK_XA_SHUTDOWN, shutdown,
	PCMK_XA_EXPECTED_UP, expected_up,
	PCMK_XA_IS_DC, pcmk__btoa(is_dc),
	PCMK_XA_RESOURCES_RUNNING, resources_running,
	PCMK_XA_TYPE, node_type,
	NULL);

	free(resources_running);
	CRM_ASSERT(rc == pcmk_rc_ok);

	if (pcmk__is_guest_or_bundle_node(node)) {
	xmlNodePtr xml_node = pcmk__output_xml_peek_parent(out);
	crm_xml_add(xml_node, PCMK_XA_ID_AS_RESOURCE,
	node->private->remote->private->launcher->id);
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc = NULL;

	for (lpc = node->details->running_rsc; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, pcmk__map_element_name(rsc->private->xml),
	show_opts, rsc, only_node, only_rsc);
	}
	}

	out->end_list(out);
	} else {
	pcmk__output_xml_create_parent(out, PCMK_XE_NODE,
	PCMK_XA_NAME, node->private->name,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_text(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}
	if (v <= 0) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is lost", name, value);
	} else if (v < expected_score) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is degraded (Expected=%d)", name, value, expected_score);
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_html(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v = 0;
	xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	if (value != NULL) {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}

	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s: %s", name, value);

	if (v <= 0) {
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "(connectivity is lost)");

	} else if (v < expected_score) {
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child,
	"(connectivity is degraded -- expected %d)",
	expected_score);
	}
	} else {
	out->list_item(out, NULL, "%s: %s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t ", "xmlNode ")
	static int
	node_and_op(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	gchar *node_str = NULL;
	char *last_change_str = NULL;

	const char *op_rsc = crm_element_value(xml_op, PCMK_XA_RESOURCE);
	int status;
	time_t last_change = 0;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
	PCMK_EXEC_UNKNOWN);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const pcmk_node_t *node = pcmk__current_node(rsc);
	const char *target_role = g_hash_table_lookup(rsc->private->meta,
	PCMK_META_TARGET_ROLE);
	uint32_t show_opts = pcmk_show_rsc_only \| pcmk_show_pending;

	if (node == NULL) {
	node = rsc->private->pending_node;
	}

	node_str = pcmk__native_output_string(rsc, rsc_printable_id(rsc), node,
	show_opts, target_role, false);
	} else {
	node_str = crm_strdup_printf("Unknown resource %s", op_rsc);
	}

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change) == pcmk_ok) {
	const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);

	last_change_str = crm_strdup_printf(", %s='%s', exec=%sms",
	PCMK_XA_LAST_RC_CHANGE,
	pcmk__trim(ctime(&last_change)),
	exec_time);
	}

	out->list_item(out, NULL, "%s: %s (node=%s, call=%s, rc=%s%s): %s",
	node_str, pcmk__xe_history_key(xml_op),
	crm_element_value(xml_op, PCMK_XA_UNAME),
	crm_element_value(xml_op, PCMK__XA_CALL_ID),
	crm_element_value(xml_op, PCMK__XA_RC_CODE),
	last_change_str ? last_change_str : "",
	pcmk_exec_status_str(status));

	g_free(node_str);
	free(last_change_str);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t ", "xmlNode ")
	static int
	node_and_op_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	const char *rc_s = crm_element_value(xml_op, PCMK__XA_RC_CODE);
	const char *status_s = NULL;
	const char *op_rsc = crm_element_value(xml_op, PCMK_XA_RESOURCE);
	int status;
	time_t last_change = 0;
	xmlNode *node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS),
	&status, PCMK_EXEC_UNKNOWN);
	status_s = pcmk_exec_status_str(status);

	node = pcmk__output_create_xml_node(out, PCMK_XE_OPERATION,
	PCMK_XA_OP, pcmk__xe_history_key(xml_op),
	PCMK_XA_NODE, uname,
	PCMK_XA_CALL, call_id,
	PCMK_XA_RC, rc_s,
	PCMK_XA_STATUS, status_s,
	NULL);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const char *class = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);
	const char *provider = crm_element_value(rsc->private->xml,
	PCMK_XA_PROVIDER);
	const char *kind = crm_element_value(rsc->private->xml, PCMK_XA_TYPE);
	bool has_provider = pcmk_is_set(pcmk_get_ra_caps(class),
	pcmk_ra_cap_provider);

	char *agent_tuple = crm_strdup_printf("%s:%s:%s",
	class,
	(has_provider? provider : ""),
	kind);

	pcmk__xe_set_props(node,
	PCMK_XA_RSC, rsc_printable_id(rsc),
	PCMK_XA_AGENT, agent_tuple,
	NULL);
	free(agent_tuple);
	}

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change) == pcmk_ok) {
	const char *last_rc_change = pcmk__trim(ctime(&last_change));
	const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);

	pcmk__xe_set_props(node,
	PCMK_XA_LAST_RC_CHANGE, last_rc_change,
	PCMK_XA_EXEC_TIME, exec_time,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_xml(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_ATTRIBUTE,
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);

	if (add_extra) {
	char *buf = pcmk__itoa(expected_score);
	crm_xml_add(node, PCMK_XA_EXPECTED, buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute-list", "pcmk_scheduler_t *", "uint32_t",
	"bool", "GList ", "GList ")
	static int
	node_attribute_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	int rc = pcmk_rc_no_output;

	/* Display each node's attributes */
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;

	GList *attr_list = NULL;
	GHashTableIter iter;
	gpointer key;

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	g_hash_table_iter_init(&iter, node->private->attrs);
	while (g_hash_table_iter_next (&iter, &key, NULL)) {
	attr_list = filter_attr_list(attr_list, key);
	}

	if (attr_list == NULL) {
	continue;
	}

	if (!pcmk__str_in_list(node->private->name, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	g_list_free(attr_list);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node Attributes");

	out->message(out, "node", node, show_opts, false, only_node, only_rsc);

	for (GList *aIter = attr_list; aIter != NULL; aIter = aIter->next) {
	const char *name = aIter->data;
	const char *value = NULL;
	int expected_score = 0;
	bool add_extra = false;

	value = pcmk__node_attr(node, name, NULL, pcmk__rsc_node_current);

	add_extra = add_extra_info(node, node->details->running_rsc,
	scheduler, name, &expected_score);

	/* Print attribute name and value */
	out->message(out, "node-attribute", name, value, add_extra,
	expected_score);
	}

	g_list_free(attr_list);
	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s capacity:",
	comment, pcmk__node_name(node));

	g_hash_table_foreach(node->private->utilization, append_dump_text,
	&dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char *uname = node->private->name;
	const char comment = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, PCMK_XE_CAPACITY,
	PCMK_XA_NODE, uname,
	PCMK_XA_COMMENT, comment,
	NULL);
	g_hash_table_foreach(node->private->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-history-list", "pcmk_scheduler_t ", "pcmk_node_t ",
	"xmlNode ", "GList ", "GList *", "uint32_t", "uint32_t")
	static int
	node_history_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	xmlNode node_state = va_arg(args, xmlNode );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	xmlNode *lrm_rsc = NULL;
	xmlNode *rsc_entry = NULL;
	int rc = pcmk_rc_no_output;

	lrm_rsc = pcmk__xe_first_child(node_state, PCMK__XE_LRM, NULL, NULL);
	lrm_rsc = pcmk__xe_first_child(lrm_rsc, PCMK__XE_LRM_RESOURCES, NULL, NULL);

	/* Print history of each of the node's resources */
	for (rsc_entry = pcmk__xe_first_child(lrm_rsc, PCMK__XE_LRM_RESOURCE, NULL,
	NULL);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next_same(rsc_entry)) {

	const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	/* We can't use is_filtered here to filter group resources. For is_filtered,
	* we have to decide whether to check the parent or not. If we check the
	* parent, all elements of a group will always be printed because that's how
	* is_filtered works for groups. If we do not check the parent, sometimes
	* this will filter everything out.
	*
	* For other resource types, is_filtered is okay.
	*/
	if (pcmk__is_group(parent)) {
	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(parent), only_rsc,
	pcmk__str_star_matches)) {
	continue;
	}
	} else if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	continue;
	}

	if (!pcmk_is_set(section_opts, pcmk_section_operations)) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	if (failcount <= 0) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-history", rsc, rsc_id, false,
	failcount, last_failure, false);
	} else {
	GList *op_list = get_operation_list(rsc_entry);
	pcmk_resource_t *rsc = NULL;

	if (op_list == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources,
	crm_element_value(rsc_entry, PCMK_XA_ID));

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-operation-list", scheduler, rsc, node,
	op_list, show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_html(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->private->name, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Node List");

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_text(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	/* space-separated lists of node names */
	GString *online_nodes = NULL;
	GString *online_remote_nodes = NULL;
	GString *online_guest_nodes = NULL;
	GString *offline_nodes = NULL;
	GString *offline_remote_nodes = NULL;

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (!pcmk__str_in_list(node->private->name, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	free(node_name);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node List");

	// Determine whether to display node individually or in a list
	if (node->details->unclean \|\| node->details->pending
	\|\| (pcmk_is_set(node->private->flags, pcmk__node_fail_standby)
	&& node->details->online)
	\|\| pcmk_is_set(node->private->flags, pcmk__node_standby)
	\|\| node->details->maintenance
	\|\| pcmk_is_set(show_opts, pcmk_show_rscs_by_node)
	\|\| pcmk_is_set(show_opts, pcmk_show_feature_set)
	\|\| (pe__node_health(node) <= 0)) {
	// Display node individually

	} else if (node->details->online) {
	// Display online node in a list
	if (pcmk__is_guest_or_bundle_node(node)) {
	pcmk__add_word(&online_guest_nodes, 1024, node_name);

	} else if (pcmk__is_remote_node(node)) {
	pcmk__add_word(&online_remote_nodes, 1024, node_name);

	} else {
	pcmk__add_word(&online_nodes, 1024, node_name);
	}
	free(node_name);
	continue;

	} else {
	// Display offline node in a list
	if (pcmk__is_remote_node(node)) {
	pcmk__add_word(&offline_remote_nodes, 1024, node_name);

	} else if (pcmk__is_guest_or_bundle_node(node)) {
	/* ignore offline guest nodes */

	} else {
	pcmk__add_word(&offline_nodes, 1024, node_name);
	}
	free(node_name);
	continue;
	}

	/* If we get here, node is in bad state, or we're grouping by node */
	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	free(node_name);
	}

	/* If we're not grouping by node, summarize nodes by status */
	if (online_nodes != NULL) {
	out->list_item(out, "Online", "[ %s ]",
	(const char *) online_nodes->str);
	g_string_free(online_nodes, TRUE);
	}
	if (offline_nodes != NULL) {
	out->list_item(out, "OFFLINE", "[ %s ]",
	(const char *) offline_nodes->str);
	g_string_free(offline_nodes, TRUE);
	}
	if (online_remote_nodes) {
	out->list_item(out, "RemoteOnline", "[ %s ]",
	(const char *) online_remote_nodes->str);
	g_string_free(online_remote_nodes, TRUE);
	}
	if (offline_remote_nodes) {
	out->list_item(out, "RemoteOFFLINE", "[ %s ]",
	(const char *) offline_remote_nodes->str);
	g_string_free(offline_remote_nodes, TRUE);
	}
	if (online_guest_nodes != NULL) {
	out->list_item(out, "GuestOnline", "[ %s ]",
	(const char *) online_guest_nodes->str);
	g_string_free(online_guest_nodes, TRUE);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_xml(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	/* PCMK_XE_NODES acts as the list's element name for CLI tools that use
	* pcmk__output_enable_list_element. Otherwise PCMK_XE_NODES is the
	* value of the list's PCMK_XA_NAME attribute.
	*/
	out->begin_list(out, NULL, NULL, PCMK_XE_NODES);
	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->private->name, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}
	out->end_list(out);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-summary", "pcmk_scheduler_t ", "GList ", "GList *",
	"uint32_t", "uint32_t", "bool")
	static int
	node_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *node_state = NULL;
	xmlNode *cib_status = pcmk_find_cib_element(scheduler->input,
	PCMK_XE_STATUS);
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(cib_status) == 0) {
	return rc;
	}

	for (node_state = pcmk__xe_first_child(cib_status, PCMK__XE_NODE_STATE,
	NULL, NULL);
	node_state != NULL; node_state = pcmk__xe_next_same(node_state)) {

	pcmk_node_t *node = pe_find_node_id(scheduler->nodes,
	pcmk__xe_id(node_state));

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	if (!pcmk__str_in_list(node->private->name, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc,
	pcmk_is_set(section_opts, pcmk_section_operations) ? "Operations" : "Migration Summary");

	out->message(out, "node-history-list", scheduler, node, node_state,
	only_node, only_rsc, section_opts, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	if (rsc) {
	out->list_item(out, NULL, "%s: %s allocation score on %s: %s",
	prefix, rsc->id, uname, score);
	} else {
	out->list_item(out, NULL, "%s: %s = %s", prefix, uname, score);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_NODE_WEIGHT,
	PCMK_XA_FUNCTION, prefix,
	PCMK_XA_NODE, uname,
	PCMK_XA_SCORE, score,
	NULL);

	if (rsc) {
	crm_xml_add(node, PCMK_XA_ID, rsc->id);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNode ", "const char ", "const char *", "int", "uint32_t")
	static int
	op_history_text(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	char *buf = op_history_string(xml_op, task, interval_ms_s, rc,
	pcmk_is_set(show_opts, pcmk_show_timing));

	out->list_item(out, NULL, "%s", buf);

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNode ", "const char ", "const char *", "int", "uint32_t")
	static int
	op_history_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	char *rc_s = pcmk__itoa(rc);
	const char *rc_text = services_ocf_exitcode_str(rc);
	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, PCMK_XE_OPERATION_HISTORY,
	PCMK_XA_CALL, call_id,
	PCMK_XA_TASK, task,
	PCMK_XA_RC, rc_s,
	PCMK_XA_RC_TEXT, rc_text,
	NULL);
	free(rc_s);

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *s = crm_strdup_printf("%sms", interval_ms_s);
	crm_xml_add(node, PCMK_XA_INTERVAL, s);
	free(s);
	}

	if (pcmk_is_set(show_opts, pcmk_show_timing)) {
	const char *value = NULL;
	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {
	char *s = pcmk__epoch2str(&epoch, 0);
	crm_xml_add(node, PCMK_XA_LAST_RC_CHANGE, s);
	free(s);
	}

	value = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, PCMK_XA_EXEC_TIME, s);
	free(s);
	}
	value = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, PCMK_XA_QUEUE_TIME, s);
	free(s);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	if (chosen == NULL) {
	out->list_item(out, NULL, "%s promotion score (inactive): %s",
	child_rsc->id, score);
	} else {
	out->list_item(out, NULL, "%s promotion score on %s: %s",
	child_rsc->id, pcmk__node_name(chosen), score);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_PROMOTION_SCORE,
	PCMK_XA_ID, child_rsc->id,
	PCMK_XA_SCORE, score,
	NULL);

	if (chosen) {
	crm_xml_add(node, PCMK_XA_NODE, chosen->private->name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	GString *xml_buf = g_string_sized_new(1024);
	bool raw = va_arg(args, int);

	formatted_xml_buf(rsc, xml_buf, raw);

	out->output_xml(out, PCMK_XE_XML, xml_buf->str);

	g_string_free(xml_buf, TRUE);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config_text(pcmk__output_t *out, va_list args) {
	pcmk__formatted_printf(out, "Resource XML:\n");
	return resource_config(out, args);
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	char *buf = resource_history_string(rsc, rsc_id, all, failcount, last_failure);

	if (as_header) {
	out->begin_list(out, NULL, NULL, "%s", buf);
	} else {
	out->list_item(out, NULL, "%s", buf);
	}

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	xmlNodePtr node = pcmk__output_xml_create_parent(out,
	PCMK_XE_RESOURCE_HISTORY,
	PCMK_XA_ID, rsc_id,
	NULL);

	if (rsc == NULL) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_ORPHAN, true);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *migration_s = pcmk__itoa(rsc->private->ban_after_failures);

	pcmk__xe_set_props(node,
	PCMK_XA_ORPHAN, PCMK_VALUE_FALSE,
	PCMK_META_MIGRATION_THRESHOLD, migration_s,
	NULL);
	free(migration_s);

	if (failcount > 0) {
	char *s = pcmk__itoa(failcount);

	crm_xml_add(node, PCMK_XA_FAIL_COUNT, s);
	free(s);
	}

	if (last_failure > 0) {
	char *s = pcmk__epoch2str(&last_failure, 0);

	crm_xml_add(node, PCMK_XA_LAST_FAILURE, s);
	free(s);
	}
	}

	if (!as_header) {
	pcmk__output_xml_pop_parent(out);
	}

	return pcmk_rc_ok;
	}

	static void
	print_resource_header(pcmk__output_t *out, uint32_t show_opts)
	{
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	/* Active resources have already been printed by node */
	out->begin_list(out, NULL, NULL, "Inactive Resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->begin_list(out, NULL, NULL, "Full List of Resources");
	} else {
	out->begin_list(out, NULL, NULL, "Active Resources");
	}
	}


	PCMK__OUTPUT_ARGS("resource-list", "pcmk_scheduler_t *", "uint32_t", "bool",
	"GList ", "GList ", "bool")
	static int
	resource_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_summary = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	bool print_spacer = va_arg(args, int);

	GList *rsc_iter;
	int rc = pcmk_rc_no_output;
	bool printed_header = false;

	/* If we already showed active resources by node, and
	* we're not showing inactive resources, we have nothing to do
	*/
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node) &&
	!pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	return rc;
	}

	/* If we haven't already printed resources grouped by node,
	* and brief output was requested, print resource summary */
	if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& !pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(scheduler->resources, only_rsc);

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;

	rc = pe__rscs_brief_output(out, rscs, show_opts);
	g_list_free(rscs);
	}

	/* For each resource, display it if appropriate */
	for (rsc_iter = scheduler->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rsc_iter->data;
	int x;

	/* Complex resources may have some sub-resources active and some inactive */
	gboolean is_active = rsc->private->fns->active(rsc, TRUE);
	gboolean partially_active = rsc->private->fns->active(rsc, FALSE);

	/* Skip inactive orphans (deleted but still in CIB) */
	if (pcmk_is_set(rsc->flags, pcmk__rsc_removed) && !is_active) {
	continue;

	/* Skip active resources if we already displayed them by node */
	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (is_active) {
	continue;
	}

	/* Skip primitives already counted in a brief summary */
	} else if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& pcmk__is_primitive(rsc)) {
	continue;

	/* Skip resources that aren't at least partially active,
	* unless we're displaying inactive resources
	*/
	} else if (!partially_active && !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	continue;

	} else if (partially_active && !pe__rsc_running_on_any(rsc, only_node)) {
	continue;
	}

	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	/* Print this resource */
	x = out->message(out, pcmk__map_element_name(rsc->private->xml),
	show_opts, rsc, only_node, only_rsc);
	if (x == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	if (print_summary && rc != pcmk_rc_ok) {
	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	out->list_item(out, NULL, "No inactive resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "No resources");
	} else {
	out->list_item(out, NULL, "No active resources");
	}
	}

	if (printed_header) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-operation-list", "pcmk_scheduler_t *",
	"pcmk_resource_t ", "pcmk_node_t ", "GList *", "uint32_t")
	static int
	resource_operation_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t *scheduler G_GNUC_UNUSED = va_arg(args,
	pcmk_scheduler_t *);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	GList op_list = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);

	GList *gIter = NULL;
	int rc = pcmk_rc_no_output;

	/* Print each operation */
	for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode xml_op = (xmlNode ) gIter->data;
	const char *task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	const char *interval_ms_s = crm_element_value(xml_op,
	PCMK_META_INTERVAL);
	const char *op_rc = crm_element_value(xml_op, PCMK__XA_RC_CODE);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* If this is the first printed operation, print heading for resource */
	if (rc == pcmk_rc_no_output) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	out->message(out, "resource-history", rsc, rsc_printable_id(rsc), true,
	failcount, last_failure, true);
	rc = pcmk_rc_ok;
	}

	/* Print the operation */
	out->message(out, "op-history", xml_op, task, interval_ms_s,
	op_rc_i, show_opts);
	}

	/* Free the list we created (no need to free the individual items) */
	g_list_free(op_list);

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
	fn, rsc->id, pcmk__node_name(node));

	g_hash_table_foreach(rsc->private->utilization, append_dump_text,
	&dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char *uname = node->private->name;
	const char fn = va_arg(args, const char );

	xmlNodePtr xml_node = NULL;

	xml_node = pcmk__output_create_xml_node(out, PCMK_XE_UTILIZATION,
	PCMK_XA_RESOURCE, rsc->id,
	PCMK_XA_NODE, uname,
	PCMK_XA_FUNCTION, fn,
	NULL);
	g_hash_table_foreach(rsc->private->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *", "bool", "bool")
	static int
	ticket_default(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );
	bool raw = va_arg(args, int);
	bool details = va_arg(args, int);

	GString *detail_str = NULL;

	if (raw) {
	out->list_item(out, ticket->id, "%s", ticket->id);
	return pcmk_rc_ok;
	}

	if (details && g_hash_table_size(ticket->state) > 0) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;
	bool already_added = false;

	detail_str = g_string_sized_new(100);
	pcmk__g_strcat(detail_str, "\t(", NULL);

	g_hash_table_iter_init(&iter, ticket->state);
	while (g_hash_table_iter_next(&iter, (void ) &name, (void ) &value)) {
	if (already_added) {
	g_string_append_printf(detail_str, ", %s=", name);
	} else {
	g_string_append_printf(detail_str, "%s=", name);
	already_added = true;
	}

	if (pcmk__str_any_of(name, PCMK_XA_LAST_GRANTED, "expires", NULL)) {
	char *epoch_str = NULL;
	long long time_ll;

	pcmk__scan_ll(value, &time_ll, 0);
	epoch_str = pcmk__epoch2str((const time_t *) &time_ll, 0);
	pcmk__g_strcat(detail_str, epoch_str, NULL);
	free(epoch_str);
	} else {
	pcmk__g_strcat(detail_str, value, NULL);
	}
	}

	pcmk__g_strcat(detail_str, ")", NULL);
	}

	if (ticket->last_granted > -1) {
	/* Prior to the introduction of the details & raw arguments to this
	* function, last-granted would always be added in this block. We need
	* to preserve that behavior. At the same time, we also need to preserve
	* the existing behavior from crm_ticket, which would include last-granted
	* as part of the (...) detail string.
	*
	* Luckily we can check detail_str - if it's NULL, either there were no
	* details, or we are preserving the previous behavior of this function.
	* If it's not NULL, we are either preserving the previous behavior of
	* crm_ticket or we were given details=true as an argument.
	*/
	if (detail_str == NULL) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, NULL, "%s\t%s%s last-granted=\"%s\"",
	ticket->id,
	(ticket->granted? "granted" : "revoked"),
	(ticket->standby? " [standby]" : ""),
	pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, NULL, "%s\t%s%s %s",
	ticket->id,
	(ticket->granted? "granted" : "revoked"),
	(ticket->standby? " [standby]" : ""),
	detail_str->str);
	}
	} else {
	out->list_item(out, NULL, "%s\t%s%s%s", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	detail_str != NULL ? detail_str->str : "");
	}

	if (detail_str != NULL) {
	g_string_free(detail_str, TRUE);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *", "bool", "bool")
	static int
	ticket_xml(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );
	bool raw G_GNUC_UNUSED = va_arg(args, int);
	bool details G_GNUC_UNUSED = va_arg(args, int);

	const char *status = NULL;
	const char *standby = pcmk__btoa(ticket->standby);

	xmlNodePtr node = NULL;
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	status = ticket->granted? PCMK_VALUE_GRANTED : PCMK_VALUE_REVOKED;

	node = pcmk__output_create_xml_node(out, PCMK_XE_TICKET,
	PCMK_XA_ID, ticket->id,
	PCMK_XA_STATUS, status,
	PCMK_XA_STANDBY, standby,
	NULL);

	if (ticket->last_granted > -1) {
	char *buf = pcmk__epoch2str(&ticket->last_granted, 0);

	crm_xml_add(node, PCMK_XA_LAST_GRANTED, buf);
	free(buf);
	}

	g_hash_table_iter_init(&iter, ticket->state);
	while (g_hash_table_iter_next(&iter, (void ) &name, (void ) &value)) {
	/* PCMK_XA_LAST_GRANTED and "expires" are already added by the check
	* for ticket->last_granted above.
	*/
	if (pcmk__str_any_of(name, PCMK_XA_LAST_GRANTED, PCMK_XA_EXPIRES,
	NULL)) {
	continue;
	}

	crm_xml_add(node, name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-list", "GHashTable *", "bool", "bool", "bool")
	static int
	ticket_list(pcmk__output_t *out, va_list args) {
	GHashTable tickets = va_arg(args, GHashTable );
	bool print_spacer = va_arg(args, int);
	bool raw = va_arg(args, int);
	bool details = va_arg(args, int);

	GHashTableIter iter;
	gpointer value;

	if (g_hash_table_size(tickets) == 0) {
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);

	/* Print section heading */
	out->begin_list(out, NULL, NULL, "Tickets");

	/* Print each ticket */
	g_hash_table_iter_init(&iter, tickets);
	while (g_hash_table_iter_next(&iter, NULL, &value)) {
	pcmk_ticket_t ticket = (pcmk_ticket_t ) value;
	out->message(out, "ticket", ticket, raw, details);
	}

	/* Close section */
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "ban", "default", ban_text },
	{ "ban", "html", ban_html },
	{ "ban", "xml", ban_xml },
	{ "ban-list", "default", ban_list },
	{ "bundle", "default", pe__bundle_text },
	{ "bundle", "xml", pe__bundle_xml },
	{ "bundle", "html", pe__bundle_html },
	{ "clone", "default", pe__clone_default },
	{ "clone", "xml", pe__clone_xml },
	{ "cluster-counts", "default", cluster_counts_text },
	{ "cluster-counts", "html", cluster_counts_html },
	{ "cluster-counts", "xml", cluster_counts_xml },
	{ "cluster-dc", "default", cluster_dc_text },
	{ "cluster-dc", "html", cluster_dc_html },
	{ "cluster-dc", "xml", cluster_dc_xml },
	{ "cluster-options", "default", cluster_options_text },
	{ "cluster-options", "html", cluster_options_html },
	{ "cluster-options", "log", cluster_options_log },
	{ "cluster-options", "xml", cluster_options_xml },
	{ "cluster-summary", "default", cluster_summary },
	{ "cluster-summary", "html", cluster_summary_html },
	{ "cluster-stack", "default", cluster_stack_text },
	{ "cluster-stack", "html", cluster_stack_html },
	{ "cluster-stack", "xml", cluster_stack_xml },
	{ "cluster-times", "default", cluster_times_text },
	{ "cluster-times", "html", cluster_times_html },
	{ "cluster-times", "xml", cluster_times_xml },
	{ "failed-action", "default", failed_action_default },
	{ "failed-action", "xml", failed_action_xml },
	{ "failed-action-list", "default", failed_action_list },
	{ "group", "default", pe__group_default},
	{ "group", "xml", pe__group_xml },
	{ "maint-mode", "text", cluster_maint_mode_text },
	{ "node", "default", node_text },
	{ "node", "html", node_html },
	{ "node", "xml", node_xml },
	{ "node-and-op", "default", node_and_op },
	{ "node-and-op", "xml", node_and_op_xml },
	{ "node-capacity", "default", node_capacity },
	{ "node-capacity", "xml", node_capacity_xml },
	{ "node-history-list", "default", node_history_list },
	{ "node-list", "default", node_list_text },
	{ "node-list", "html", node_list_html },
	{ "node-list", "xml", node_list_xml },
	{ "node-weight", "default", node_weight },
	{ "node-weight", "xml", node_weight_xml },
	{ "node-attribute", "default", node_attribute_text },
	{ "node-attribute", "html", node_attribute_html },
	{ "node-attribute", "xml", node_attribute_xml },
	{ "node-attribute-list", "default", node_attribute_list },
	{ "node-summary", "default", node_summary },
	{ "op-history", "default", op_history_text },
	{ "op-history", "xml", op_history_xml },
	{ "primitive", "default", pe__resource_text },
	{ "primitive", "xml", pe__resource_xml },
	{ "primitive", "html", pe__resource_html },
	{ "promotion-score", "default", promotion_score },
	{ "promotion-score", "xml", promotion_score_xml },
	{ "resource-config", "default", resource_config },
	{ "resource-config", "text", resource_config_text },
	{ "resource-history", "default", resource_history_text },
	{ "resource-history", "xml", resource_history_xml },
	{ "resource-list", "default", resource_list },
	{ "resource-operation-list", "default", resource_operation_list },
	{ "resource-util", "default", resource_util },
	{ "resource-util", "xml", resource_util_xml },
	{ "ticket", "default", ticket_default },
	{ "ticket", "xml", ticket_xml },
	{ "ticket-list", "default", ticket_list },

	{ NULL, NULL, NULL }
	};

	void
	pe__register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/lib/pengine/status.c b/lib/pengine/status.c
	index 0bc3510b1b..83e6365baf 100644
	--- a/lib/pengine/status.c
	+++ b/lib/pengine/status.c
	@@ -1,522 +1,523 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/cib_internal.h>

	#include <glib.h>

	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	/*!
	* \brief Create a new object to hold scheduler data
	*
	* \return New, initialized scheduler data on success, else NULL (and set errno)
	* \note Only pcmk_scheduler_t objects created with this function (as opposed
	* to statically declared or directly allocated) should be used with the
	* functions in this library, to allow for future extensions to the
	* data type. The caller is responsible for freeing the memory with
	* pe_free_working_set() when the instance is no longer needed.
	*/
	pcmk_scheduler_t *
	pe_new_working_set(void)
	{
	pcmk_scheduler_t *scheduler = calloc(1, sizeof(pcmk_scheduler_t));

	if (scheduler != NULL) {
	set_working_set_defaults(scheduler);
	}
	return scheduler;
	}

	/*!
	* \brief Free scheduler data
	*
	* \param[in,out] scheduler Scheduler data to free
	*/
	void
	pe_free_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler != NULL) {
	pe_reset_working_set(scheduler);
	scheduler->priv = NULL;
	free(scheduler);
	}
	}

	#define XPATH_DEPRECATED_RULES \
	"//" PCMK_XE_OP_DEFAULTS "//" PCMK_XE_EXPRESSION \
	"\|//" PCMK_XE_OP "//" PCMK_XE_EXPRESSION

	/*!
	* \internal
	* \brief Log a warning for deprecated rule syntax in operations
	*
	* \param[in] scheduler Scheduler data
	*/
	static void
	check_for_deprecated_rules(pcmk_scheduler_t *scheduler)
	{
	// @COMPAT Drop this function when support for the syntax is dropped
	xmlNode *deprecated = get_xpath_object(XPATH_DEPRECATED_RULES,
	scheduler->input, LOG_NEVER);

	if (deprecated != NULL) {
	pcmk__warn_once(pcmk__wo_op_attr_expr,
	"Support for rules with node attribute expressions in "
	PCMK_XE_OP " or " PCMK_XE_OP_DEFAULTS " is deprecated "
	"and will be dropped in a future release");
	}
	}

	/*
	* Unpack everything
	* At the end you'll have:
	* - A list of nodes
	* - A list of resources (each with any dependencies on other resources)
	* - A list of constraints between resources and nodes
	* - A list of constraints between start/stop actions
	* - A list of nodes that need to be stonith'd
	* - A list of nodes that need to be shutdown
	* - A list of the possible stop/start actions (without dependencies)
	*/
	gboolean
	cluster_status(pcmk_scheduler_t * scheduler)
	{
	const char *new_version = NULL;
	xmlNode *section = NULL;

	if ((scheduler == NULL) \|\| (scheduler->input == NULL)) {
	return FALSE;
	}

	new_version = crm_element_value(scheduler->input, PCMK_XA_CRM_FEATURE_SET);

	if (pcmk__check_feature_set(new_version) != pcmk_rc_ok) {
	pcmk__config_err("Can't process CIB with feature set '%s' greater than our own '%s'",
	new_version, CRM_FEATURE_SET);
	return FALSE;
	}

	crm_trace("Beginning unpack");

	if (scheduler->failed != NULL) {
	pcmk__xml_free(scheduler->failed);
	}
	scheduler->failed = pcmk__xe_create(NULL, "failed-ops");

	if (scheduler->now == NULL) {
	scheduler->now = crm_time_new(NULL);
	}

	if (scheduler->dc_uuid == NULL) {
	scheduler->dc_uuid = crm_element_value_copy(scheduler->input,
	PCMK_XA_DC_UUID);
	}

	if (pcmk__xe_attr_is_true(scheduler->input, PCMK_XA_HAVE_QUORUM)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_quorate);
	} else {
	pcmk__clear_scheduler_flags(scheduler, pcmk_sched_quorate);
	}

	scheduler->op_defaults = get_xpath_object("//" PCMK_XE_OP_DEFAULTS,
	scheduler->input, LOG_NEVER);
	check_for_deprecated_rules(scheduler);

	scheduler->rsc_defaults = get_xpath_object("//" PCMK_XE_RSC_DEFAULTS,
	scheduler->input, LOG_NEVER);

	section = get_xpath_object("//" PCMK_XE_CRM_CONFIG, scheduler->input,
	LOG_TRACE);
	unpack_config(section, scheduler);

	if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_location_only\|pcmk_sched_quorate)
	&& (scheduler->no_quorum_policy != pcmk_no_quorum_ignore)) {
	pcmk__sched_warn("Fencing and resource management disabled "
	"due to lack of quorum");
	}

	section = get_xpath_object("//" PCMK_XE_NODES, scheduler->input, LOG_TRACE);
	unpack_nodes(section, scheduler);

	section = get_xpath_object("//" PCMK_XE_RESOURCES, scheduler->input,
	LOG_TRACE);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	unpack_remote_nodes(section, scheduler);
	}
	unpack_resources(section, scheduler);

	section = get_xpath_object("//" PCMK_XE_TAGS, scheduler->input, LOG_NEVER);
	unpack_tags(section, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	section = get_xpath_object("//" PCMK_XE_STATUS, scheduler->input,
	LOG_TRACE);
	unpack_status(section, scheduler);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_no_counts)) {
	for (GList *item = scheduler->resources; item != NULL;
	item = item->next) {
	pcmk_resource_t *rsc = item->data;

	rsc->private->fns->count(item->data);
	}
	crm_trace("Cluster resource count: %d (%d disabled, %d blocked)",
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_status);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Free a list of pcmk_resource_t
	*
	* \param[in,out] resources List to free
	*
	* \note When the scheduler's resource list is freed, that includes the original
	* storage for the uname and id of any Pacemaker Remote nodes in the
	* scheduler's node list, so take care not to use those afterward.
	* \todo Refactor pcmk_node_t to strdup() the node name.
	*/
	static void
	pe_free_resources(GList *resources)
	{
	pcmk_resource_t *rsc = NULL;
	GList *iterator = resources;

	while (iterator != NULL) {
	rsc = (pcmk_resource_t *) iterator->data;
	iterator = iterator->next;
	rsc->private->fns->free(rsc);
	}
	if (resources != NULL) {
	g_list_free(resources);
	}
	}

	static void
	pe_free_actions(GList *actions)
	{
	GList *iterator = actions;

	while (iterator != NULL) {
	pe_free_action(iterator->data);
	iterator = iterator->next;
	}
	if (actions != NULL) {
	g_list_free(actions);
	}
	}

	static void
	pe_free_nodes(GList *nodes)
	{
	for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) {
	pcmk_node_t node = (pcmk_node_t ) iterator->data;

	// Shouldn't be possible, but to be safe ...
	if (node == NULL) {
	continue;
	}
	if (node->details == NULL) {
	free(node);
	continue;
	}

	/* This is called after pe_free_resources(), which means that we can't
	* use node->private->name for Pacemaker Remote nodes.
	*/
	crm_trace("Freeing node %s", (pcmk__is_pacemaker_remote_node(node)?
	"(guest or remote)" : pcmk__node_name(node)));

	if (node->private->attrs != NULL) {
	g_hash_table_destroy(node->private->attrs);
	}
	if (node->private->utilization != NULL) {
	g_hash_table_destroy(node->private->utilization);
	}
	if (node->private->digest_cache != NULL) {
	g_hash_table_destroy(node->private->digest_cache);
	}
	g_list_free(node->details->running_rsc);
	g_list_free(node->private->assigned_resources);
	free(node->private);
	free(node->details);
	+ free(node->assign);
	free(node);
	}
	if (nodes != NULL) {
	g_list_free(nodes);
	}
	}

	static void
	pe__free_ordering(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__action_relation_t *order = iterator->data;

	iterator = iterator->next;

	free(order->task1);
	free(order->task2);
	free(order);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	static void
	pe__free_location(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__location_t *cons = iterator->data;

	iterator = iterator->next;

	g_list_free_full(cons->nodes, free);
	free(cons->id);
	free(cons);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	/*!
	* \brief Reset scheduler data to defaults without freeing it or constraints
	*
	* \param[in,out] scheduler Scheduler data to reset
	*
	* \deprecated This function is deprecated as part of the API;
	* pe_reset_working_set() should be used instead.
	*/
	void
	cleanup_calculations(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	pcmk__clear_scheduler_flags(scheduler, pcmk_sched_have_status);
	if (scheduler->config_hash != NULL) {
	g_hash_table_destroy(scheduler->config_hash);
	}

	if (scheduler->singletons != NULL) {
	g_hash_table_destroy(scheduler->singletons);
	}

	if (scheduler->tickets) {
	g_hash_table_destroy(scheduler->tickets);
	}

	if (scheduler->template_rsc_sets) {
	g_hash_table_destroy(scheduler->template_rsc_sets);
	}

	if (scheduler->tags) {
	g_hash_table_destroy(scheduler->tags);
	}

	free(scheduler->dc_uuid);

	crm_trace("deleting resources");
	pe_free_resources(scheduler->resources);

	crm_trace("deleting actions");
	pe_free_actions(scheduler->actions);

	crm_trace("deleting nodes");
	pe_free_nodes(scheduler->nodes);

	pe__free_param_checks(scheduler);
	g_list_free(scheduler->stop_needed);
	pcmk__xml_free(scheduler->graph);
	crm_time_free(scheduler->now);
	pcmk__xml_free(scheduler->input);
	pcmk__xml_free(scheduler->failed);

	set_working_set_defaults(scheduler);

	CRM_CHECK(scheduler->ordering_constraints == NULL,;
	);
	CRM_CHECK(scheduler->placement_constraints == NULL,;
	);
	}

	/*!
	* \brief Reset scheduler data to default state without freeing it
	*
	* \param[in,out] scheduler Scheduler data to reset
	*/
	void
	pe_reset_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	crm_trace("Deleting %d ordering constraints",
	g_list_length(scheduler->ordering_constraints));
	pe__free_ordering(scheduler->ordering_constraints);
	scheduler->ordering_constraints = NULL;

	crm_trace("Deleting %d location constraints",
	g_list_length(scheduler->placement_constraints));
	pe__free_location(scheduler->placement_constraints);
	scheduler->placement_constraints = NULL;

	crm_trace("Deleting %d colocation constraints",
	g_list_length(scheduler->colocation_constraints));
	g_list_free_full(scheduler->colocation_constraints, free);
	scheduler->colocation_constraints = NULL;

	crm_trace("Deleting %d ticket constraints",
	g_list_length(scheduler->ticket_constraints));
	g_list_free_full(scheduler->ticket_constraints, free);
	scheduler->ticket_constraints = NULL;

	cleanup_calculations(scheduler);
	}

	void
	set_working_set_defaults(pcmk_scheduler_t *scheduler)
	{
	void *priv = scheduler->priv;

	memset(scheduler, 0, sizeof(pcmk_scheduler_t));

	scheduler->priv = priv;
	scheduler->order_id = 1;
	scheduler->action_id = 1;
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;

	scheduler->flags = 0x0ULL;

	pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_symmetric_cluster
	\|pcmk_sched_stop_removed_resources
	\|pcmk_sched_cancel_removed_actions);
	if (!strcmp(PCMK__CONCURRENT_FENCING_DEFAULT, PCMK_VALUE_TRUE)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_concurrent_fencing);
	}
	}

	pcmk_resource_t *
	pe_find_resource(GList rsc_list, const char id)
	{
	return pe_find_resource_with_flags(rsc_list, id, pcmk_rsc_match_history);
	}

	pcmk_resource_t *
	pe_find_resource_with_flags(GList rsc_list, const char id, enum pe_find flags)
	{
	GList *rIter = NULL;

	for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
	pcmk_resource_t *parent = rIter->data;
	pcmk_resource_t *match = parent->private->fns->find_rsc(parent, id,
	NULL, flags);

	if (match != NULL) {
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	/*!
	* \brief Find a node by name or ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id If not NULL, ID of node to find
	* \param[in] node_name If not NULL, name of node to find
	*
	* \return Node from \p nodes that matches \p id if any,
	* otherwise node from \p nodes that matches \p uname if any,
	* otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_any(const GList nodes, const char id, const char *uname)
	{
	pcmk_node_t *match = NULL;

	if (id != NULL) {
	match = pe_find_node_id(nodes, id);
	}
	if ((match == NULL) && (uname != NULL)) {
	match = pcmk__find_node_in_list(nodes, uname);
	}
	return match;
	}

	/*!
	* \brief Find a node by ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id ID of node to find
	*
	* \return Node from \p nodes that matches \p id if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_id(const GList nodes, const char id)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	/* @TODO Whether node IDs should be considered case-sensitive should
	* probably depend on the node type, so functionizing the comparison
	* would be worthwhile
	*/
	if (pcmk__str_eq(node->private->id, id, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/pengine/status_compat.h>

	/*!
	* \brief Find a node by name in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] node_name Name of node to find
	*
	* \return Node from \p nodes that matches \p node_name if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node(const GList nodes, const char node_name)
	{
	return pcmk__find_node_in_list(nodes, node_name);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index 4db814bf0b..aa1d2ee33f 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,5208 +1,5208 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <glib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	CRM_TRACE_INIT_DATA(pe_status);

	// A (parsed) resource action history entry
	struct action_history {
	pcmk_resource_t *rsc; // Resource that history is for
	pcmk_node_t *node; // Node that history is for
	xmlNode *xml; // History entry XML

	// Parsed from entry XML
	const char *id; // XML ID of history entry
	const char *key; // Operation key of action
	const char *task; // Action name
	const char *exit_reason; // Exit reason given for result
	guint interval_ms; // Action interval
	int call_id; // Call ID of action
	int expected_exit_status; // Expected exit status of action
	int exit_status; // Actual exit status of action
	int execution_status; // Execution status of action
	};

	/* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
	* use pcmk__set_scheduler_flags()/pcmk__clear_scheduler_flags() so that the
	* flag is stringified more readably in log messages.
	*/
	#define set_config_flag(scheduler, option, flag) do { \
	GHashTable *config_hash = (scheduler)->config_hash; \
	const char *scf_value = pcmk__cluster_option(config_hash, (option)); \
	\
	if (scf_value != NULL) { \
	if (crm_is_true(scf_value)) { \
	(scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} else { \
	(scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} \
	} \
	} while(0)

	static void unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node,
	xmlNode xml_op, xmlNode *last_failure,
	enum action_fail_response *failed);
	static void determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node);
	static void add_node_attrs(const xmlNode xml_obj, pcmk_node_t node,
	bool overwrite, pcmk_scheduler_t *scheduler);
	static void determine_online_status(const xmlNode *node_state,
	pcmk_node_t *this_node,
	pcmk_scheduler_t *scheduler);

	static void unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler);


	/*!
	* \internal
	* \brief Check whether a node is a dangling guest node
	*
	* \param[in] node Node to check
	*
	* \return true if \p node had a Pacemaker Remote connection resource with a
	* launcher that was removed from the CIB, otherwise false.
	*/
	static bool
	is_dangling_guest_node(pcmk_node_t *node)
	{
	return pcmk__is_pacemaker_remote_node(node)
	&& (node->private->remote != NULL)
	&& (node->private->remote->private->launcher == NULL)
	&& pcmk_is_set(node->private->remote->flags,
	pcmk__rsc_removed_launched);
	}

	/*!
	* \brief Schedule a fence action for a node
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in,out] node Node to fence
	* \param[in] reason Text description of why fencing is needed
	* \param[in] priority_delay Whether to consider
	* \c PCMK_OPT_PRIORITY_FENCING_DELAY
	*/
	void
	pe_fence_node(pcmk_scheduler_t scheduler, pcmk_node_t node,
	const char *reason, bool priority_delay)
	{
	CRM_CHECK(node, return);

	if (pcmk__is_guest_or_bundle_node(node)) {
	// Fence a guest or bundle node by marking its launcher as failed
	pcmk_resource_t *rsc = node->private->remote->private->launcher;

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	crm_notice("Not fencing guest node %s "
	"(otherwise would because %s): "
	"its guest resource %s is unmanaged",
	pcmk__node_name(node), reason, rsc->id);
	} else {
	pcmk__sched_warn("Guest node %s will be fenced "
	"(by recovering its guest resource %s): %s",
	pcmk__node_name(node), rsc->id, reason);

	/* We don't mark the node as unclean because that would prevent the
	* node from running resources. We want to allow it to run resources
	* in this transition if the recovery succeeds.
	*/
	pcmk__set_node_flags(node, pcmk__node_remote_reset);
	pcmk__set_rsc_flags(rsc,
	pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);
	}
	}

	} else if (is_dangling_guest_node(node)) {
	crm_info("Cleaning up dangling connection for guest node %s: "
	"fencing was already done because %s, "
	"and guest resource no longer exists",
	pcmk__node_name(node), reason);
	pcmk__set_rsc_flags(node->private->remote,
	pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);

	} else if (pcmk__is_remote_node(node)) {
	pcmk_resource_t *rsc = node->private->remote;

	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	crm_notice("Not fencing remote node %s "
	"(otherwise would because %s): connection is unmanaged",
	pcmk__node_name(node), reason);
	} else if (!pcmk_is_set(node->private->flags, pcmk__node_remote_reset)) {
	pcmk__set_node_flags(node, pcmk__node_remote_reset);
	pcmk__sched_warn("Remote node %s %s: %s",
	pcmk__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	}
	node->details->unclean = TRUE;
	// No need to apply PCMK_OPT_PRIORITY_FENCING_DELAY for remote nodes
	pe_fence_op(node, NULL, TRUE, reason, FALSE, scheduler);

	} else if (node->details->unclean) {
	crm_trace("Cluster node %s %s because %s",
	pcmk__node_name(node),
	pe_can_fence(scheduler, node)? "would also be fenced" : "also is unclean",
	reason);

	} else {
	pcmk__sched_warn("Cluster node %s %s: %s",
	pcmk__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	node->details->unclean = TRUE;
	pe_fence_op(node, NULL, TRUE, reason, priority_delay, scheduler);
	}
	}

	// @TODO xpaths can't handle templates, rules, or id-refs

	// nvpair with provides or requires set to unfencing
	#define XPATH_UNFENCING_NVPAIR PCMK_XE_NVPAIR \
	"[(@" PCMK_XA_NAME "='" PCMK_STONITH_PROVIDES "'" \
	"or @" PCMK_XA_NAME "='" PCMK_META_REQUIRES "') " \
	"and @" PCMK_XA_VALUE "='" PCMK_VALUE_UNFENCING "']"

	// unfencing in rsc_defaults or any resource
	#define XPATH_ENABLE_UNFENCING \
	"/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES \
	"//" PCMK_XE_META_ATTRIBUTES "/" XPATH_UNFENCING_NVPAIR \
	"\|/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RSC_DEFAULTS \
	"/" PCMK_XE_META_ATTRIBUTES "/" XPATH_UNFENCING_NVPAIR

	static void
	set_if_xpath(uint64_t flag, const char xpath, pcmk_scheduler_t scheduler)
	{
	xmlXPathObjectPtr result = NULL;

	if (!pcmk_is_set(scheduler->flags, flag)) {
	result = xpath_search(scheduler->input, xpath);
	if (result && (numXpathResults(result) > 0)) {
	pcmk__set_scheduler_flags(scheduler, flag);
	}
	freeXpathObject(result);
	}
	}

	gboolean
	unpack_config(xmlNode config, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	guint interval_ms = 0U;
	GHashTable *config_hash = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	scheduler->config_hash = config_hash;

	pe__unpack_dataset_nvpairs(config, PCMK_XE_CLUSTER_PROPERTY_SET, &rule_data,
	config_hash, PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS,
	FALSE, scheduler);

	pcmk__validate_cluster_options(config_hash);

	set_config_flag(scheduler, PCMK_OPT_ENABLE_STARTUP_PROBES,
	pcmk_sched_probe_resources);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pcmk__cluster_option(config_hash, PCMK_OPT_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_info("Watchdog-based self-fencing will be performed via SBD if "
	"fencing is required and " PCMK_OPT_STONITH_WATCHDOG_TIMEOUT
	" is nonzero");
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_fencing);
	}

	/* Set certain flags via xpath here, so they can be used before the relevant
	* configuration sections are unpacked.
	*/
	set_if_xpath(pcmk_sched_enable_unfencing, XPATH_ENABLE_UNFENCING,
	scheduler);

	value = pcmk__cluster_option(config_hash, PCMK_OPT_STONITH_TIMEOUT);
	pcmk_parse_interval_spec(value, &interval_ms);

	if (interval_ms >= INT_MAX) {
	scheduler->stonith_timeout = INT_MAX;
	} else {
	scheduler->stonith_timeout = (int) interval_ms;
	}
	crm_debug("STONITH timeout: %d", scheduler->stonith_timeout);

	set_config_flag(scheduler, PCMK_OPT_STONITH_ENABLED,
	pcmk_sched_fencing_enabled);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	crm_debug("STONITH of failed nodes is enabled");
	} else {
	crm_debug("STONITH of failed nodes is disabled");
	}

	scheduler->stonith_action = pcmk__cluster_option(config_hash,
	PCMK_OPT_STONITH_ACTION);
	if (!strcmp(scheduler->stonith_action, PCMK__ACTION_POWEROFF)) {
	pcmk__warn_once(pcmk__wo_poweroff,
	"Support for " PCMK_OPT_STONITH_ACTION " of "
	"'" PCMK__ACTION_POWEROFF "' is deprecated and will be "
	"removed in a future release "
	"(use '" PCMK_ACTION_OFF "' instead)");
	scheduler->stonith_action = PCMK_ACTION_OFF;
	}
	crm_trace("STONITH will %s nodes", scheduler->stonith_action);

	set_config_flag(scheduler, PCMK_OPT_CONCURRENT_FENCING,
	pcmk_sched_concurrent_fencing);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	crm_debug("Concurrent fencing is enabled");
	} else {
	crm_debug("Concurrent fencing is disabled");
	}

	value = pcmk__cluster_option(config_hash, PCMK_OPT_PRIORITY_FENCING_DELAY);
	if (value) {
	pcmk_parse_interval_spec(value, &interval_ms);
	scheduler->priority_fencing_delay = (int) (interval_ms / 1000);
	crm_trace("Priority fencing delay is %ds",
	scheduler->priority_fencing_delay);
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ALL_RESOURCES,
	pcmk_sched_stop_all);
	crm_debug("Stop all active resources: %s",
	pcmk__flag_text(scheduler->flags, pcmk_sched_stop_all));

	set_config_flag(scheduler, PCMK_OPT_SYMMETRIC_CLUSTER,
	pcmk_sched_symmetric_cluster);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pcmk__cluster_option(config_hash, PCMK_OPT_NO_QUORUM_POLICY);

	if (pcmk__str_eq(value, PCMK_VALUE_IGNORE, pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_ignore;

	} else if (pcmk__str_eq(value, PCMK_VALUE_FREEZE, pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_freeze;

	} else if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_demote;

	} else if (pcmk__str_eq(value, PCMK_VALUE_FENCE_LEGACY, pcmk__str_casei)) {
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	int do_panic = 0;

	crm_element_value_int(scheduler->input, PCMK_XA_NO_QUORUM_PANIC,
	&do_panic);
	if (do_panic \|\| pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_fence;
	} else {
	crm_notice("Resetting " PCMK_OPT_NO_QUORUM_POLICY
	" to 'stop': cluster has never had quorum");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}
	} else {
	pcmk__config_err("Resetting " PCMK_OPT_NO_QUORUM_POLICY
	" to 'stop' because fencing is disabled");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	} else {
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	crm_debug("On loss of quorum: Freeze resources");
	break;
	case pcmk_no_quorum_stop:
	crm_debug("On loss of quorum: Stop ALL resources");
	break;
	case pcmk_no_quorum_demote:
	crm_debug("On loss of quorum: "
	"Demote promotable resources and stop other resources");
	break;
	case pcmk_no_quorum_fence:
	crm_notice("On loss of quorum: Fence all remaining nodes");
	break;
	case pcmk_no_quorum_ignore:
	crm_notice("On loss of quorum: Ignore");
	break;
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_RESOURCES,
	pcmk_sched_stop_removed_resources);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	crm_trace("Orphan resources are stopped");
	} else {
	crm_trace("Orphan resources are ignored");
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_ACTIONS,
	pcmk_sched_cancel_removed_actions);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_cancel_removed_actions)) {
	crm_trace("Orphan resource actions are stopped");
	} else {
	crm_trace("Orphan resource actions are ignored");
	}

	value = pcmk__cluster_option(config_hash, PCMK__OPT_REMOVE_AFTER_STOP);
	if (value != NULL) {
	if (crm_is_true(value)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_remove_after_stop);
	pcmk__warn_once(pcmk__wo_remove_after,
	"Support for the " PCMK__OPT_REMOVE_AFTER_STOP
	" cluster property is deprecated and will be "
	"removed in a future release");
	} else {
	pcmk__clear_scheduler_flags(scheduler,
	pcmk_sched_remove_after_stop);
	}
	}

	set_config_flag(scheduler, PCMK_OPT_MAINTENANCE_MODE,
	pcmk_sched_in_maintenance);
	crm_trace("Maintenance mode: %s",
	pcmk__flag_text(scheduler->flags, pcmk_sched_in_maintenance));

	set_config_flag(scheduler, PCMK_OPT_START_FAILURE_IS_FATAL,
	pcmk_sched_start_failure_fatal);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
	crm_trace("Start failures are always fatal");
	} else {
	crm_trace("Start failures are handled by failcount");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	set_config_flag(scheduler, PCMK_OPT_STARTUP_FENCING,
	pcmk_sched_startup_fencing);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	crm_trace("Unseen nodes will be fenced");
	} else {
	pcmk__warn_once(pcmk__wo_blind,
	"Blind faith: not fencing unseen nodes");
	}

	pe__unpack_node_health_scores(scheduler);

	scheduler->placement_strategy =
	pcmk__cluster_option(config_hash, PCMK_OPT_PLACEMENT_STRATEGY);
	crm_trace("Placement strategy: %s", scheduler->placement_strategy);

	set_config_flag(scheduler, PCMK_OPT_SHUTDOWN_LOCK,
	pcmk_sched_shutdown_lock);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	value = pcmk__cluster_option(config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
	pcmk_parse_interval_spec(value, &(scheduler->shutdown_lock));
	scheduler->shutdown_lock /= 1000;
	crm_trace("Resources will be locked to nodes that were cleanly "
	"shut down (locks expire after %s)",
	pcmk__readable_interval(scheduler->shutdown_lock));
	} else {
	crm_trace("Resources will not be locked to nodes that were cleanly "
	"shut down");
	}

	value = pcmk__cluster_option(config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
	pcmk_parse_interval_spec(value, &(scheduler->node_pending_timeout));
	scheduler->node_pending_timeout /= 1000;
	if (scheduler->node_pending_timeout == 0) {
	crm_trace("Do not fence pending nodes");
	} else {
	crm_trace("Fence pending nodes after %s",
	pcmk__readable_interval(scheduler->node_pending_timeout
	* 1000));
	}

	return TRUE;
	}

	pcmk_node_t *
	pe_create_node(const char id, const char uname, const char *type,
	const char score, pcmk_scheduler_t scheduler)
	{
	pcmk_node_t *new_node = NULL;

	if (pcmk_find_node(scheduler, uname) != NULL) {
	pcmk__config_warn("More than one node entry has name '%s'", uname);
	}

	new_node = calloc(1, sizeof(pcmk_node_t));
	if (new_node == NULL) {
	pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	- new_node->weight = char2score(score);
	-
	+ new_node->assign = calloc(1, sizeof(struct pcmk__node_assignment));
	new_node->details = calloc(1, sizeof(struct pe_node_shared_s));
	new_node->private = calloc(1, sizeof(pcmk__node_private_t));
	- if ((new_node->details == NULL) \|\| (new_node->private == NULL)) {
	+ if ((new_node->assign == NULL) \|\| (new_node->details == NULL)
	+ \|\| (new_node->private == NULL)) {
	+ free(new_node->assign);
	free(new_node->details);
	free(new_node->private);
	free(new_node);
	pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	+ new_node->assign->score = char2score(score);
	new_node->private->id = id;
	new_node->private->name = uname;
	new_node->private->flags = pcmk__node_probes_allowed;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->running_rsc = NULL;
	new_node->private->scheduler = scheduler;

	if (pcmk__str_eq(type, PCMK_VALUE_MEMBER,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	new_node->private->variant = pcmk__node_variant_cluster;

	} else if (pcmk__str_eq(type, PCMK_VALUE_REMOTE, pcmk__str_casei)) {
	new_node->private->variant = pcmk__node_variant_remote;
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_remote_nodes);

	} else {
	/* @COMPAT 'ping' is the default for backward compatibility, but it
	* should be changed to 'member' at a compatibility break
	*/
	if (!pcmk__str_eq(type, PCMK__VALUE_PING, pcmk__str_casei)) {
	pcmk__config_warn("Node %s has unrecognized type '%s', "
	"assuming '" PCMK__VALUE_PING "'",
	pcmk__s(uname, "without name"), type);
	}
	pcmk__warn_once(pcmk__wo_ping_node,
	"Support for nodes of type '" PCMK__VALUE_PING "' "
	"(such as %s) is deprecated and will be removed in a "
	"future release",
	pcmk__s(uname, "unnamed node"));
	new_node->private->variant = pcmk__node_variant_ping;
	}

	new_node->private->attrs = pcmk__strkey_table(free, free);

	if (pcmk__is_pacemaker_remote_node(new_node)) {
	pcmk__insert_dup(new_node->private->attrs, CRM_ATTR_KIND, "remote");
	} else {
	pcmk__insert_dup(new_node->private->attrs, CRM_ATTR_KIND, "cluster");
	}

	new_node->private->utilization = pcmk__strkey_table(free, free);
	new_node->private->digest_cache = pcmk__strkey_table(free,
	pe__free_digests);

	scheduler->nodes = g_list_insert_sorted(scheduler->nodes, new_node,
	pe__cmp_node_name);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, pcmk_scheduler_t *data)
	{
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = pcmk__xe_id(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	const char *is_managed = NULL;

	for (attr_set = pcmk__xe_first_child(xml_obj, NULL, NULL, NULL);
	attr_set != NULL; attr_set = pcmk__xe_next(attr_set)) {

	if (!pcmk__xe_is(attr_set, PCMK_XE_META_ATTRIBUTES)) {
	continue;
	}

	for (attr = pcmk__xe_first_child(attr_set, NULL, NULL, NULL);
	attr != NULL; attr = pcmk__xe_next(attr)) {

	const char *value = crm_element_value(attr, PCMK_XA_VALUE);
	const char *name = crm_element_value(attr, PCMK_XA_NAME);

	if (name == NULL) { // Sanity
	continue;
	}

	if (strcmp(name, PCMK_META_REMOTE_NODE) == 0) {
	remote_name = value;

	} else if (strcmp(name, PCMK_META_REMOTE_ADDR) == 0) {
	remote_server = value;

	} else if (strcmp(name, PCMK_META_REMOTE_PORT) == 0) {
	remote_port = value;

	} else if (strcmp(name, PCMK_META_REMOTE_CONNECT_TIMEOUT) == 0) {
	connect_timeout = value;

	} else if (strcmp(name, PCMK_META_REMOTE_ALLOW_MIGRATE) == 0) {
	remote_allow_migrate = value;

	} else if (strcmp(name, PCMK_META_IS_MANAGED) == 0) {
	is_managed = value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (pe_find_resource(data->resources, remote_name) != NULL) {
	return NULL;
	}

	pe_create_remote_xml(parent, remote_name, container_id,
	remote_allow_migrate, is_managed,
	connect_timeout, remote_server, remote_port);
	return remote_name;
	}

	static void
	handle_startup_fencing(pcmk_scheduler_t scheduler, pcmk_node_t new_node)
	{
	if ((new_node->private->variant == pcmk__node_variant_remote)
	&& (new_node->private->remote == NULL)) {
	/* Ignore fencing for remote nodes that don't have a connection resource
	* associated with them. This happens when remote node entries get left
	* in the nodes section after the connection resource is removed.
	*/
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	// All nodes are unclean until we've seen their status entry
	new_node->details->unclean = TRUE;

	} else {
	// Blind faith ...
	new_node->details->unclean = FALSE;
	}
	}

	gboolean
	unpack_nodes(xmlNode xml_nodes, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	pcmk_node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_nodes, NULL, NULL, NULL);
	xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj)) {

	if (pcmk__xe_is(xml_obj, PCMK_XE_NODE)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, PCMK_XA_ID);
	uname = crm_element_value(xml_obj, PCMK_XA_UNAME);
	type = crm_element_value(xml_obj, PCMK_XA_TYPE);
	score = crm_element_value(xml_obj, PCMK_XA_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	pcmk__config_err("Ignoring <" PCMK_XE_NODE
	"> entry in configuration without id");
	continue;
	}
	new_node = pe_create_node(id, uname, type, score, scheduler);

	if (new_node == NULL) {
	return FALSE;
	}

	handle_startup_fencing(scheduler, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, scheduler);

	crm_trace("Done with node %s",
	crm_element_value(xml_obj, PCMK_XA_UNAME));
	}
	}

	if (scheduler->localhost
	&& (pcmk_find_node(scheduler, scheduler->localhost) == NULL)) {
	crm_info("Creating a fake local node");
	pe_create_node(scheduler->localhost, scheduler->localhost, NULL, 0,
	scheduler);
	}

	return TRUE;
	}

	static void
	unpack_launcher(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *launcher_id = NULL;

	if (rsc->private->children != NULL) {
	g_list_foreach(rsc->private->children, (GFunc) unpack_launcher,
	scheduler);
	return;
	}

	launcher_id = g_hash_table_lookup(rsc->private->meta, PCMK__META_CONTAINER);
	if ((launcher_id != NULL)
	&& !pcmk__str_eq(launcher_id, rsc->id, pcmk__str_none)) {
	pcmk_resource_t *launcher = pe_find_resource(scheduler->resources,
	launcher_id);

	if (launcher != NULL) {
	rsc->private->launcher = launcher;
	launcher->private->launched =
	g_list_append(launcher->private->launched, rsc);
	pcmk__rsc_trace(rsc, "Resource %s's launcher is %s",
	rsc->id, launcher_id);
	} else {
	pcmk__config_err("Resource %s: Unknown " PCMK__META_CONTAINER " %s",
	rsc->id, launcher_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	/* Create remote nodes and guest nodes from the resource configuration
	* before unpacking resources.
	*/
	for (xml_obj = pcmk__xe_first_child(xml_resources, NULL, NULL, NULL);
	xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj)) {

	const char *new_node_id = NULL;

	/* Check for remote nodes, which are defined by ocf:pacemaker:remote
	* primitives.
	*/
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = pcmk__xe_id(xml_obj);
	/* The pcmk_find_node() check ensures we don't iterate over an
	* expanded node that has already been added to the node list
	*/
	if (new_node_id
	&& (pcmk_find_node(scheduler, new_node_id) == NULL)) {
	crm_trace("Found remote node %s defined by resource %s",
	new_node_id, pcmk__xe_id(xml_obj));
	pe_create_node(new_node_id, new_node_id, PCMK_VALUE_REMOTE,
	NULL, scheduler);
	}
	continue;
	}

	/* Check for guest nodes, which are defined by special meta-attributes
	* of a primitive of any type (for example, VirtualDomain or Xen).
	*/
	if (pcmk__xe_is(xml_obj, PCMK_XE_PRIMITIVE)) {
	/* This will add an ocf:pacemaker:remote primitive to the
	* configuration for the guest node's connection, to be unpacked
	* later.
	*/
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources,
	scheduler);
	if (new_node_id
	&& (pcmk_find_node(scheduler, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s",
	new_node_id, pcmk__xe_id(xml_obj));
	pe_create_node(new_node_id, new_node_id, PCMK_VALUE_REMOTE,
	NULL, scheduler);
	}
	continue;
	}

	/* Check for guest nodes inside a group. Clones are currently not
	* supported as guest nodes.
	*/
	if (pcmk__xe_is(xml_obj, PCMK_XE_GROUP)) {
	xmlNode *xml_obj2 = NULL;
	for (xml_obj2 = pcmk__xe_first_child(xml_obj, NULL, NULL, NULL);
	xml_obj2 != NULL; xml_obj2 = pcmk__xe_next(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources,
	scheduler);

	if (new_node_id
	&& (pcmk_find_node(scheduler, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s inside group %s",
	new_node_id, pcmk__xe_id(xml_obj2),
	pcmk__xe_id(xml_obj));
	pe_create_node(new_node_id, new_node_id, PCMK_VALUE_REMOTE,
	NULL, scheduler);
	}
	}
	}
	}
	return TRUE;
	}

	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the scheduler calculations.
	*/
	static void
	link_rsc2remotenode(pcmk_scheduler_t scheduler, pcmk_resource_t new_rsc)
	{
	pcmk_node_t *remote_node = NULL;

	if (!pcmk_is_set(new_rsc->flags, pcmk__rsc_is_remote_connection)) {
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	remote_node = pcmk_find_node(scheduler, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return);

	pcmk__rsc_trace(new_rsc, "Linking remote connection resource %s to %s",
	new_rsc->id, pcmk__node_name(remote_node));
	remote_node->private->remote = new_rsc;

	if (new_rsc->private->launcher == NULL) {
	/* Handle start-up fencing for remote nodes (as opposed to guest nodes)
	* the same as is done for cluster nodes.
	*/
	handle_startup_fencing(scheduler, remote_node);

	} else {
	/* pe_create_node() marks the new node as "remote" or "cluster"; now
	* that we know the node is a guest node, update it correctly.
	*/
	pcmk__insert_dup(remote_node->private->attrs,
	CRM_ATTR_KIND, "container");
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	pcmk_tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	/*!
	* \internal
	* \brief Parse configuration XML for resource information
	*
	* \param[in] xml_resources Top of resource configuration XML
	* \param[in,out] scheduler Scheduler data
	*
	* \return TRUE
	*
	* \note unpack_remote_nodes() MUST be called before this, so that the nodes can
	* be used when pe__unpack_resource() calls resource_location()
	*/
	gboolean
	unpack_resources(const xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	GList *gIter = NULL;

	scheduler->template_rsc_sets = pcmk__strkey_table(free, destroy_tag);

	for (xml_obj = pcmk__xe_first_child(xml_resources, NULL, NULL, NULL);
	xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj)) {

	pcmk_resource_t *new_rsc = NULL;
	const char *id = pcmk__xe_id(xml_obj);

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring <%s> resource without ID",
	xml_obj->name);
	continue;
	}

	if (pcmk__xe_is(xml_obj, PCMK_XE_TEMPLATE)) {
	if (g_hash_table_lookup_extended(scheduler->template_rsc_sets, id,
	NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	pcmk__insert_dup(scheduler->template_rsc_sets, id, NULL);
	}
	continue;
	}

	crm_trace("Unpacking <%s " PCMK_XA_ID "='%s'>", xml_obj->name, id);
	if (pe__unpack_resource(xml_obj, &new_rsc, NULL,
	scheduler) == pcmk_rc_ok) {
	scheduler->resources = g_list_append(scheduler->resources, new_rsc);
	pcmk__rsc_trace(new_rsc, "Added resource %s", new_rsc->id);

	} else {
	pcmk__config_err("Ignoring <%s> resource '%s' "
	"because configuration is invalid",
	xml_obj->name, id);
	}
	}

	for (gIter = scheduler->resources; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	unpack_launcher(rsc, scheduler);
	link_rsc2remotenode(scheduler, rsc);
	}

	scheduler->resources = g_list_sort(scheduler->resources,
	pe__cmp_rsc_priority);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* Ignore */

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {

	pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
	pcmk__config_err("Either configure some or disable STONITH with the "
	PCMK_OPT_STONITH_ENABLED " option");
	pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode xml_tags, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_tag = NULL;

	scheduler->tags = pcmk__strkey_table(free, destroy_tag);

	for (xml_tag = pcmk__xe_first_child(xml_tags, NULL, NULL, NULL);
	xml_tag != NULL; xml_tag = pcmk__xe_next(xml_tag)) {

	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = pcmk__xe_id(xml_tag);

	if (!pcmk__xe_is(xml_tag, PCMK_XE_TAG)) {
	continue;
	}

	if (tag_id == NULL) {
	pcmk__config_err("Ignoring <%s> without " PCMK_XA_ID,
	(const char *) xml_tag->name);
	continue;
	}

	for (xml_obj_ref = pcmk__xe_first_child(xml_tag, NULL, NULL, NULL);
	xml_obj_ref != NULL; xml_obj_ref = pcmk__xe_next(xml_obj_ref)) {

	const char *obj_ref = pcmk__xe_id(xml_obj_ref);

	if (!pcmk__xe_is(xml_obj_ref, PCMK_XE_OBJ_REF)) {
	continue;
	}

	if (obj_ref == NULL) {
	pcmk__config_err("Ignoring <%s> for tag '%s' without " PCMK_XA_ID,
	xml_obj_ref->name, tag_id);
	continue;
	}

	if (add_tag_ref(scheduler->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode xml_ticket, pcmk_scheduler_t scheduler)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	pcmk_ticket_t *ticket = NULL;

	ticket_id = pcmk__xe_id(xml_ticket);
	if (pcmk__str_empty(ticket_id)) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, scheduler);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = pcmk__xml_attr_value(xIter);

	if (pcmk__str_eq(prop_name, PCMK_XA_ID, pcmk__str_none)) {
	continue;
	}
	pcmk__insert_dup(ticket->state, prop_name, prop_value);
	}

	granted = g_hash_table_lookup(ticket->state, PCMK__XA_GRANTED);
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, PCMK_XA_LAST_GRANTED);
	if (last_granted) {
	long long last_granted_ll;

	pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
	ticket->last_granted = (time_t) last_granted_ll;
	}

	standby = g_hash_table_lookup(ticket->state, PCMK_XA_STANDBY);
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode xml_tickets, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_tickets, NULL, NULL, NULL);
	xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj)) {

	if (!pcmk__xe_is(xml_obj, PCMK__XE_TICKET_STATE)) {
	continue;
	}
	unpack_ticket_state(xml_obj, scheduler);
	}

	return TRUE;
	}

	static void
	unpack_handle_remote_attrs(pcmk_node_t this_node, const xmlNode state,
	pcmk_scheduler_t *scheduler)
	{
	const char *discovery = NULL;
	const xmlNode *attrs = NULL;
	pcmk_resource_t *rsc = NULL;
	int maint = 0;

	if (!pcmk__xe_is(state, PCMK__XE_NODE_STATE)) {
	return;
	}

	if ((this_node == NULL) \|\| !pcmk__is_pacemaker_remote_node(this_node)) {
	return;
	}
	crm_trace("Processing Pacemaker Remote node %s",
	pcmk__node_name(this_node));

	pcmk__scan_min_int(crm_element_value(state, PCMK__XA_NODE_IN_MAINTENANCE),
	&maint, 0);
	if (maint) {
	pcmk__set_node_flags(this_node, pcmk__node_remote_maint);
	} else {
	pcmk__clear_node_flags(this_node, pcmk__node_remote_maint);
	}

	rsc = this_node->private->remote;
	if (!pcmk_is_set(this_node->private->flags, pcmk__node_remote_reset)) {
	this_node->details->unclean = FALSE;
	pcmk__set_node_flags(this_node, pcmk__node_seen);
	}
	attrs = pcmk__xe_first_child(state, PCMK__XE_TRANSIENT_ATTRIBUTES, NULL,
	NULL);
	add_node_attrs(attrs, this_node, TRUE, scheduler);

	if (pe__shutdown_requested(this_node)) {
	crm_info("%s is shutting down", pcmk__node_name(this_node));
	this_node->details->shutdown = TRUE;
	}

	if (crm_is_true(pcmk__node_attr(this_node, PCMK_NODE_ATTR_STANDBY, NULL,
	pcmk__rsc_node_current))) {
	crm_info("%s is in standby mode", pcmk__node_name(this_node));
	pcmk__set_node_flags(this_node, pcmk__node_standby);
	}

	if (crm_is_true(pcmk__node_attr(this_node, PCMK_NODE_ATTR_MAINTENANCE, NULL,
	pcmk__rsc_node_current))
	\|\| ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk__rsc_managed))) {
	crm_info("%s is in maintenance mode", pcmk__node_name(this_node));
	this_node->details->maintenance = TRUE;
	}

	discovery = pcmk__node_attr(this_node,
	PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED,
	NULL, pcmk__rsc_node_current);
	if ((discovery != NULL) && !crm_is_true(discovery)) {
	pcmk__warn_once(pcmk__wo_rdisc_enabled,
	"Support for the "
	PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED
	" node attribute is deprecated and will be removed"
	" (and behave as 'true') in a future release.");

	if (pcmk__is_remote_node(this_node)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Ignoring "
	PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED
	" attribute on Pacemaker Remote node %s"
	" because fencing is disabled",
	pcmk__node_name(this_node));
	} else {
	/* This is either a remote node with fencing enabled, or a guest
	* node. We don't care whether fencing is enabled when fencing guest
	* nodes, because they are "fenced" by recovering their containing
	* resource.
	*/
	crm_info("%s has resource discovery disabled",
	pcmk__node_name(this_node));
	pcmk__clear_node_flags(this_node, pcmk__node_probes_allowed);
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack a cluster node's transient attributes
	*
	* \param[in] state CIB node state XML
	* \param[in,out] node Cluster node whose attributes are being unpacked
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_transient_attributes(const xmlNode state, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	const char *discovery = NULL;
	const xmlNode *attrs = pcmk__xe_first_child(state,
	PCMK__XE_TRANSIENT_ATTRIBUTES,
	NULL, NULL);

	add_node_attrs(attrs, node, TRUE, scheduler);

	if (crm_is_true(pcmk__node_attr(node, PCMK_NODE_ATTR_STANDBY, NULL,
	pcmk__rsc_node_current))) {
	crm_info("%s is in standby mode", pcmk__node_name(node));
	pcmk__set_node_flags(node, pcmk__node_standby);
	}

	if (crm_is_true(pcmk__node_attr(node, PCMK_NODE_ATTR_MAINTENANCE, NULL,
	pcmk__rsc_node_current))) {
	crm_info("%s is in maintenance mode", pcmk__node_name(node));
	node->details->maintenance = TRUE;
	}

	discovery = pcmk__node_attr(node,
	PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED,
	NULL, pcmk__rsc_node_current);
	if ((discovery != NULL) && !crm_is_true(discovery)) {
	pcmk__config_warn("Ignoring "
	PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED
	" attribute for %s because disabling resource"
	" discovery is not allowed for cluster nodes",
	pcmk__node_name(node));
	}
	}

	/*!
	* \internal
	* \brief Unpack a node state entry (first pass)
	*
	* Unpack one node state entry from status. This unpacks information from the
	* \C PCMK__XE_NODE_STATE element itself and node attributes inside it, but not
	* the resource history inside it. Multiple passes through the status are needed
	* to fully unpack everything.
	*
	* \param[in] state CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_state(const xmlNode state, pcmk_scheduler_t scheduler)
	{
	const char *id = NULL;
	const char *uname = NULL;
	pcmk_node_t *this_node = NULL;

	id = crm_element_value(state, PCMK_XA_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring invalid " PCMK__XE_NODE_STATE " entry without "
	PCMK_XA_ID);
	crm_log_xml_info(state, "missing-id");
	return;
	}

	uname = crm_element_value(state, PCMK_XA_UNAME);
	if (uname == NULL) {
	/* If a joining peer makes the cluster acquire the quorum from corosync
	* meanwhile it has not joined CPG membership of pacemaker-controld yet,
	* it's possible that the created PCMK__XE_NODE_STATE entry doesn't have
	* a PCMK_XA_UNAME yet. We should recognize the node as `pending` and
	* wait for it to join CPG.
	*/
	crm_trace("Handling " PCMK__XE_NODE_STATE " entry with id=\"%s\" "
	"without " PCMK_XA_UNAME,
	id);
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	crm_notice("Ignoring recorded state for removed node with name %s and "
	PCMK_XA_ID " %s", pcmk__s(uname, "unknown"), id);
	return;
	}

	if (pcmk__is_pacemaker_remote_node(this_node)) {
	int remote_fenced = 0;

	/* We can't determine the online status of Pacemaker Remote nodes until
	* after all resource history has been unpacked. In this first pass, we
	* do need to mark whether the node has been fenced, as this plays a
	* role during unpacking cluster node resource state.
	*/
	pcmk__scan_min_int(crm_element_value(state, PCMK__XA_NODE_FENCED),
	&remote_fenced, 0);
	if (remote_fenced) {
	pcmk__set_node_flags(this_node, pcmk__node_remote_fenced);
	} else {
	pcmk__clear_node_flags(this_node, pcmk__node_remote_fenced);
	}
	return;
	}

	unpack_transient_attributes(state, this_node, scheduler);

	/* Provisionally mark this cluster node as clean. We have at least seen it
	* in the current cluster's lifetime.
	*/
	this_node->details->unclean = FALSE;
	pcmk__set_node_flags(this_node, pcmk__node_seen);

	crm_trace("Determining online status of cluster node %s (id %s)",
	pcmk__node_name(this_node), id);
	determine_online_status(state, this_node, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	&& this_node->details->online
	&& (scheduler->no_quorum_policy == pcmk_no_quorum_fence)) {
	/* Everything else should flow from this automatically
	* (at least until the scheduler becomes able to migrate off
	* healthy resources)
	*/
	pe_fence_node(scheduler, this_node, "cluster does not have quorum",
	FALSE);
	}
	}

	/*!
	* \internal
	* \brief Unpack nodes' resource history as much as possible
	*
	* Unpack as many nodes' resource history as possible in one pass through the
	* status. We need to process Pacemaker Remote nodes' connections/containers
	* before unpacking their history; the connection/container history will be
	* in another node's history, so it might take multiple passes to unpack
	* everything.
	*
	* \param[in] status CIB XML status section
	* \param[in] fence If true, treat any not-yet-unpacked nodes as unseen
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
	* or EAGAIN if more unpacking remains to be done)
	*/
	static int
	unpack_node_history(const xmlNode *status, bool fence,
	pcmk_scheduler_t *scheduler)
	{
	int rc = pcmk_rc_ok;

	// Loop through all PCMK__XE_NODE_STATE entries in CIB status
	for (const xmlNode *state = pcmk__xe_first_child(status,
	PCMK__XE_NODE_STATE, NULL,
	NULL);
	state != NULL; state = pcmk__xe_next_same(state)) {

	const char *id = pcmk__xe_id(state);
	const char *uname = crm_element_value(state, PCMK_XA_UNAME);
	pcmk_node_t *this_node = NULL;

	if ((id == NULL) \|\| (uname == NULL)) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history from malformed "
	PCMK__XE_NODE_STATE " without id and/or uname");
	continue;
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history for node %s because "
	"no longer in configuration", id);
	continue;
	}

	if (pcmk_is_set(this_node->private->flags, pcmk__node_unpacked)) {
	crm_trace("Not unpacking resource history for node %s because "
	"already unpacked", id);
	continue;
	}

	if (fence) {
	// We're processing all remaining nodes

	} else if (pcmk__is_guest_or_bundle_node(this_node)) {
	/* We can unpack a guest node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection and containing resource are both up.
	*/
	const pcmk_resource_t *remote = this_node->private->remote;
	const pcmk_resource_t *launcher = remote->private->launcher;

	if ((remote->private->orig_role != pcmk_role_started)
	\|\| (launcher->private->orig_role != pcmk_role_started)) {
	crm_trace("Not unpacking resource history for guest node %s "
	"because launcher and connection are not known to "
	"be up", id);
	continue;
	}

	} else if (pcmk__is_remote_node(this_node)) {
	/* We can unpack a remote node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection is up, with the exception of when shutdown locks are
	* in use.
	*/
	pcmk_resource_t *rsc = this_node->private->remote;

	if ((rsc == NULL)
	\|\| (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)
	&& (rsc->private->orig_role != pcmk_role_started))) {
	crm_trace("Not unpacking resource history for remote node %s "
	"because connection is not known to be up", id);
	continue;
	}

	/* If fencing and shutdown locks are disabled and we're not processing
	* unseen nodes, then we don't want to unpack offline nodes until online
	* nodes have been unpacked. This allows us to number active clone
	* instances first.
	*/
	} else if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_fencing_enabled
	\|pcmk_sched_shutdown_lock)
	&& !this_node->details->online) {
	crm_trace("Not unpacking resource history for offline "
	"cluster node %s", id);
	continue;
	}

	if (pcmk__is_pacemaker_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);
	unpack_handle_remote_attrs(this_node, state, scheduler);
	}

	crm_trace("Unpacking resource history for %snode %s",
	(fence? "unseen " : ""), id);

	pcmk__set_node_flags(this_node, pcmk__node_unpacked);
	unpack_node_lrm(this_node, state, scheduler);

	rc = EAGAIN; // Other node histories might depend on this one
	}
	return rc;
	}

	/* remove nodes that are down, stopping */
	/* create positive rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode status, pcmk_scheduler_t scheduler)
	{
	xmlNode *state = NULL;

	crm_trace("Beginning unpack");

	if (scheduler->tickets == NULL) {
	scheduler->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	for (state = pcmk__xe_first_child(status, NULL, NULL, NULL); state != NULL;
	state = pcmk__xe_next(state)) {

	if (pcmk__xe_is(state, PCMK_XE_TICKETS)) {
	unpack_tickets_state((xmlNode *) state, scheduler);

	} else if (pcmk__xe_is(state, PCMK__XE_NODE_STATE)) {
	unpack_node_state(state, scheduler);
	}
	}

	while (unpack_node_history(status, FALSE, scheduler) == EAGAIN) {
	crm_trace("Another pass through node resource histories is needed");
	}

	// Now catch any nodes we didn't see
	unpack_node_history(status,
	pcmk_is_set(scheduler->flags,
	pcmk_sched_fencing_enabled),
	scheduler);

	/* Now that we know where resources are, we can schedule stops of containers
	* with failed bundle connections
	*/
	if (scheduler->stop_needed != NULL) {
	for (GList *item = scheduler->stop_needed; item; item = item->next) {
	pcmk_resource_t *container = item->data;
	pcmk_node_t *node = pcmk__current_node(container);

	if (node) {
	stop_action(container, node, FALSE);
	}
	}
	g_list_free(scheduler->stop_needed);
	scheduler->stop_needed = NULL;
	}

	/* Now that we know status of all Pacemaker Remote connections and nodes,
	* we can stop connections for node shutdowns, and check the online status
	* of remote/guest nodes that didn't have any node history to unpack.
	*/
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *this_node = gIter->data;

	if (!pcmk__is_pacemaker_remote_node(this_node)) {
	continue;
	}
	if (this_node->details->shutdown
	&& (this_node->private->remote != NULL)) {
	pe__set_next_role(this_node->private->remote, pcmk_role_stopped,
	"remote shutdown");
	}
	if (!pcmk_is_set(this_node->private->flags, pcmk__node_unpacked)) {
	determine_remote_online_status(scheduler, this_node);
	}
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became a member at the cluster layer
	*
	* \param[in] node_state Node's \c PCMK__XE_NODE_STATE entry
	* \param[in,out] scheduler Scheduler data
	*
	* \return Epoch time when node became a cluster member
	* (or scheduler effective time for legacy entries) if a member,
	* 0 if not a member, or -1 if no valid information available
	*/
	static long long
	unpack_node_member(const xmlNode node_state, pcmk_scheduler_t scheduler)
	{
	const char *member_time = crm_element_value(node_state, PCMK__XA_IN_CCM);
	int member = 0;

	if (member_time == NULL) {
	return -1LL;

	} else if (crm_str_to_boolean(member_time, &member) == 1) {
	/* If in_ccm=0, we'll return 0 here. If in_ccm=1, either the entry was
	* recorded as a boolean for a DC < 2.1.7, or the node is pending
	* shutdown and has left the CPG, in which case it was set to 1 to avoid
	* fencing for PCMK_OPT_NODE_PENDING_TIMEOUT.
	*
	* We return the effective time for in_ccm=1 because what's important to
	* avoid fencing is that effective time minus this value is less than
	* the pending node timeout.
	*/
	return member? (long long) get_effective_time(scheduler) : 0LL;

	} else {
	long long when_member = 0LL;

	if ((pcmk__scan_ll(member_time, &when_member,
	0LL) != pcmk_rc_ok) \|\| (when_member < 0LL)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_IN_CCM
	" in " PCMK__XE_NODE_STATE " entry", member_time);
	return -1LL;
	}
	return when_member;
	}
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became online in process group
	*
	* \param[in] node_state Node's \c PCMK__XE_NODE_STATE entry
	*
	* \return Epoch time when node became online in process group (or 0 if not
	* online, or 1 for legacy online entries)
	*/
	static long long
	unpack_node_online(const xmlNode *node_state)
	{
	const char *peer_time = crm_element_value(node_state, PCMK_XA_CRMD);

	// @COMPAT Entries recorded for DCs < 2.1.7 have "online" or "offline"
	if (pcmk__str_eq(peer_time, PCMK_VALUE_OFFLINE,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	return 0LL;

	} else if (pcmk__str_eq(peer_time, PCMK_VALUE_ONLINE, pcmk__str_casei)) {
	return 1LL;

	} else {
	long long when_online = 0LL;

	if ((pcmk__scan_ll(peer_time, &when_online, 0LL) != pcmk_rc_ok)
	\|\| (when_online < 0)) {
	crm_warn("Unrecognized value '%s' for " PCMK_XA_CRMD " in "
	PCMK__XE_NODE_STATE " entry, assuming offline", peer_time);
	return 0LL;
	}
	return when_online;
	}
	}

	/*!
	* \internal
	* \brief Unpack node attribute for user-requested fencing
	*
	* \param[in] node Node to check
	* \param[in] node_state Node's \c PCMK__XE_NODE_STATE entry in CIB status
	*
	* \return \c true if fencing has been requested for \p node, otherwise \c false
	*/
	static bool
	unpack_node_terminate(const pcmk_node_t node, const xmlNode node_state)
	{
	long long value = 0LL;
	int value_i = 0;
	const char *value_s = pcmk__node_attr(node, PCMK_NODE_ATTR_TERMINATE,
	NULL, pcmk__rsc_node_current);

	// Value may be boolean or an epoch time
	if (crm_str_to_boolean(value_s, &value_i) == 1) {
	return (value_i != 0);
	}
	if (pcmk__scan_ll(value_s, &value, 0LL) == pcmk_rc_ok) {
	return (value > 0);
	}
	crm_warn("Ignoring unrecognized value '%s' for " PCMK_NODE_ATTR_TERMINATE
	"node attribute for %s", value_s, pcmk__node_name(node));
	return false;
	}

	static gboolean
	determine_online_status_no_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK_XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	if (when_member <= 0) {
	crm_trace("Node %s is %sdown", pcmk__node_name(this_node),
	((when_member < 0)? "presumed " : ""));

	} else if (when_online > 0) {
	if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	online = TRUE;
	} else {
	crm_debug("Node %s is not ready to run resources: %s",
	pcmk__node_name(this_node), join);
	}

	} else if (!pcmk_is_set(this_node->private->flags,
	pcmk__node_expected_up)) {
	crm_trace("Node %s controller is down: "
	"member@%lld online@%lld join=%s expected=%s",
	pcmk__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));

	} else {
	/* mark it unclean */
	pe_fence_node(scheduler, this_node, "peer is unexpectedly down", FALSE);
	crm_info("Node %s member@%lld online@%lld join=%s expected=%s",
	pcmk__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
	}
	return online;
	}

	/*!
	* \internal
	* \brief Check whether a node has taken too long to join controller group
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node to check
	* \param[in] when_member Epoch time when node became a cluster member
	* \param[in] when_online Epoch time when node joined controller group
	*
	* \return true if node has been pending (on the way up) longer than
	* \c PCMK_OPT_NODE_PENDING_TIMEOUT, otherwise false
	* \note This will also update the cluster's recheck time if appropriate.
	*/
	static inline bool
	pending_too_long(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	long long when_member, long long when_online)
	{
	if ((scheduler->node_pending_timeout > 0)
	&& (when_member > 0) && (when_online <= 0)) {
	// There is a timeout on pending nodes, and node is pending

	time_t timeout = when_member + scheduler->node_pending_timeout;

	if (get_effective_time(node->private->scheduler) >= timeout) {
	return true; // Node has timed out
	}

	// Node is pending, but still has time
	pe__update_recheck_time(timeout, scheduler, "pending node timeout");
	}
	return false;
	}

	static bool
	determine_online_status_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	bool termination_requested = unpack_node_terminate(this_node, node_state);
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK_XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	/*
	- PCMK__XA_JOIN ::= member\|down\|pending\|banned
	- PCMK_XA_EXPECTED ::= member\|down

	@COMPAT with entries recorded for DCs < 2.1.7
	- PCMK__XA_IN_CCM ::= true\|false
	- PCMK_XA_CRMD ::= online\|offline

	Since crm_feature_set 3.18.0 (pacemaker-2.1.7):
	- PCMK__XA_IN_CCM ::= <timestamp>\|0
	Since when node has been a cluster member. A value 0 of means the node is not
	a cluster member.

	- PCMK_XA_CRMD ::= <timestamp>\|0
	Since when peer has been online in CPG. A value 0 means the peer is offline
	in CPG.
	*/

	crm_trace("Node %s member@%lld online@%lld join=%s expected=%s%s",
	pcmk__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"),
	(termination_requested? " (termination requested)" : ""));

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", pcmk__node_name(this_node));

	/* Slightly different criteria since we can't shut down a dead peer */
	return (when_online > 0);
	}

	if (when_member < 0) {
	pe_fence_node(scheduler, this_node,
	"peer has not been seen by the cluster", FALSE);
	return false;
	}

	if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_none)) {
	pe_fence_node(scheduler, this_node,
	"peer failed Pacemaker membership criteria", FALSE);

	} else if (termination_requested) {
	if ((when_member <= 0) && (when_online <= 0)
	&& pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_none)) {
	crm_info("%s was fenced as requested", pcmk__node_name(this_node));
	return false;
	}
	pe_fence_node(scheduler, this_node, "fencing was requested", false);

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN,
	pcmk__str_null_matches)) {

	if (pending_too_long(scheduler, this_node, when_member, when_online)) {
	pe_fence_node(scheduler, this_node,
	"peer pending timed out on joining the process group",
	FALSE);

	} else if ((when_member > 0) \|\| (when_online > 0)) {
	crm_info("- %s is not ready to run resources",
	pcmk__node_name(this_node));
	pcmk__set_node_flags(this_node, pcmk__node_standby);
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up",
	pcmk__node_name(this_node));
	}

	} else if (when_member <= 0) {
	// Consider PCMK_OPT_PRIORITY_FENCING_DELAY for lost nodes
	pe_fence_node(scheduler, this_node,
	"peer is no longer part of the cluster", TRUE);

	} else if (when_online <= 0) {
	pe_fence_node(scheduler, this_node,
	"peer process is no longer available", FALSE);

	/* Everything is running at this point, now check join state */

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_none)) {
	crm_info("%s is active", pcmk__node_name(this_node));

	} else if (pcmk__str_any_of(join, CRMD_JOINSTATE_PENDING,
	CRMD_JOINSTATE_DOWN, NULL)) {
	crm_info("%s is not ready to run resources",
	pcmk__node_name(this_node));
	pcmk__set_node_flags(this_node, pcmk__node_standby);
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(scheduler, this_node, "peer was in an unknown state",
	FALSE);
	}

	return (when_member > 0);
	}

	static void
	determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node)
	{
	pcmk_resource_t *rsc = this_node->private->remote;
	pcmk_resource_t *launcher = NULL;
	pcmk_node_t *host = NULL;
	const char *node_type = "Remote";

	if (rsc == NULL) {
	/* This is a leftover node state entry for a former Pacemaker Remote
	* node whose connection resource was removed. Consider it offline.
	*/
	crm_trace("Pacemaker Remote node %s is considered OFFLINE because "
	"its connection resource has been removed from the CIB",
	this_node->private->id);
	this_node->details->online = FALSE;
	return;
	}

	launcher = rsc->private->launcher;
	if (launcher != NULL) {
	node_type = "Guest";
	if (pcmk__list_of_1(rsc->private->active_nodes)) {
	host = rsc->private->active_nodes->data;
	}
	}

	/* If the resource is currently started, mark it online. */
	if (rsc->private->orig_role == pcmk_role_started) {
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if ((rsc->private->orig_role == pcmk_role_started)
	&& (rsc->private->next_role == pcmk_role_stopped)) {

	crm_trace("%s node %s shutting down because connection resource is stopping",
	node_type, this_node->private->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if ((launcher != NULL) && pcmk_is_set(launcher->flags, pcmk__rsc_failed)) {
	crm_trace("Guest node %s UNCLEAN because guest resource failed",
	this_node->private->id);
	this_node->details->online = FALSE;
	pcmk__set_node_flags(this_node, pcmk__node_remote_reset);

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
	crm_trace("%s node %s OFFLINE because connection resource failed",
	node_type, this_node->private->id);
	this_node->details->online = FALSE;

	} else if ((rsc->private->orig_role == pcmk_role_stopped)
	\|\| ((launcher != NULL)
	&& (launcher->private->orig_role == pcmk_role_stopped))) {

	crm_trace("%s node %s OFFLINE because its resource is stopped",
	node_type, this_node->private->id);
	this_node->details->online = FALSE;
	pcmk__clear_node_flags(this_node, pcmk__node_remote_reset);

	} else if (host && (host->details->online == FALSE)
	&& host->details->unclean) {
	crm_trace("Guest node %s UNCLEAN because host is unclean",
	this_node->private->id);
	this_node->details->online = FALSE;
	pcmk__set_node_flags(this_node, pcmk__node_remote_reset);

	} else {
	crm_trace("%s node %s is %s",
	node_type, this_node->private->id,
	this_node->details->online? "ONLINE" : "OFFLINE");
	}
	}

	static void
	determine_online_status(const xmlNode node_state, pcmk_node_t this_node,
	pcmk_scheduler_t *scheduler)
	{
	gboolean online = FALSE;
	const char *exp_state = crm_element_value(node_state, PCMK_XA_EXPECTED);

	CRM_CHECK(this_node != NULL, return);

	this_node->details->shutdown = FALSE;

	if (pe__shutdown_requested(this_node)) {
	this_node->details->shutdown = TRUE;

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	pcmk__set_node_flags(this_node, pcmk__node_expected_up);
	}

	if (this_node->private->variant == pcmk__node_variant_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	online = determine_online_status_no_fencing(scheduler, node_state,
	this_node);

	} else {
	online = determine_online_status_fencing(scheduler, node_state,
	this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	- this_node->fixed = TRUE; // @COMPAT deprecated and unused
	- this_node->weight = -PCMK_SCORE_INFINITY;
	+ this_node->assign->score = -PCMK_SCORE_INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* don't run resources here */
	- this_node->fixed = TRUE; // @COMPAT deprecated and unused
	- this_node->weight = -PCMK_SCORE_INFINITY;
	+ this_node->assign->score = -PCMK_SCORE_INFINITY;
	}

	if (this_node->private->variant == pcmk__node_variant_ping) {
	crm_info("%s is not a Pacemaker node", pcmk__node_name(this_node));

	} else if (this_node->details->unclean) {
	pcmk__sched_warn("%s is unclean", pcmk__node_name(this_node));

	} else if (!this_node->details->online) {
	crm_trace("%s is offline", pcmk__node_name(this_node));

	} else if (this_node->details->shutdown) {
	crm_info("%s is shutting down", pcmk__node_name(this_node));

	} else if (this_node->details->pending) {
	crm_info("%s is pending", pcmk__node_name(this_node));

	} else if (pcmk_is_set(this_node->private->flags, pcmk__node_standby)) {
	crm_info("%s is in standby", pcmk__node_name(this_node));

	} else if (this_node->details->maintenance) {
	crm_info("%s is in maintenance", pcmk__node_name(this_node));

	} else {
	crm_info("%s is online", pcmk__node_name(this_node));
	}
	}

	/*!
	* \internal
	* \brief Find the end of a resource's name, excluding any clone suffix
	*
	* \param[in] id Resource ID to check
	*
	* \return Pointer to last character of resource's base name
	*/
	const char *
	pe_base_name_end(const char *id)
	{
	if (!pcmk__str_empty(id)) {
	const char *end = id + strlen(id) - 1;

	for (const char *s = end; s > id; --s) {
	switch (*s) {
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	return (s == end)? s : (s - 1);
	default:
	return end;
	}
	}
	return end;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get a resource name excluding any clone suffix
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_strip(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	char *basename = NULL;

	CRM_ASSERT(end);
	basename = strndup(last_rsc_id, end - last_rsc_id + 1);
	CRM_ASSERT(basename);
	return basename;
	}

	/*!
	* \internal
	* \brief Get the name of the first instance of a cloned resource
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name plus :0
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_zero(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	size_t base_name_len = end - last_rsc_id + 1;
	char *zero = NULL;

	CRM_ASSERT(end);
	zero = pcmk__assert_alloc(base_name_len + 3, sizeof(char));
	memcpy(zero, last_rsc_id, base_name_len);
	zero[base_name_len] = ':';
	zero[base_name_len + 1] = '0';
	return zero;
	}

	static pcmk_resource_t *
	create_fake_resource(const char rsc_id, const xmlNode rsc_entry,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	xmlNode *xml_rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);

	pcmk__xe_copy_attrs(xml_rsc, rsc_entry, pcmk__xaf_none);
	crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (pe__unpack_resource(xml_rsc, &rsc, NULL, scheduler) != pcmk_rc_ok) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	pcmk_node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	node = pcmk_find_node(scheduler, rsc_id);
	if (node == NULL) {
	node = pe_create_node(rsc_id, rsc_id, PCMK_VALUE_REMOTE, NULL,
	scheduler);
	}
	link_rsc2remotenode(scheduler, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, PCMK__META_CONTAINER)) {
	// This removed resource needs to be mapped to a launcher
	crm_trace("Launched resource %s was removed from the configuration",
	rsc_id);
	pcmk__set_rsc_flags(rsc, pcmk__rsc_removed_launched);
	}
	pcmk__set_rsc_flags(rsc, pcmk__rsc_removed);
	scheduler->resources = g_list_append(scheduler->resources, rsc);
	return rsc;
	}

	/*!
	* \internal
	* \brief Create orphan instance for anonymous clone resource history
	*
	* \param[in,out] parent Clone resource that orphan will be added to
	* \param[in] rsc_id Orphan's resource ID
	* \param[in] node Where orphan is active (for logging only)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly added orphaned instance of \p parent
	*/
	static pcmk_resource_t *
	create_anonymous_orphan(pcmk_resource_t parent, const char rsc_id,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *top = pe__create_clone_child(parent, scheduler);
	pcmk_resource_t *orphan = NULL;

	// find_rsc() because we might be a cloned group
	orphan = top->private->fns->find_rsc(top, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	pcmk__rsc_debug(parent, "Created orphan %s for %s: %s on %s",
	top->id, parent->id, rsc_id, pcmk__node_name(node));
	return orphan;
	}

	/*!
	* \internal
	* \brief Check a node for an instance of an anonymous clone
	*
	* Return a child instance of the specified anonymous clone, in order of
	* preference: (1) the instance running on the specified node, if any;
	* (2) an inactive instance (i.e. within the total of \c PCMK_META_CLONE_MAX
	* instances); (3) a newly created orphan (that is, \c PCMK_META_CLONE_MAX
	* instances are already active).
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node on which to check for instance
	* \param[in,out] parent Clone to check
	* \param[in] rsc_id Name of cloned resource in history (no instance)
	*/
	static pcmk_resource_t *
	find_anonymous_clone(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	pcmk_resource_t parent, const char rsc_id)
	{
	GList *rIter = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *inactive_instance = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(pcmk__is_anonymous_clone(parent));

	// Check for active (or partially active, for cloned groups) instance
	pcmk__rsc_trace(parent, "Looking for %s on %s in %s",
	rsc_id, pcmk__node_name(node), parent->id);

	for (rIter = parent->private->children;
	(rIter != NULL) && (rsc == NULL); rIter = rIter->next) {

	GList *locations = NULL;
	pcmk_resource_t *child = rIter->data;

	/* Check whether this instance is already known to be active or pending
	* anywhere, at this stage of unpacking. Because this function is called
	* for a resource before the resource's individual operation history
	* entries are unpacked, locations will generally not contain the
	* desired node.
	*
	* However, there are three exceptions:
	* (1) when child is a cloned group and we have already unpacked the
	* history of another member of the group on the same node;
	* (2) when we've already unpacked the history of another numbered
	* instance on the same node (which can happen if
	* PCMK_META_GLOBALLY_UNIQUE was flipped from true to false); and
	* (3) when we re-run calculations on the same scheduler data as part of
	* a simulation.
	*/
	child->private->fns->location(child, &locations, 2);
	if (locations) {
	/* We should never associate the same numbered anonymous clone
	* instance with multiple nodes, and clone instances can't migrate,
	* so there must be only one location, regardless of history.
	*/
	CRM_LOG_ASSERT(locations->next == NULL);

	if (pcmk__same_node((pcmk_node_t *) locations->data, node)) {
	/* This child instance is active on the requested node, so check
	* for a corresponding configured resource. We use find_rsc()
	* instead of child because child may be a cloned group, and we
	* need the particular member corresponding to rsc_id.
	*
	* If the history entry is orphaned, rsc will be NULL.
	*/
	rsc = parent->private->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);
	if (rsc) {
	/* If there are multiple instance history entries for an
	* anonymous clone in a single node's history (which can
	* happen if PCMK_META_GLOBALLY_UNIQUE is switched from true
	* to false), we want to consider the instances beyond the
	* first as orphans, even if there are inactive instance
	* numbers available.
	*/
	if (rsc->private->active_nodes != NULL) {
	crm_notice("Active (now-)anonymous clone %s has "
	"multiple (orphan) instance histories on %s",
	parent->id, pcmk__node_name(node));
	skip_inactive = TRUE;
	rsc = NULL;
	} else {
	pcmk__rsc_trace(parent, "Resource %s, active", rsc->id);
	}
	}
	}
	g_list_free(locations);

	} else {
	pcmk__rsc_trace(parent, "Resource %s, skip inactive", child->id);
	if (!skip_inactive && !inactive_instance
	&& !pcmk_is_set(child->flags, pcmk__rsc_blocked)) {
	// Remember one inactive instance in case we don't find active
	inactive_instance =
	parent->private->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	/* ... but don't use it if it was already associated with a
	* pending action on another node
	*/
	if (inactive_instance != NULL) {
	const pcmk_node_t *pending_node = NULL;

	pending_node = inactive_instance->private->pending_node;
	if ((pending_node != NULL)
	&& !pcmk__same_node(pending_node, node)) {
	inactive_instance = NULL;
	}
	}
	}
	}
	}

	if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
	pcmk__rsc_trace(parent, "Resource %s, empty slot",
	inactive_instance->id);
	rsc = inactive_instance;
	}

	/* If the resource has PCMK_META_REQUIRES set to PCMK_VALUE_QUORUM or
	* PCMK_VALUE_NOTHING, and we don't have a clone instance for every node, we
	* don't want to consume a valid instance number for unclean nodes. Such
	* instances may appear to be active according to the history, but should be
	* considered inactive, so we can start an instance elsewhere. Treat such
	* instances as orphans.
	*
	* An exception is instances running on guest nodes -- since guest node
	* "fencing" is actually just a resource stop, requires shouldn't apply.
	*
	* @TODO Ideally, we'd use an inactive instance number if it is not needed
	* for any clean instances. However, we don't know that at this point.
	*/
	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)
	&& (!node->details->online \|\| node->details->unclean)
	&& !pcmk__is_guest_or_bundle_node(node)
	&& !pe__is_universal_clone(parent, scheduler)) {

	rsc = NULL;
	}

	if (rsc == NULL) {
	rsc = create_anonymous_orphan(parent, rsc_id, node, scheduler);
	pcmk__rsc_trace(parent, "Resource %s, orphan", rsc->id);
	}
	return rsc;
	}

	static pcmk_resource_t *
	unpack_find_resource(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	const char *rsc_id)
	{
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	/* If we didn't find the resource by its name in the operation history,
	* check it again as a clone instance. Even when PCMK_META_CLONE_MAX=0,
	* we create a single :0 orphan to match against here.
	*/
	char *clone0_id = clone_zero(rsc_id);
	pcmk_resource_t *clone0 = pe_find_resource(scheduler->resources,
	clone0_id);

	if (clone0 && !pcmk_is_set(clone0->flags, pcmk__rsc_unique)) {
	rsc = clone0;
	parent = uber_parent(clone0);
	crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
	} else {
	crm_trace("%s is not known as %s either (orphan)",
	rsc_id, clone0_id);
	}
	free(clone0_id);

	} else if (rsc->private->variant > pcmk__rsc_variant_primitive) {
	crm_trace("Resource history for %s is orphaned "
	"because it is no longer primitive", rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (pcmk__is_anonymous_clone(parent)) {

	if (pcmk__is_bundled(parent)) {
	rsc = pe__find_bundle_replica(parent->private->parent, node);
	} else {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(scheduler, node, parent, base);
	free(base);
	CRM_ASSERT(rsc != NULL);
	}
	}

	if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_none)
	&& !pcmk__str_eq(rsc_id, rsc->private->history_id, pcmk__str_none)) {

	pcmk__str_update(&(rsc->private->history_id), rsc_id);
	pcmk__rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, pcmk__node_name(node), rsc->id,
	pcmk_is_set(rsc->flags, pcmk__rsc_removed)? " (ORPHAN)" : "");
	}
	return rsc;
	}

	static pcmk_resource_t *
	process_orphan_resource(const xmlNode rsc_entry, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);

	crm_debug("Detected orphan resource %s on %s",
	rsc_id, pcmk__node_name(node));
	rsc = create_fake_resource(rsc_id, rsc_entry, scheduler);
	if (rsc == NULL) {
	return NULL;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed);

	} else {
	CRM_CHECK(rsc != NULL, return NULL);
	pcmk__rsc_trace(rsc, "Added orphan %s", rsc->id);
	resource_location(rsc, NULL, -PCMK_SCORE_INFINITY,
	"__orphan_do_not_run__", scheduler);
	}
	return rsc;
	}

	static void
	process_rsc_state(pcmk_resource_t rsc, pcmk_node_t node,
	enum action_fail_response on_fail)
	{
	pcmk_node_t *tmpnode = NULL;
	char *reason = NULL;
	enum action_fail_response save_on_fail = pcmk_on_fail_ignore;
	pcmk_scheduler_t *scheduler = NULL;
	bool known_active = false;

	CRM_ASSERT(rsc);
	scheduler = rsc->private->scheduler;
	known_active = (rsc->private->orig_role > pcmk_role_stopped);
	pcmk__rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, pcmk_role_text(rsc->private->orig_role),
	pcmk__node_name(node), pcmk_on_fail_text(on_fail));

	/* process current state */
	if (rsc->private->orig_role != pcmk_role_unknown) {
	pcmk_resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->private->probed_nodes,
	node->private->id) == NULL) {
	pcmk_node_t *n = pe__copy_node(node);

	pcmk__rsc_trace(rsc, "%s (%s in history) known on %s",
	rsc->id,
	pcmk__s(rsc->private->history_id, "the same"),
	pcmk__node_name(n));
	g_hash_table_insert(iter->private->probed_nodes,
	(gpointer) n->private->id, n);
	}
	if (pcmk_is_set(iter->flags, pcmk__rsc_unique)) {
	break;
	}
	iter = iter->private->parent;
	}
	}

	/* If a managed resource is believed to be running, but node is down ... */
	if (known_active && !node->details->online && !node->details->maintenance
	&& pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {

	gboolean should_fence = FALSE;

	/* If this is a guest node, fence it (regardless of whether fencing is
	* enabled, because guest node fencing is done by recovery of the
	* container resource rather than by the fencer). Mark the resource
	* we're processing as failed. When the guest comes back up, its
	* operation history in the CIB will be cleared, freeing the affected
	* resource to run again once we are sure we know its state.
	*/
	if (pcmk__is_guest_or_bundle_node(node)) {
	pcmk__set_rsc_flags(rsc, pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);
	should_fence = TRUE;

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	if (pcmk__is_remote_node(node)
	&& (node->private->remote != NULL)
	&& !pcmk_is_set(node->private->remote->flags,
	pcmk__rsc_failed)) {

	/* Setting unseen means that fencing of the remote node will
	* occur only if the connection resource is not going to start
	* somewhere. This allows connection resources on a failed
	* cluster node to move to another node without requiring the
	* remote nodes to be fenced as well.
	*/
	pcmk__clear_node_flags(node, pcmk__node_seen);
	reason = crm_strdup_printf("%s is active there (fencing will be"
	" revoked if remote connection can "
	"be re-established elsewhere)",
	rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
	}
	pe_fence_node(scheduler, node, reason, FALSE);
	}
	free(reason);
	}

	/* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
	save_on_fail = on_fail;

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = pcmk_on_fail_ignore;
	}

	switch (on_fail) {
	case pcmk_on_fail_ignore:
	/* nothing to do */
	break;

	case pcmk_on_fail_demote:
	pcmk__set_rsc_flags(rsc, pcmk__rsc_failed);
	demote_action(rsc, node, FALSE);
	break;

	case pcmk_on_fail_fence_node:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	reason = crm_strdup_printf("%s failed there", rsc->id);
	pe_fence_node(scheduler, node, reason, FALSE);
	free(reason);
	break;

	case pcmk_on_fail_standby_node:
	pcmk__set_node_flags(node,
	pcmk__node_standby\|pcmk__node_fail_standby);
	break;

	case pcmk_on_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed);
	pcmk__set_rsc_flags(rsc, pcmk__rsc_blocked);
	break;

	case pcmk_on_fail_ban:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -PCMK_SCORE_INFINITY,
	"__action_migration_auto__", scheduler);
	break;

	case pcmk_on_fail_stop:
	pe__set_next_role(rsc, pcmk_role_stopped,
	PCMK_META_ON_FAIL "=" PCMK_VALUE_STOP);
	break;

	case pcmk_on_fail_restart:
	if (known_active) {
	pcmk__set_rsc_flags(rsc,
	pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_restart_container:
	pcmk__set_rsc_flags(rsc, pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);
	if ((rsc->private->launcher != NULL) && pcmk__is_bundled(rsc)) {
	/* A bundle's remote connection can run on a different node than
	* the bundle's container. We don't necessarily know where the
	* container is running yet, so remember it and add a stop
	* action for it later.
	*/
	scheduler->stop_needed = g_list_prepend(scheduler->stop_needed,
	rsc->private->launcher);
	} else if (rsc->private->launcher != NULL) {
	stop_action(rsc->private->launcher, node, FALSE);
	} else if (known_active) {
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_reset_remote:
	pcmk__set_rsc_flags(rsc, pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	tmpnode = NULL;
	if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
	tmpnode = pcmk_find_node(scheduler, rsc->id);
	}
	if (pcmk__is_remote_node(tmpnode)
	&& !pcmk_is_set(tmpnode->private->flags,
	pcmk__node_remote_fenced)) {
	/* The remote connection resource failed in a way that
	* should result in fencing the remote node.
	*/
	pe_fence_node(scheduler, tmpnode,
	"remote connection is unrecoverable", FALSE);
	}
	}

	/* require the stop action regardless if fencing is occurring or not. */
	if (known_active) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->private->remote_reconnect_ms > 0U) {
	pe__set_next_role(rsc, pcmk_role_stopped, "remote reset");
	}
	break;
	}

	/* Ensure a remote connection failure forces an unclean Pacemaker Remote
	* node to be fenced. By marking the node as seen, the failure will result
	* in a fencing operation regardless if we're going to attempt to reconnect
	* in this transition.
	*/
	if (pcmk_all_flags_set(rsc->flags,
	pcmk__rsc_failed\|pcmk__rsc_is_remote_connection)) {
	tmpnode = pcmk_find_node(scheduler, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	pcmk__set_node_flags(tmpnode, pcmk__node_seen);
	}
	}

	if (known_active) {
	if (pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
	if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	crm_notice("Removed resource %s is active on %s and will be "
	"stopped when possible",
	rsc->id, pcmk__node_name(node));
	} else {
	crm_notice("Removed resource %s must be stopped manually on %s "
	"because " PCMK_OPT_STOP_ORPHAN_RESOURCES
	" is set to false", rsc->id, pcmk__node_name(node));
	}
	}

	native_add_running(rsc, node, scheduler,
	(save_on_fail != pcmk_on_fail_ignore));
	switch (on_fail) {
	case pcmk_on_fail_ignore:
	break;
	case pcmk_on_fail_demote:
	case pcmk_on_fail_block:
	pcmk__set_rsc_flags(rsc, pcmk__rsc_failed);
	break;
	default:
	pcmk__set_rsc_flags(rsc,
	pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);
	break;
	}

	} else if ((rsc->private->history_id != NULL)
	&& (strchr(rsc->private->history_id, ':') != NULL)) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pcmk__rsc_trace(rsc, "Clearing history ID %s for %s (stopped)",
	rsc->private->history_id, rsc->id);
	free(rsc->private->history_id);
	rsc->private->history_id = NULL;

	} else {
	GList *possible_matches = pe__resource_actions(rsc, node,
	PCMK_ACTION_STOP, FALSE);
	GList *gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t stop = (pcmk_action_t ) gIter->data;

	pcmk__set_action_flags(stop, pcmk_action_optional);
	}

	g_list_free(possible_matches);
	}

	/* A successful stop after migrate_to on the migration source doesn't make
	* the partially migrated resource stopped on the migration target.
	*/
	if ((rsc->private->orig_role == pcmk_role_stopped)
	&& (rsc->private->active_nodes != NULL)
	&& (rsc->private->partial_migration_target != NULL)
	&& pcmk__same_node(rsc->private->partial_migration_source, node)) {

	rsc->private->orig_role = pcmk_role_started;
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(pcmk_node_t node, pcmk_resource_t rsc,
	int start_index, int stop_index,
	GList sorted_op_list, pcmk_scheduler_t scheduler)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GList *gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "%s: Start index %d, stop index = %d",
	rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	guint interval_ms = 0;
	char *key = NULL;
	const char *id = pcmk__xe_id(rsc_op);

	counter++;

	if (node->details->online == FALSE) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: node is offline",
	rsc->id, pcmk__node_name(node));
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: resource is not active",
	id, pcmk__node_name(node));
	continue;

	} else if (counter < start_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: old %d",
	id, pcmk__node_name(node), counter);
	continue;
	}

	crm_element_value_ms(rsc_op, PCMK_META_INTERVAL, &interval_ms);
	if (interval_ms == 0) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: non-recurring",
	id, pcmk__node_name(node));
	continue;
	}

	status = crm_element_value(rsc_op, PCMK__XA_OP_STATUS);
	if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: status",
	id, pcmk__node_name(node));
	continue;
	}
	task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	/* create the action */
	key = pcmk__op_key(rsc->id, task, interval_ms);
	pcmk__rsc_trace(rsc, "Creating %s on %s", key, pcmk__node_name(node));
	custom_action(rsc, key, task, node, TRUE, scheduler);
	}
	}

	void
	calculate_active_ops(const GList sorted_op_list, int start_index,
	int *stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_clone_start = -1;
	const char *task = NULL;
	const char *status = NULL;

	*stop_index = -1;
	*start_index = -1;

	for (const GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	const xmlNode rsc_op = (const xmlNode ) iter->data;

	counter++;

	task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	status = crm_element_value(rsc_op, PCMK__XA_OP_STATUS);

	if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& pcmk__str_eq(status, "0", pcmk__str_casei)) {
	*stop_index = counter;

	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_START,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR,
	pcmk__str_casei)) {
	const char *rc = crm_element_value(rsc_op, PCMK__XA_RC_CODE);

	if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
	implied_monitor_start = counter;
	}
	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_DEMOTE, NULL)) {
	implied_clone_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_clone_start != -1) {
	*start_index = implied_clone_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	// If resource history entry has shutdown lock, remember lock node and time
	static void
	unpack_shutdown_lock(const xmlNode rsc_entry, pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	time_t lock_time = 0; // When lock started (i.e. node shutdown time)

	if ((crm_element_value_epoch(rsc_entry, PCMK_OPT_SHUTDOWN_LOCK,
	&lock_time) == pcmk_ok) && (lock_time != 0)) {

	if ((scheduler->shutdown_lock > 0)
	&& (get_effective_time(scheduler)
	> (lock_time + scheduler->shutdown_lock))) {
	pcmk__rsc_info(rsc, "Shutdown lock for %s on %s expired",
	rsc->id, pcmk__node_name(node));
	pe__clear_resource_history(rsc, node);
	} else {
	rsc->private->lock_node = node;
	rsc->private->lock_time = lock_time;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack one \c PCMK__XE_LRM_RESOURCE entry from a node's CIB status
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] rsc_entry \c PCMK__XE_LRM_RESOURCE XML being unpacked
	* \param[in,out] scheduler Scheduler data
	*
	* \return Resource corresponding to the entry, or NULL if no operation history
	*/
	static pcmk_resource_t *
	unpack_lrm_resource(pcmk_node_t node, const xmlNode lrm_resource,
	pcmk_scheduler_t *scheduler)
	{
	GList *gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = pcmk_role_unknown;

	const char *rsc_id = pcmk__xe_id(lrm_resource);

	pcmk_resource_t *rsc = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	xmlNode *rsc_op = NULL;
	xmlNode *last_failure = NULL;

	enum action_fail_response on_fail = pcmk_on_fail_ignore;
	enum rsc_role_e saved_role = pcmk_role_unknown;

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring invalid " PCMK__XE_LRM_RESOURCE
	" entry: No " PCMK_XA_ID);
	crm_log_xml_info(lrm_resource, "missing-id");
	return NULL;
	}
	crm_trace("Unpacking " PCMK__XE_LRM_RESOURCE " for %s on %s",
	rsc_id, pcmk__node_name(node));

	/* Build a list of individual PCMK__XE_LRM_RSC_OP entries, so we can sort
	* them
	*/
	for (rsc_op = pcmk__xe_first_child(lrm_resource, PCMK__XE_LRM_RSC_OP, NULL,
	NULL);
	rsc_op != NULL; rsc_op = pcmk__xe_next_same(rsc_op)) {

	op_list = g_list_prepend(op_list, rsc_op);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	}
	}

	/* find the resource */
	rsc = unpack_find_resource(scheduler, node, rsc_id);
	if (rsc == NULL) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	} else {
	rsc = process_orphan_resource(lrm_resource, node, scheduler);
	}
	}
	CRM_ASSERT(rsc != NULL);

	// Check whether the resource is "shutdown-locked" to this node
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	unpack_shutdown_lock(lrm_resource, rsc, node, scheduler);
	}

	/* process operations */
	saved_role = rsc->private->orig_role;
	rsc->private->orig_role = pcmk_role_unknown;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list,
	scheduler);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail);

	if (get_target_role(rsc, &req_role)) {
	if ((rsc->private->next_role == pcmk_role_unknown)
	\|\| (req_role < rsc->private->next_role)) {

	pe__set_next_role(rsc, req_role, PCMK_META_TARGET_ROLE);

	} else if (req_role > rsc->private->next_role) {
	pcmk__rsc_info(rsc,
	"%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, pcmk_role_text(rsc->private->next_role),
	pcmk_role_text(req_role));
	}
	}

	if (saved_role > rsc->private->orig_role) {
	rsc->private->orig_role = saved_role;
	}

	return rsc;
	}

	static void
	handle_removed_launched_resources(const xmlNode *lrm_rsc_list,
	pcmk_scheduler_t *scheduler)
	{
	for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rsc_list, NULL,
	NULL, NULL);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next(rsc_entry)) {

	pcmk_resource_t *rsc;
	pcmk_resource_t *launcher = NULL;
	const char *rsc_id;
	const char *launcher_id = NULL;

	if (!pcmk__xe_is(rsc_entry, PCMK__XE_LRM_RESOURCE)) {
	continue;
	}

	launcher_id = crm_element_value(rsc_entry, PCMK__META_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
	if ((launcher_id == NULL) \|\| (rsc_id == NULL)) {
	continue;
	}

	launcher = pe_find_resource(scheduler->resources, launcher_id);
	if (launcher == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources, rsc_id);
	if ((rsc == NULL) \|\| (rsc->private->launcher != NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk__rsc_removed_launched)) {
	continue;
	}

	pcmk__rsc_trace(rsc, "Mapped launcher of removed resource %s to %s",
	rsc->id, launcher_id);
	rsc->private->launcher = launcher;
	launcher->private->launched = g_list_append(launcher->private->launched,
	rsc);
	}
	}

	/*!
	* \internal
	* \brief Unpack one node's lrm status section
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] xml CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler)
	{
	bool found_removed_launched_resource = false;

	// Drill down to PCMK__XE_LRM_RESOURCES section
	xml = pcmk__xe_first_child(xml, PCMK__XE_LRM, NULL, NULL);
	if (xml == NULL) {
	return;
	}
	xml = pcmk__xe_first_child(xml, PCMK__XE_LRM_RESOURCES, NULL, NULL);
	if (xml == NULL) {
	return;
	}

	// Unpack each PCMK__XE_LRM_RESOURCE entry
	for (const xmlNode *rsc_entry = pcmk__xe_first_child(xml,
	PCMK__XE_LRM_RESOURCE,
	NULL, NULL);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next_same(rsc_entry)) {

	pcmk_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, scheduler);

	if ((rsc != NULL)
	&& pcmk_is_set(rsc->flags, pcmk__rsc_removed_launched)) {
	found_removed_launched_resource = true;
	}
	}

	/* Now that all resource state has been unpacked for this node, map any
	* removed launched resources to their launchers.
	*/
	if (found_removed_launched_resource) {
	handle_removed_launched_resources(xml, scheduler);
	}
	}

	static void
	set_active(pcmk_resource_t *rsc)
	{
	const pcmk_resource_t *top = pe__const_top_resource(rsc, false);

	if (top && pcmk_is_set(top->flags, pcmk__rsc_promotable)) {
	rsc->private->orig_role = pcmk_role_unpromoted;
	} else {
	rsc->private->orig_role = pcmk_role_started;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk_node_t *node = value;
	int *score = user_data;

	- node->weight = *score;
	+ node->assign->score = *score;
	}

	#define XPATH_NODE_STATE "/" PCMK_XE_CIB "/" PCMK_XE_STATUS \
	"/" PCMK__XE_NODE_STATE
	#define SUB_XPATH_LRM_RESOURCE "/" PCMK__XE_LRM \
	"/" PCMK__XE_LRM_RESOURCES \
	"/" PCMK__XE_LRM_RESOURCE
	#define SUB_XPATH_LRM_RSC_OP "/" PCMK__XE_LRM_RSC_OP

	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	int target_rc, pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((resource != NULL) && (op != NULL) && (node != NULL),
	return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", resource, "']"
	SUB_XPATH_LRM_RSC_OP "[@" PCMK_XA_OPERATION "='", op, "'",
	NULL);

	/* Need to check against transition_magic too? */
	if ((source != NULL) && (strcmp(op, PCMK_ACTION_MIGRATE_TO) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" PCMK__META_MIGRATE_TARGET "='", source, "']",
	NULL);

	} else if ((source != NULL)
	&& (strcmp(op, PCMK_ACTION_MIGRATE_FROM) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" PCMK__META_MIGRATE_SOURCE "='", source, "']",
	NULL);
	} else {
	g_string_append_c(xpath, ']');
	}

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);
	g_string_free(xpath, TRUE);

	if (xml && target_rc >= 0) {
	int rc = PCMK_OCF_UNKNOWN_ERROR;
	int status = PCMK_EXEC_ERROR;

	crm_element_value_int(xml, PCMK__XA_RC_CODE, &rc);
	crm_element_value_int(xml, PCMK__XA_OP_STATUS, &status);
	if ((rc != target_rc) \|\| (status != PCMK_EXEC_DONE)) {
	return NULL;
	}
	}
	return xml;
	}

	static xmlNode *
	find_lrm_resource(const char rsc_id, const char node_name,
	pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((rsc_id != NULL) && (node_name != NULL), return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", rsc_id, "']",
	NULL);

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);

	g_string_free(xpath, TRUE);
	return xml;
	}

	/*!
	* \internal
	* \brief Check whether a resource has no completed action history on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node_name Node to check
	*
	* \return true if \p rsc_id is unknown on \p node_name, otherwise false
	*/
	static bool
	unknown_on_node(pcmk_resource_t rsc, const char node_name)
	{
	bool result = false;
	xmlXPathObjectPtr search;
	char *xpath = NULL;

	xpath = crm_strdup_printf(XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='%s']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='%s']"
	SUB_XPATH_LRM_RSC_OP
	"[@" PCMK__XA_RC_CODE "!='%d']",
	node_name, rsc->id, PCMK_OCF_UNKNOWN);

	search = xpath_search(rsc->private->scheduler->input, xpath);
	result = (numXpathResults(search) == 0);
	freeXpathObject(search);
	free(xpath);
	return result;
	}

	/*!
	* \brief Check whether a probe/monitor indicating the resource was not running
	* on a node happened after some event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a monitor happened after event, false otherwise
	*/
	static bool
	monitor_not_running_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	/* Any probe/monitor operation on the node indicating it was not running
	* there
	*/
	xmlNode *monitor = find_lrm_op(rsc_id, PCMK_ACTION_MONITOR, node_name,
	NULL, PCMK_OCF_NOT_RUNNING, scheduler);

	return (monitor && pe__is_newer_op(monitor, xml_op, same_node) > 0);
	}

	/*!
	* \brief Check whether any non-monitor operation on a node happened after some
	* event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that non-monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	non_monitor_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *lrm_resource = NULL;

	lrm_resource = find_lrm_resource(rsc_id, node_name, scheduler);
	if (lrm_resource == NULL) {
	return false;
	}

	for (xmlNode *op = pcmk__xe_first_child(lrm_resource, PCMK__XE_LRM_RSC_OP,
	NULL, NULL);
	op != NULL; op = pcmk__xe_next_same(op)) {

	const char * task = NULL;

	if (op == xml_op) {
	continue;
	}

	task = crm_element_value(op, PCMK_XA_OPERATION);

	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	NULL)
	&& pe__is_newer_op(op, xml_op, same_node) > 0) {
	return true;
	}
	}

	return false;
	}

	/*!
	* \brief Check whether the resource has newer state on a node after a migration
	* attempt
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] migrate_to Any migrate_to event that is being compared to
	* \param[in] migrate_from Any migrate_from event that is being compared to
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	newer_state_after_migrate(const char rsc_id, const char node_name,
	const xmlNode *migrate_to,
	const xmlNode *migrate_from,
	pcmk_scheduler_t *scheduler)
	{
	const xmlNode *xml_op = migrate_to;
	const char *source = NULL;
	const char *target = NULL;
	bool same_node = false;

	if (migrate_from) {
	xml_op = migrate_from;
	}

	source = crm_element_value(xml_op, PCMK__META_MIGRATE_SOURCE);
	target = crm_element_value(xml_op, PCMK__META_MIGRATE_TARGET);

	/* It's preferred to compare to the migrate event on the same node if
	* existing, since call ids are more reliable.
	*/
	if (pcmk__str_eq(node_name, target, pcmk__str_casei)) {
	if (migrate_from) {
	xml_op = migrate_from;
	same_node = true;

	} else {
	xml_op = migrate_to;
	}

	} else if (pcmk__str_eq(node_name, source, pcmk__str_casei)) {
	if (migrate_to) {
	xml_op = migrate_to;
	same_node = true;

	} else {
	xml_op = migrate_from;
	}
	}

	/* If there's any newer non-monitor operation on the node, or any newer
	* probe/monitor operation on the node indicating it was not running there,
	* the migration events potentially no longer matter for the node.
	*/
	return non_monitor_after(rsc_id, node_name, xml_op, same_node, scheduler)
	\|\| monitor_not_running_after(rsc_id, node_name, xml_op, same_node,
	scheduler);
	}

	/*!
	* \internal
	* \brief Parse migration source and target node names from history entry
	*
	* \param[in] entry Resource history entry for a migration action
	* \param[in] source_node If not NULL, source must match this node
	* \param[in] target_node If not NULL, target must match this node
	* \param[out] source_name Where to store migration source node name
	* \param[out] target_name Where to store migration target node name
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_migration_node_names(const xmlNode entry, const pcmk_node_t source_node,
	const pcmk_node_t *target_node,
	const char source_name, const char target_name)
	{
	*source_name = crm_element_value(entry, PCMK__META_MIGRATE_SOURCE);
	*target_name = crm_element_value(entry, PCMK__META_MIGRATE_TARGET);
	if ((source_name == NULL) \|\| (target_name == NULL)) {
	pcmk__config_err("Ignoring resource history entry %s without "
	PCMK__META_MIGRATE_SOURCE " and "
	PCMK__META_MIGRATE_TARGET, pcmk__xe_id(entry));
	return pcmk_rc_unpack_error;
	}

	if ((source_node != NULL)
	&& !pcmk__str_eq(*source_name, source_node->private->name,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_err("Ignoring resource history entry %s because "
	PCMK__META_MIGRATE_SOURCE "='%s' does not match %s",
	pcmk__xe_id(entry), *source_name,
	pcmk__node_name(source_node));
	return pcmk_rc_unpack_error;
	}

	if ((target_node != NULL)
	&& !pcmk__str_eq(*target_name, target_node->private->name,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_err("Ignoring resource history entry %s because "
	PCMK__META_MIGRATE_TARGET "='%s' does not match %s",
	pcmk__xe_id(entry), *target_name,
	pcmk__node_name(target_node));
	return pcmk_rc_unpack_error;
	}

	return pcmk_rc_ok;
	}

	/*
	* \internal
	* \brief Add a migration source to a resource's list of dangling migrations
	*
	* If the migrate_to and migrate_from actions in a live migration both
	* succeeded, but there is no stop on the source, the migration is considered
	* "dangling." Add the source to the resource's dangling migration list, which
	* will be used to schedule a stop on the source without affecting the target.
	*
	* \param[in,out] rsc Resource involved in migration
	* \param[in] node Migration source
	*/
	static void
	add_dangling_migration(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk__rsc_trace(rsc, "Dangling migration of %s requires stop on %s",
	rsc->id, pcmk__node_name(node));
	rsc->private->orig_role = pcmk_role_stopped;
	rsc->private->dangling_migration_sources =
	g_list_prepend(rsc->private->dangling_migration_sources,
	(gpointer) node);
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a successful migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_success(struct action_history *history)
	{
	/* A complete migration sequence is:
	* 1. migrate_to on source node (which succeeded if we get to this function)
	* 2. migrate_from on target node
	* 3. stop on source node
	*
	* If no migrate_from has happened, the migration is considered to be
	* "partial". If the migrate_from succeeded but no stop has happened, the
	* migration is considered to be "dangling".
	*
	* If a successful migrate_to and stop have happened on the source node, we
	* still need to check for a partial migration, due to scenarios (easier to
	* produce with batch-limit=1) like:
	*
	* - A resource is migrating from node1 to node2, and a migrate_to is
	* initiated for it on node1.
	*
	* - node2 goes into standby mode while the migrate_to is pending, which
	* aborts the transition.
	*
	* - Upon completion of the migrate_to, a new transition schedules a stop
	* on both nodes and a start on node1.
	*
	* - If the new transition is aborted for any reason while the resource is
	* stopping on node1, the transition after that stop completes will see
	* the migrate_to and stop on the source, but it's still a partial
	* migration, and the resource must be stopped on node2 because it is
	* potentially active there due to the migrate_to.
	*
	* We also need to take into account that either node's history may be
	* cleared at any point in the migration process.
	*/
	int from_rc = PCMK_OCF_OK;
	int from_status = PCMK_EXEC_PENDING;
	pcmk_node_t *target_node = NULL;
	xmlNode *migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;
	bool source_newer_op = false;
	bool target_newer_state = false;
	bool active_on_target = false;
	pcmk_scheduler_t *scheduler = history->rsc->private->scheduler;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	// Check for newer state on the source
	source_newer_op = non_monitor_after(history->rsc->id, source, history->xml,
	true, scheduler);

	// Check for a migrate_from action from this source on the target
	migrate_from = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_FROM,
	target, source, -1, scheduler);
	if (migrate_from != NULL) {
	if (source_newer_op) {
	/* There's a newer non-monitor operation on the source and a
	* migrate_from on the target, so this migrate_to is irrelevant to
	* the resource's state.
	*/
	return;
	}
	crm_element_value_int(migrate_from, PCMK__XA_RC_CODE, &from_rc);
	crm_element_value_int(migrate_from, PCMK__XA_OP_STATUS, &from_status);
	}

	/* If the resource has newer state on both the source and target after the
	* migration events, this migrate_to is irrelevant to the resource's state.
	*/
	target_newer_state = newer_state_after_migrate(history->rsc->id, target,
	history->xml, migrate_from,
	scheduler);
	if (source_newer_op && target_newer_state) {
	return;
	}

	/* Check for dangling migration (migrate_from succeeded but stop not done).
	* We know there's no stop because we already returned if the target has a
	* migrate_from and the source has any newer non-monitor operation.
	*/
	if ((from_rc == PCMK_OCF_OK) && (from_status == PCMK_EXEC_DONE)) {
	add_dangling_migration(history->rsc, history->node);
	return;
	}

	/* Without newer state, this migrate_to implies the resource is active.
	* (Clones are not allowed to migrate, so role can't be promoted.)
	*/
	history->rsc->private->orig_role = pcmk_role_started;

	target_node = pcmk_find_node(scheduler, target);
	active_on_target = !target_newer_state && (target_node != NULL)
	&& target_node->details->online;

	if (from_status != PCMK_EXEC_PENDING) { // migrate_from failed on target
	if (active_on_target) {
	native_add_running(history->rsc, target_node, scheduler, TRUE);
	} else {
	// Mark resource as failed, require recovery, and prevent migration
	pcmk__set_rsc_flags(history->rsc,
	pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);
	pcmk__clear_rsc_flags(history->rsc, pcmk__rsc_migratable);
	}
	return;
	}

	// The migrate_from is pending, complete but erased, or to be scheduled

	/* If there is no history at all for the resource on an online target, then
	* it was likely cleaned. Just return, and we'll schedule a probe. Once we
	* have the probe result, it will be reflected in target_newer_state.
	*/
	if ((target_node != NULL) && target_node->details->online
	&& unknown_on_node(history->rsc, target)) {
	return;
	}

	if (active_on_target) {
	pcmk_node_t *source_node = pcmk_find_node(scheduler, source);

	native_add_running(history->rsc, target_node, scheduler, FALSE);
	if ((source_node != NULL) && source_node->details->online) {
	/* This is a partial migration: the migrate_to completed
	* successfully on the source, but the migrate_from has not
	* completed. Remember the source and target; if the newly
	* chosen target remains the same when we schedule actions
	* later, we may continue with the migration.
	*/
	history->rsc->private->partial_migration_target = target_node;
	history->rsc->private->partial_migration_source = source_node;
	}

	} else if (!source_newer_op) {
	// Mark resource as failed, require recovery, and prevent migration
	pcmk__set_rsc_flags(history->rsc,
	pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);
	pcmk__clear_rsc_flags(history->rsc, pcmk__rsc_migratable);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_failure(struct action_history *history)
	{
	xmlNode *target_migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;
	pcmk_scheduler_t *scheduler = history->rsc->private->scheduler;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->private->orig_role = pcmk_role_started;

	// Check for migrate_from on the target
	target_migrate_from = find_lrm_op(history->rsc->id,
	PCMK_ACTION_MIGRATE_FROM, target, source,
	PCMK_OCF_OK, scheduler);

	if (/* If the resource state is unknown on the target, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, target)
	/* If the resource has newer state on the target after the migration
	* events, this migrate_to no longer matters for the target.
	*/
	&& !newer_state_after_migrate(history->rsc->id, target, history->xml,
	target_migrate_from, scheduler)) {
	/* The resource has no newer state on the target, so assume it's still
	* active there.
	* (if it is up).
	*/
	pcmk_node_t *target_node = pcmk_find_node(scheduler, target);

	if (target_node && target_node->details->online) {
	native_add_running(history->rsc, target_node, scheduler, FALSE);
	}

	} else if (!non_monitor_after(history->rsc->id, source, history->xml, true,
	scheduler)) {
	/* We know the resource has newer state on the target, but this
	* migrate_to still matters for the source as long as there's no newer
	* non-monitor operation there.
	*/

	// Mark node as having dangling migration so we can force a stop later
	history->rsc->private->dangling_migration_sources =
	g_list_prepend(history->rsc->private->dangling_migration_sources,
	(gpointer) history->node);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_from action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_from_failure(struct action_history *history)
	{
	xmlNode *source_migrate_to = NULL;
	const char *source = NULL;
	const char *target = NULL;
	pcmk_scheduler_t *scheduler = history->rsc->private->scheduler;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, NULL, history->node, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->private->orig_role = pcmk_role_started;

	// Check for a migrate_to on the source
	source_migrate_to = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_TO,
	source, target, PCMK_OCF_OK, scheduler);

	if (/* If the resource state is unknown on the source, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, source)
	/* If the resource has newer state on the source after the migration
	* events, this migrate_from no longer matters for the source.
	*/
	&& !newer_state_after_migrate(history->rsc->id, source,
	source_migrate_to, history->xml,
	scheduler)) {
	/* The resource has no newer state on the source, so assume it's still
	* active there (if it is up).
	*/
	pcmk_node_t *source_node = pcmk_find_node(scheduler, source);

	if (source_node && source_node->details->online) {
	native_add_running(history->rsc, source_node, scheduler, TRUE);
	}
	}
	}

	/*!
	* \internal
	* \brief Add an action to cluster's list of failed actions
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	record_failed_op(struct action_history *history)
	{
	const pcmk_scheduler_t *scheduler = history->rsc->private->scheduler;

	if (!(history->node->details->online)) {
	return;
	}

	for (const xmlNode *xIter = scheduler->failed->children;
	xIter != NULL; xIter = xIter->next) {

	const char *key = pcmk__xe_history_key(xIter);
	const char *uname = crm_element_value(xIter, PCMK_XA_UNAME);

	if (pcmk__str_eq(history->key, key, pcmk__str_none)
	&& pcmk__str_eq(uname, history->node->private->name,
	pcmk__str_casei)) {
	crm_trace("Skipping duplicate entry %s on %s",
	history->key, pcmk__node_name(history->node));
	return;
	}
	}

	crm_trace("Adding entry for %s on %s to failed action list",
	history->key, pcmk__node_name(history->node));
	crm_xml_add(history->xml, PCMK_XA_UNAME, history->node->private->name);
	crm_xml_add(history->xml, PCMK__XA_RSC_ID, history->rsc->id);
	pcmk__xml_copy(scheduler->failed, history->xml);
	}

	static char *
	last_change_str(const xmlNode *xml_op)
	{
	time_t when;
	char *result = NULL;

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&when) == pcmk_ok) {
	char *when_s = pcmk__epoch2str(&when, 0);
	const char *p = strchr(when_s, ' ');

	// Skip day of week to make message shorter
	if ((p != NULL) && (*(++p) != '\0')) {
	result = pcmk__str_copy(p);
	}
	free(when_s);
	}

	if (result == NULL) {
	result = pcmk__str_copy("unknown_time");
	}

	return result;
	}

	/*!
	* \internal
	* \brief Compare two on-fail values
	*
	* \param[in] first One on-fail value to compare
	* \param[in] second The other on-fail value to compare
	*
	* \return A negative number if second is more severe than first, zero if they
	* are equal, or a positive number if first is more severe than second.
	* \note This is only needed until the action_fail_response values can be
	* renumbered at the next API compatibility break.
	*/
	static int
	cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
	{
	switch (first) {
	case pcmk_on_fail_demote:
	switch (second) {
	case pcmk_on_fail_ignore:
	return 1;
	case pcmk_on_fail_demote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_reset_remote:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return 1;
	case pcmk_on_fail_reset_remote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return 1;
	case pcmk_on_fail_restart_container:
	return 0;
	default:
	return -1;
	}
	break;

	default:
	break;
	}
	switch (second) {
	case pcmk_on_fail_demote:
	return (first == pcmk_on_fail_ignore)? -1 : 1;

	case pcmk_on_fail_reset_remote:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return -1;
	default:
	return 1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return -1;
	default:
	return 1;
	}
	break;

	default:
	break;
	}
	return first - second;
	}

	/*!
	* \internal
	* \brief Ban a resource (or its clone if an anonymous instance) from all nodes
	*
	* \param[in,out] rsc Resource to ban
	*/
	static void
	ban_from_all_nodes(pcmk_resource_t *rsc)
	{
	int score = -PCMK_SCORE_INFINITY;
	const pcmk_scheduler_t *scheduler = rsc->private->scheduler;

	if (rsc->private->parent != NULL) {
	pcmk_resource_t *parent = uber_parent(rsc);

	if (pcmk__is_anonymous_clone(parent)) {
	/* For anonymous clones, if an operation with
	* PCMK_META_ON_FAIL=PCMK_VALUE_STOP fails for any instance, the
	* entire clone must stop.
	*/
	rsc = parent;
	}
	}

	// Ban the resource from all nodes
	crm_notice("%s will not be started under current conditions", rsc->id);
	if (rsc->private->allowed_nodes != NULL) {
	g_hash_table_destroy(rsc->private->allowed_nodes);
	}
	rsc->private->allowed_nodes = pe__node_list2table(scheduler->nodes);
	g_hash_table_foreach(rsc->private->allowed_nodes, set_node_score, &score);
	}

	/*!
	* \internal
	* \brief Get configured failure handling and role after failure for an action
	*
	* \param[in,out] history Unpacked action history entry
	* \param[out] on_fail Where to set configured failure handling
	* \param[out] fail_role Where to set to role after failure
	*/
	static void
	unpack_failure_handling(struct action_history *history,
	enum action_fail_response *on_fail,
	enum rsc_role_e *fail_role)
	{
	xmlNode *config = pcmk__find_action_config(history->rsc, history->task,
	history->interval_ms, true);

	GHashTable *meta = pcmk__unpack_action_meta(history->rsc, history->node,
	history->task,
	history->interval_ms, config);

	const char *on_fail_str = g_hash_table_lookup(meta, PCMK_META_ON_FAIL);

	*on_fail = pcmk__parse_on_fail(history->rsc, history->task,
	history->interval_ms, on_fail_str);
	fail_role = pcmk__role_after_failure(history->rsc, history->task, on_fail,
	meta);
	g_hash_table_destroy(meta);
	}

	/*!
	* \internal
	* \brief Update resource role, failure handling, etc., after a failed action
	*
	* \param[in,out] history Parsed action result history
	* \param[in] config_on_fail Action failure handling from configuration
	* \param[in] fail_role Resource's role after failure of this action
	* \param[out] last_failure This will be set to the history XML
	* \param[in,out] on_fail Actual handling of action result
	*/
	static void
	unpack_rsc_op_failure(struct action_history *history,
	enum action_fail_response config_on_fail,
	enum rsc_role_e fail_role, xmlNode **last_failure,
	enum action_fail_response *on_fail)
	{
	bool is_probe = false;
	char *last_change_s = NULL;
	pcmk_scheduler_t *scheduler = history->rsc->private->scheduler;

	*last_failure = history->xml;

	is_probe = pcmk_xe_is_probe(history->xml);
	last_change_s = last_change_str(history->xml);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)
	&& (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	crm_trace("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " QB_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pcmk__node_name(history->node), last_change_s,
	history->exit_status, history->id);
	} else {
	pcmk__sched_warn("Unexpected result (%s%s%s) was recorded for %s of "
	"%s on %s at %s " QB_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pcmk__node_name(history->node), last_change_s,
	history->exit_status, history->id);

	if (is_probe && (history->exit_status != PCMK_OCF_OK)
	&& (history->exit_status != PCMK_OCF_NOT_RUNNING)
	&& (history->exit_status != PCMK_OCF_RUNNING_PROMOTED)) {

	/* A failed (not just unexpected) probe result could mean the user
	* didn't know resources will be probed even where they can't run.
	*/
	crm_notice("If it is not possible for %s to run on %s, see "
	"the " PCMK_XA_RESOURCE_DISCOVERY " option for location "
	"constraints",
	history->rsc->id, pcmk__node_name(history->node));
	}

	record_failed_op(history);
	}

	free(last_change_s);

	if (cmp_on_fail(*on_fail, config_on_fail) < 0) {
	pcmk__rsc_trace(history->rsc, "on-fail %s -> %s for %s",
	pcmk_on_fail_text(*on_fail),
	pcmk_on_fail_text(config_on_fail), history->key);
	*on_fail = config_on_fail;
	}

	if (strcmp(history->task, PCMK_ACTION_STOP) == 0) {
	resource_location(history->rsc, history->node, -PCMK_SCORE_INFINITY,
	"__stop_fail__", scheduler);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0) {
	unpack_migrate_to_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_FROM) == 0) {
	unpack_migrate_from_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->private->orig_role = pcmk_role_promoted;

	} else if (strcmp(history->task, PCMK_ACTION_DEMOTE) == 0) {
	if (config_on_fail == pcmk_on_fail_block) {
	history->rsc->private->orig_role = pcmk_role_promoted;
	pe__set_next_role(history->rsc, pcmk_role_stopped,
	"demote with " PCMK_META_ON_FAIL "=block");

	} else if (history->exit_status == PCMK_OCF_NOT_RUNNING) {
	history->rsc->private->orig_role = pcmk_role_stopped;

	} else {
	/* Staying in the promoted role would put the scheduler and
	* controller into a loop. Setting the role to unpromoted is not
	* dangerous because the resource will be stopped as part of
	* recovery, and any promotion will be ordered after that stop.
	*/
	history->rsc->private->orig_role = pcmk_role_unpromoted;
	}
	}

	if (is_probe && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	/* leave stopped */
	pcmk__rsc_trace(history->rsc, "Leaving %s stopped", history->rsc->id);
	history->rsc->private->orig_role = pcmk_role_stopped;

	} else if (history->rsc->private->orig_role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "Setting %s active", history->rsc->id);
	set_active(history->rsc);
	}

	pcmk__rsc_trace(history->rsc,
	"Resource %s: role=%s unclean=%s on_fail=%s fail_role=%s",
	history->rsc->id,
	pcmk_role_text(history->rsc->private->orig_role),
	pcmk__btoa(history->node->details->unclean),
	pcmk_on_fail_text(config_on_fail),
	pcmk_role_text(fail_role));

	if ((fail_role != pcmk_role_started)
	&& (history->rsc->private->next_role < fail_role)) {
	pe__set_next_role(history->rsc, fail_role, "failure");
	}

	if (fail_role == pcmk_role_stopped) {
	ban_from_all_nodes(history->rsc);
	}
	}

	/*!
	* \internal
	* \brief Block a resource with a failed action if it cannot be recovered
	*
	* If resource action is a failed stop and fencing is not possible, mark the
	* resource as unmanaged and blocked, since recovery cannot be done.
	*
	* \param[in,out] history Parsed action history entry
	*/
	static void
	block_if_unrecoverable(struct action_history *history)
	{
	char *last_change_s = NULL;

	if (strcmp(history->task, PCMK_ACTION_STOP) != 0) {
	return; // All actions besides stop are always recoverable
	}
	if (pe_can_fence(history->node->private->scheduler, history->node)) {
	return; // Failed stops are recoverable via fencing
	}

	last_change_s = last_change_str(history->xml);
	pcmk__sched_err("No further recovery can be attempted for %s "
	"because %s on %s failed (%s%s%s) at %s "
	QB_XS " rc=%d id=%s",
	history->rsc->id, history->task,
	pcmk__node_name(history->node),
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	last_change_s, history->exit_status, history->id);

	free(last_change_s);

	pcmk__clear_rsc_flags(history->rsc, pcmk__rsc_managed);
	pcmk__set_rsc_flags(history->rsc, pcmk__rsc_blocked);
	}

	/*!
	* \internal
	* \brief Update action history's execution status and why
	*
	* \param[in,out] history Parsed action history entry
	* \param[out] why Where to store reason for update
	* \param[in] value New value
	* \param[in] reason Description of why value was changed
	*/
	static inline void
	remap_because(struct action_history history, const char *why, int value,
	const char *reason)
	{
	if (history->execution_status != value) {
	history->execution_status = value;
	*why = reason;
	}
	}

	/*!
	* \internal
	* \brief Remap informational monitor results and operation status
	*
	* For the monitor results, certain OCF codes are for providing extended information
	* to the user about services that aren't yet failed but not entirely healthy either.
	* These must be treated as the "normal" result by Pacemaker.
	*
	* For operation status, the action result can be used to determine an appropriate
	* status for the purposes of responding to the action. The status provided by the
	* executor is not directly usable since the executor does not know what was expected.
	*
	* \param[in,out] history Parsed action history entry
	* \param[in,out] on_fail What should be done about the result
	* \param[in] expired Whether result is expired
	*
	* \note If the result is remapped and the node is not shutting down or failed,
	* the operation will be recorded in the scheduler data's list of failed
	* operations to highlight it for the user.
	*
	* \note This may update the resource's current and next role.
	*/
	static void
	remap_operation(struct action_history *history,
	enum action_fail_response *on_fail, bool expired)
	{
	bool is_probe = false;
	int orig_exit_status = history->exit_status;
	int orig_exec_status = history->execution_status;
	const char *why = NULL;
	const char *task = history->task;

	// Remap degraded results to their successful counterparts
	history->exit_status = pcmk__effective_rc(history->exit_status);
	if (history->exit_status != orig_exit_status) {
	why = "degraded result";
	if (!expired && (!history->node->details->shutdown
	\|\| history->node->details->online)) {
	record_failed_op(history);
	}
	}

	if (!pcmk__is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& ((history->execution_status != PCMK_EXEC_DONE)
	\|\| (history->exit_status != PCMK_OCF_NOT_RUNNING))) {
	history->execution_status = PCMK_EXEC_DONE;
	history->exit_status = PCMK_OCF_NOT_RUNNING;
	why = "equivalent probe result";
	}

	/* If the executor reported an execution status of anything but done or
	* error, consider that final. But for done or error, we know better whether
	* it should be treated as a failure or not, because we know the expected
	* result.
	*/
	switch (history->execution_status) {
	case PCMK_EXEC_DONE:
	case PCMK_EXEC_ERROR:
	break;

	// These should be treated as node-fatal
	case PCMK_EXEC_NO_FENCE_DEVICE:
	case PCMK_EXEC_NO_SECRETS:
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"node-fatal error");
	goto remap_done;

	default:
	goto remap_done;
	}

	is_probe = pcmk_xe_is_probe(history->xml);
	if (is_probe) {
	task = "probe";
	}

	if (history->expected_exit_status < 0) {
	/* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
	* Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
	* expected exit status in the transition key, which (along with the
	* similar case of a corrupted transition key in the CIB) will be
	* reported to this function as -1. Pacemaker 2.0+ does not support
	* rolling upgrades from those versions or processing of saved CIB files
	* from those versions, so we do not need to care much about this case.
	*/
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"obsolete history format");
	pcmk__config_warn("Expected result not found for %s on %s "
	"(corrupt or obsolete CIB?)",
	history->key, pcmk__node_name(history->node));

	} else if (history->exit_status == history->expected_exit_status) {
	remap_because(history, &why, PCMK_EXEC_DONE, "expected result");

	} else {
	remap_because(history, &why, PCMK_EXEC_ERROR, "unexpected result");
	pcmk__rsc_debug(history->rsc,
	"%s on %s: expected %d (%s), got %d (%s%s%s)",
	history->key, pcmk__node_name(history->node),
	history->expected_exit_status,
	services_ocf_exitcode_str(history->expected_exit_status),
	history->exit_status,
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""));
	}

	switch (history->exit_status) {
	case PCMK_OCF_OK:
	if (is_probe
	&& (history->expected_exit_status == PCMK_OCF_NOT_RUNNING)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active on %s at %s",
	history->rsc->id, pcmk__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe
	\|\| (history->expected_exit_status == history->exit_status)
	\|\| !pcmk_is_set(history->rsc->flags, pcmk__rsc_managed)) {

	/* For probes, recurring monitors for the Stopped role, and
	* unmanaged resources, "not running" is not considered a
	* failure.
	*/
	remap_because(history, &why, PCMK_EXEC_DONE, "exit status");
	history->rsc->private->orig_role = pcmk_role_stopped;
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"not running");
	}
	break;

	case PCMK_OCF_RUNNING_PROMOTED:
	if (is_probe
	&& (history->exit_status != history->expected_exit_status)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active and promoted on %s at %s",
	history->rsc->id,
	pcmk__node_name(history->node), last_change_s);
	free(last_change_s);
	}
	if (!expired
	\|\| (history->exit_status == history->expected_exit_status)) {
	history->rsc->private->orig_role = pcmk_role_promoted;
	}
	break;

	case PCMK_OCF_FAILED_PROMOTED:
	if (!expired) {
	history->rsc->private->orig_role = pcmk_role_promoted;
	}
	remap_because(history, &why, PCMK_EXEC_ERROR, "exit status");
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	remap_because(history, &why, PCMK_EXEC_ERROR_FATAL, "exit status");
	break;

	case PCMK_OCF_UNIMPLEMENT_FEATURE:
	{
	guint interval_ms = 0;
	crm_element_value_ms(history->xml, PCMK_META_INTERVAL,
	&interval_ms);

	if (interval_ms == 0) {
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"exit status");
	} else {
	remap_because(history, &why, PCMK_EXEC_NOT_SUPPORTED,
	"exit status");
	}
	}
	break;

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD, "exit status");
	break;

	default:
	if (history->execution_status == PCMK_EXEC_DONE) {
	char *last_change_s = last_change_str(history->xml);

	crm_info("Treating unknown exit status %d from %s of %s "
	"on %s at %s as failure",
	history->exit_status, task, history->rsc->id,
	pcmk__node_name(history->node), last_change_s);
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"unknown exit status");
	free(last_change_s);
	}
	break;
	}

	remap_done:
	if (why != NULL) {
	pcmk__rsc_trace(history->rsc,
	"Remapped %s result from [%s: %s] to [%s: %s] "
	"because of %s",
	history->key, pcmk_exec_status_str(orig_exec_status),
	crm_exit_str(orig_exit_status),
	pcmk_exec_status_str(history->execution_status),
	crm_exit_str(history->exit_status), why);
	}
	}

	// return TRUE if start or monitor last failure but parameters changed
	static bool
	should_clear_for_param_change(const xmlNode xml_op, const char task,
	pcmk_resource_t rsc, pcmk_node_t node)
	{
	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_MONITOR, NULL)) {
	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't allocated resources yet, so we can't reliably
	* substitute addr parameters for the REMOTE_CONTAINER_HACK.
	* When that's needed, defer the check until later.
	*/
	pe__add_param_check(xml_op, rsc, node, pcmk__check_last_failure,
	rsc->private->scheduler);

	} else {
	pcmk__op_digest_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
	rsc->private->scheduler);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s"
	" has no digest to compare",
	rsc->id, pcmk__xe_history_key(xml_op),
	node->private->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	// Order action after fencing of remote node, given connection rsc
	static void
	order_after_remote_fencing(pcmk_action_t action, pcmk_resource_t remote_conn,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_node_t *remote_node = pcmk_find_node(scheduler, remote_conn->id);

	if (remote_node) {
	pcmk_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
	FALSE, scheduler);

	order_actions(fence, action, pcmk__ar_first_implies_then);
	}
	}

	static bool
	should_ignore_failure_timeout(const pcmk_resource_t rsc, const char task,
	guint interval_ms, bool is_last_failure)
	{
	/* Clearing failures of recurring monitors has special concerns. The
	* executor reports only changes in the monitor result, so if the
	* monitor is still active and still getting the same failure result,
	* that will go undetected after the failure is cleared.
	*
	* Also, the operation history will have the time when the recurring
	* monitor result changed to the given code, not the time when the
	* result last happened.
	*
	* @TODO We probably should clear such failures only when the failure
	* timeout has passed since the last occurrence of the failed result.
	* However we don't record that information. We could maybe approximate
	* that by clearing only if there is a more recent successful monitor or
	* stop result, but we don't even have that information at this point
	* since we are still unpacking the resource's operation history.
	*
	* This is especially important for remote connection resources with a
	* reconnect interval, so in that case, we skip clearing failures
	* if the remote node hasn't been fenced.
	*/
	if ((rsc->private->remote_reconnect_ms > 0U)
	&& pcmk_is_set(rsc->private->scheduler->flags,
	pcmk_sched_fencing_enabled)
	&& (interval_ms != 0)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {

	pcmk_node_t *remote_node = pcmk_find_node(rsc->private->scheduler,
	rsc->id);

	if (remote_node && !pcmk_is_set(remote_node->private->flags,
	pcmk__node_remote_fenced)) {
	if (is_last_failure) {
	crm_info("Waiting to clear monitor failure for remote node %s"
	" until fencing has occurred", rsc->id);
	}
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Check operation age and schedule failure clearing when appropriate
	*
	* This function has two distinct purposes. The first is to check whether an
	* operation history entry is expired (i.e. the resource has a failure timeout,
	* the entry is older than the timeout, and the resource either has no fail
	* count or its fail count is entirely older than the timeout). The second is to
	* schedule fail count clearing when appropriate (i.e. the operation is expired
	* and either the resource has an expired fail count or the operation is a
	* last_failure for a remote connection resource with a reconnect interval,
	* or the operation is a last_failure for a start or monitor operation and the
	* resource's parameters have changed since the operation).
	*
	* \param[in,out] history Parsed action result history
	*
	* \return true if operation history entry is expired, otherwise false
	*/
	static bool
	check_operation_expiry(struct action_history *history)
	{
	bool expired = false;
	bool is_last_failure = pcmk__ends_with(history->id, "_last_failure_0");
	time_t last_run = 0;
	int unexpired_fail_count = 0;
	const char *clear_reason = NULL;
	const guint expiration_sec =
	history->rsc->private->failure_expiration_ms / 1000;
	pcmk_scheduler_t *scheduler = history->rsc->private->scheduler;

	if (history->execution_status == PCMK_EXEC_NOT_INSTALLED) {
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not expired: "
	"Not Installed does not expire",
	history->id, pcmk__node_name(history->node));
	return false; // "Not installed" must always be cleared manually
	}

	if ((expiration_sec > 0)
	&& (crm_element_value_epoch(history->xml, PCMK_XA_LAST_RC_CHANGE,
	&last_run) == 0)) {

	/* Resource has a PCMK_META_FAILURE_TIMEOUT and history entry has a
	* timestamp
	*/

	time_t now = get_effective_time(scheduler);
	time_t last_failure = 0;

	// Is this particular operation history older than the failure timeout?
	if ((now >= (last_run + expiration_sec))
	&& !should_ignore_failure_timeout(history->rsc, history->task,
	history->interval_ms,
	is_last_failure)) {
	expired = true;
	}

	// Does the resource as a whole have an unexpired fail count?
	unexpired_fail_count = pe_get_failcount(history->node, history->rsc,
	&last_failure,
	pcmk__fc_effective,
	history->xml);

	// Update scheduler recheck time according to last failure
	crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d "
	"expiration=%s last-failure@%lld",
	history->id, (long long) last_run, (expired? "" : "not "),
	(long long) now, unexpired_fail_count,
	pcmk__readable_interval(expiration_sec * 1000),
	(long long) last_failure);
	last_failure += expiration_sec + 1;
	if (unexpired_fail_count && (now < last_failure)) {
	pe__update_recheck_time(last_failure, scheduler,
	"fail count expiration");
	}
	}

	if (expired) {
	if (pe_get_failcount(history->node, history->rsc, NULL,
	pcmk__fc_default, history->xml)) {
	// There is a fail count ignoring timeout

	if (unexpired_fail_count == 0) {
	// There is no fail count considering timeout
	clear_reason = "it expired";

	} else {
	/* This operation is old, but there is an unexpired fail count.
	* In a properly functioning cluster, this should only be
	* possible if this operation is not a failure (otherwise the
	* fail count should be expired too), so this is really just a
	* failsafe.
	*/
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Unexpired fail count",
	history->id, pcmk__node_name(history->node));
	expired = false;
	}

	} else if (is_last_failure
	&& (history->rsc->private->remote_reconnect_ms > 0U)) {
	/* Clear any expired last failure when reconnect interval is set,
	* even if there is no fail count.
	*/
	clear_reason = "reconnect interval is set";
	}
	}

	if (!expired && is_last_failure
	&& should_clear_for_param_change(history->xml, history->task,
	history->rsc, history->node)) {
	clear_reason = "resource parameters have changed";
	}

	if (clear_reason != NULL) {
	pcmk_action_t *clear_op = NULL;

	// Schedule clearing of the fail count
	clear_op = pe__clear_failcount(history->rsc, history->node,
	clear_reason, scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)
	&& (history->rsc->private->remote_reconnect_ms > 0)) {
	/* If we're clearing a remote connection due to a reconnect
	* interval, we want to wait until any scheduled fencing
	* completes.
	*
	* We could limit this to remote_node->details->unclean, but at
	* this point, that's always true (it won't be reliable until
	* after unpack_node_history() is done).
	*/
	crm_info("Clearing %s failure will wait until any scheduled "
	"fencing of %s completes",
	history->task, history->rsc->id);
	order_after_remote_fencing(clear_op, history->rsc, scheduler);
	}
	}

	if (expired && (history->interval_ms == 0)
	&& pcmk__str_eq(history->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	switch (history->exit_status) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_PROMOTED:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_PROMOTED:
	// Don't expire probes that return these values
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Probe result",
	history->id, pcmk__node_name(history->node));
	expired = false;
	break;
	}
	}

	return expired;
	}

	int
	pe__target_rc_from_xml(const xmlNode *xml_op)
	{
	int target_rc = 0;
	const char *key = crm_element_value(xml_op, PCMK__XA_TRANSITION_KEY);

	if (key == NULL) {
	return -1;
	}
	decode_transition_key(key, NULL, NULL, NULL, &target_rc);
	return target_rc;
	}

	/*!
	* \internal
	* \brief Update a resource's state for an action result
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] exit_status Exit status to base new state on
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	update_resource_state(struct action_history *history, int exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	bool clear_past_failure = false;

	if ((exit_status == PCMK_OCF_NOT_INSTALLED)
	\|\| (!pcmk__is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml))) {
	history->rsc->private->orig_role = pcmk_role_stopped;

	} else if (exit_status == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MONITOR,
	pcmk__str_none)) {
	if ((last_failure != NULL)
	&& pcmk__str_eq(history->key, pcmk__xe_history_key(last_failure),
	pcmk__str_none)) {
	clear_past_failure = true;
	}
	if (history->rsc->private->orig_role < pcmk_role_started) {
	set_active(history->rsc);
	}

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_START, pcmk__str_none)) {
	history->rsc->private->orig_role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	history->rsc->private->orig_role = pcmk_role_stopped;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	history->rsc->private->orig_role = pcmk_role_promoted;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	if (*on_fail == pcmk_on_fail_demote) {
	/* Demote clears an error only if
	* PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE
	*/
	clear_past_failure = true;
	}
	history->rsc->private->orig_role = pcmk_role_unpromoted;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	history->rsc->private->orig_role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {
	unpack_migrate_to_success(history);

	} else if (history->rsc->private->orig_role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "%s active on %s",
	history->rsc->id, pcmk__node_name(history->node));
	set_active(history->rsc);
	}

	if (!clear_past_failure) {
	return;
	}

	switch (*on_fail) {
	case pcmk_on_fail_stop:
	case pcmk_on_fail_ban:
	case pcmk_on_fail_standby_node:
	case pcmk_on_fail_fence_node:
	pcmk__rsc_trace(history->rsc,
	"%s (%s) is not cleared by a completed %s",
	history->rsc->id, pcmk_on_fail_text(*on_fail),
	history->task);
	break;

	case pcmk_on_fail_block:
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_restart_container:
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures");
	break;

	case pcmk_on_fail_reset_remote:
	if (history->rsc->private->remote_reconnect_ms == 0U) {
	/* With no reconnect interval, the connection is allowed to
	* start again after the remote node is fenced and
	* completely stopped. (With a reconnect interval, we wait
	* for the failure to be cleared entirely before attempting
	* to reconnect.)
	*/
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures and reset remote");
	}
	break;
	}
	}

	/*!
	* \internal
	* \brief Check whether a given history entry matters for resource state
	*
	* \param[in] history Parsed action history entry
	*
	* \return true if action can affect resource state, otherwise false
	*/
	static inline bool
	can_affect_state(struct action_history *history)
	{
	#if 0
	/* @COMPAT It might be better to parse only actions we know we're interested
	* in, rather than exclude a couple we don't. However that would be a
	* behavioral change that should be done at a major or minor series release.
	* Currently, unknown operations can affect whether a resource is considered
	* active and/or failed.
	*/
	return pcmk__str_any_of(history->task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	"asyncmon", NULL);
	#else
	return !pcmk__str_any_of(history->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_META_DATA, NULL);
	#endif
	}

	/*!
	* \internal
	* \brief Unpack execution/exit status and exit reason from a history entry
	*
	* \param[in,out] history Action history entry to unpack
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_action_result(struct action_history *history)
	{
	if ((crm_element_value_int(history->xml, PCMK__XA_OP_STATUS,
	&(history->execution_status)) < 0)
	\|\| (history->execution_status < PCMK_EXEC_PENDING)
	\|\| (history->execution_status > PCMK_EXEC_MAX)
	\|\| (history->execution_status == PCMK_EXEC_CANCELLED)) {
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " PCMK__XA_OP_STATUS " '%s'",
	history->id, history->rsc->id,
	pcmk__node_name(history->node),
	pcmk__s(crm_element_value(history->xml,
	PCMK__XA_OP_STATUS),
	""));
	return pcmk_rc_unpack_error;
	}
	if ((crm_element_value_int(history->xml, PCMK__XA_RC_CODE,
	&(history->exit_status)) < 0)
	\|\| (history->exit_status < 0) \|\| (history->exit_status > CRM_EX_MAX)) {
	#if 0
	/* @COMPAT We should ignore malformed entries, but since that would
	* change behavior, it should be done at a major or minor series
	* release.
	*/
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " PCMK__XA_RC_CODE " '%s'",
	history->id, history->rsc->id,
	pcmk__node_name(history->node),
	pcmk__s(crm_element_value(history->xml,
	PCMK__XA_RC_CODE),
	""));
	return pcmk_rc_unpack_error;
	#else
	history->exit_status = CRM_EX_ERROR;
	#endif
	}
	history->exit_reason = crm_element_value(history->xml, PCMK_XA_EXIT_REASON);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Process an action history entry whose result expired
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	*
	* \return Standard Pacemaker return code (in particular, pcmk_rc_ok means the
	* entry needs no further processing)
	*/
	static int
	process_expired_result(struct action_history *history, int orig_exit_status)
	{
	if (!pcmk__is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& (orig_exit_status != history->expected_exit_status)) {

	if (history->rsc->private->orig_role <= pcmk_role_stopped) {
	history->rsc->private->orig_role = pcmk_role_unknown;
	}
	crm_trace("Ignoring resource history entry %s for probe of %s on %s: "
	"Masked failure expired",
	history->id, history->rsc->id,
	pcmk__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->exit_status == history->expected_exit_status) {
	return pcmk_rc_undetermined; // Only failures expire
	}

	if (history->interval_ms == 0) {
	crm_notice("Ignoring resource history entry %s for %s of %s on %s: "
	"Expired failure",
	history->id, history->task, history->rsc->id,
	pcmk__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->node->details->online && !history->node->details->unclean) {
	/* Reschedule the recurring action. schedule_cancel() won't work at
	* this stage, so as a hacky workaround, forcibly change the restart
	* digest so pcmk__check_action_config() does what we want later.
	*
	* @TODO We should skip this if there is a newer successful monitor.
	* Also, this causes rescheduling only if the history entry
	* has a PCMK__XA_OP_DIGEST (which the expire-non-blocked-failure
	* scheduler regression test doesn't, but that may not be a
	* realistic scenario in production).
	*/
	crm_notice("Rescheduling %s-interval %s of %s on %s "
	"after failure expired",
	pcmk__readable_interval(history->interval_ms), history->task,
	history->rsc->id, pcmk__node_name(history->node));
	crm_xml_add(history->xml, PCMK__XA_OP_RESTART_DIGEST,
	"calculated-failure-timeout");
	return pcmk_rc_ok;
	}

	return pcmk_rc_undetermined;
	}

	/*!
	* \internal
	* \brief Process a masked probe failure
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	mask_probe_failure(struct action_history *history, int orig_exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	pcmk_resource_t *ban_rsc = history->rsc;

	if (!pcmk_is_set(history->rsc->flags, pcmk__rsc_unique)) {
	ban_rsc = uber_parent(history->rsc);
	}

	crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
	services_ocf_exitcode_str(orig_exit_status), history->rsc->id,
	pcmk__node_name(history->node));
	update_resource_state(history, history->expected_exit_status, last_failure,
	on_fail);
	crm_xml_add(history->xml, PCMK_XA_UNAME, history->node->private->name);

	record_failed_op(history);
	resource_location(ban_rsc, history->node, -PCMK_SCORE_INFINITY,
	"masked-probe-failure", ban_rsc->private->scheduler);
	}

	/*!
	* \internal Check whether a given failure is for a given pending action
	*
	* \param[in] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*
	* \return true if \p last_failure is failure of pending action in \p history,
	* otherwise false
	* \note Both \p history and \p last_failure must come from the same
	* \c PCMK__XE_LRM_RESOURCE block, as node and resource are assumed to be
	* the same.
	*/
	static bool
	failure_is_newer(const struct action_history *history,
	const xmlNode *last_failure)
	{
	guint failure_interval_ms = 0U;
	long long failure_change = 0LL;
	long long this_change = 0LL;

	if (last_failure == NULL) {
	return false; // Resource has no last_failure entry
	}

	if (!pcmk__str_eq(history->task,
	crm_element_value(last_failure, PCMK_XA_OPERATION),
	pcmk__str_none)) {
	return false; // last_failure is for different action
	}

	if ((crm_element_value_ms(last_failure, PCMK_META_INTERVAL,
	&failure_interval_ms) != pcmk_ok)
	\|\| (history->interval_ms != failure_interval_ms)) {
	return false; // last_failure is for action with different interval
	}

	if ((pcmk__scan_ll(crm_element_value(history->xml, PCMK_XA_LAST_RC_CHANGE),
	&this_change, 0LL) != pcmk_rc_ok)
	\|\| (pcmk__scan_ll(crm_element_value(last_failure,
	PCMK_XA_LAST_RC_CHANGE),
	&failure_change, 0LL) != pcmk_rc_ok)
	\|\| (failure_change < this_change)) {
	return false; // Failure is not known to be newer
	}

	return true;
	}

	/*!
	* \internal
	* \brief Update a resource's role etc. for a pending action
	*
	* \param[in,out] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*/
	static void
	process_pending_action(struct action_history *history,
	const xmlNode *last_failure)
	{
	/* For recurring monitors, a failure is recorded only in RSC_last_failure_0,
	* and there might be a RSC_monitor_INTERVAL entry with the last successful
	* or pending result.
	*
	* If last_failure contains the failure of the pending recurring monitor
	* we're processing here, and is newer, the action is no longer pending.
	* (Pending results have call ID -1, which sorts last, so the last failure
	* if any should be known.)
	*/
	if (failure_is_newer(history, last_failure)) {
	return;
	}

	if (strcmp(history->task, PCMK_ACTION_START) == 0) {
	pcmk__set_rsc_flags(history->rsc, pcmk__rsc_start_pending);
	set_active(history->rsc);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->private->orig_role = pcmk_role_promoted;

	} else if ((strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0)
	&& history->node->details->unclean) {
	/* A migrate_to action is pending on a unclean source, so force a stop
	* on the target.
	*/
	const char *migrate_target = NULL;
	pcmk_node_t *target = NULL;

	migrate_target = crm_element_value(history->xml,
	PCMK__META_MIGRATE_TARGET);
	target = pcmk_find_node(history->rsc->private->scheduler,
	migrate_target);
	if (target != NULL) {
	stop_action(history->rsc, target, FALSE);
	}
	}

	if (history->rsc->private->pending_action != NULL) {
	/* There should never be multiple pending actions, but as a failsafe,
	* just remember the first one processed for display purposes.
	*/
	return;
	}

	if (pcmk_is_probe(history->task, history->interval_ms)) {
	/* Pending probes are currently never displayed, even if pending
	* operations are requested. If we ever want to change that,
	* enable the below and the corresponding part of
	* native.c:native_pending_action().
	*/
	#if 0
	history->rsc->private->pending_action = strdup("probe");
	history->rsc->private->pending_node = history->node;
	#endif
	} else {
	history->rsc->private->pending_action = strdup(history->task);
	history->rsc->private->pending_node = history->node;
	}
	}

	static void
	unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node, xmlNode *xml_op,
	xmlNode *last_failure, enum action_fail_response on_fail)
	{
	int old_rc = 0;
	bool expired = false;
	pcmk_resource_t *parent = rsc;
	enum rsc_role_e fail_role = pcmk_role_unknown;
	enum action_fail_response failure_strategy = pcmk_on_fail_restart;

	struct action_history history = {
	.rsc = rsc,
	.node = node,
	.xml = xml_op,
	.execution_status = PCMK_EXEC_UNKNOWN,
	};

	CRM_CHECK(rsc && node && xml_op, return);

	history.id = pcmk__xe_id(xml_op);
	if (history.id == NULL) {
	pcmk__config_err("Ignoring resource history entry for %s on %s "
	"without ID", rsc->id, pcmk__node_name(node));
	return;
	}

	// Task and interval
	history.task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	if (history.task == NULL) {
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"without " PCMK_XA_OPERATION,
	history.id, rsc->id, pcmk__node_name(node));
	return;
	}
	crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &(history.interval_ms));
	if (!can_affect_state(&history)) {
	pcmk__rsc_trace(rsc,
	"Ignoring resource history entry %s for %s on %s "
	"with irrelevant action '%s'",
	history.id, rsc->id, pcmk__node_name(node),
	history.task);
	return;
	}

	if (unpack_action_result(&history) != pcmk_rc_ok) {
	return; // Error already logged
	}

	history.expected_exit_status = pe__target_rc_from_xml(xml_op);
	history.key = pcmk__xe_history_key(xml_op);
	crm_element_value_int(xml_op, PCMK__XA_CALL_ID, &(history.call_id));

	pcmk__rsc_trace(rsc, "Unpacking %s (%s call %d on %s): %s (%s)",
	history.id, history.task, history.call_id,
	pcmk__node_name(node),
	pcmk_exec_status_str(history.execution_status),
	crm_exit_str(history.exit_status));

	if (node->details->unclean) {
	pcmk__rsc_trace(rsc,
	"%s is running on %s, which is unclean (further action "
	"depends on value of stop's on-fail attribute)",
	rsc->id, pcmk__node_name(node));
	}

	expired = check_operation_expiry(&history);
	old_rc = history.exit_status;

	remap_operation(&history, on_fail, expired);

	if (expired && (process_expired_result(&history, old_rc) == pcmk_rc_ok)) {
	goto done;
	}

	if (!pcmk__is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
	mask_probe_failure(&history, old_rc, *last_failure, on_fail);
	goto done;
	}

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
	parent = uber_parent(rsc);
	}

	switch (history.execution_status) {
	case PCMK_EXEC_PENDING:
	process_pending_action(&history, *last_failure);
	goto done;

	case PCMK_EXEC_DONE:
	update_resource_state(&history, history.exit_status, *last_failure,
	on_fail);
	goto done;

	case PCMK_EXEC_NOT_INSTALLED:
	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if (failure_strategy == pcmk_on_fail_ignore) {
	crm_warn("Cannot ignore failed %s of %s on %s: "
	"Resource agent doesn't exist "
	QB_XS " status=%d rc=%d id=%s",
	history.task, rsc->id, pcmk__node_name(node),
	history.execution_status, history.exit_status,
	history.id);
	/* Also for printing it as "FAILED" by marking it as
	* pcmk__rsc_failed later
	*/
	*on_fail = pcmk_on_fail_ban;
	}
	resource_location(parent, node, -PCMK_SCORE_INFINITY,
	"hard-error", rsc->private->scheduler);
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);
	goto done;

	case PCMK_EXEC_NOT_CONNECTED:
	if (pcmk__is_pacemaker_remote_node(node)
	&& pcmk_is_set(node->private->remote->flags,
	pcmk__rsc_managed)) {
	/* We should never get into a situation where a managed remote
	* connection resource is considered OK but a resource action
	* behind the connection gets a "not connected" status. But as a
	* fail-safe in case a bug or unusual circumstances do lead to
	* that, ensure the remote connection is considered failed.
	*/
	pcmk__set_rsc_flags(node->private->remote,
	pcmk__rsc_failed\|pcmk__rsc_stop_if_failed);
	}
	break; // Not done, do error handling

	case PCMK_EXEC_ERROR:
	case PCMK_EXEC_ERROR_HARD:
	case PCMK_EXEC_ERROR_FATAL:
	case PCMK_EXEC_TIMEOUT:
	case PCMK_EXEC_NOT_SUPPORTED:
	case PCMK_EXEC_INVALID:
	break; // Not done, do error handling

	default: // No other value should be possible at this point
	break;
	}

	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if ((failure_strategy == pcmk_on_fail_ignore)
	\|\| ((failure_strategy == pcmk_on_fail_restart_container)
	&& (strcmp(history.task, PCMK_ACTION_STOP) == 0))) {

	char *last_change_s = last_change_str(xml_op);

	crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s succeeded "
	QB_XS " %s",
	history.task, services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), rsc->id,
	pcmk__node_name(node), last_change_s, history.id);
	free(last_change_s);

	update_resource_state(&history, history.expected_exit_status,
	*last_failure, on_fail);
	crm_xml_add(xml_op, PCMK_XA_UNAME, node->private->name);
	pcmk__set_rsc_flags(rsc, pcmk__rsc_ignore_failure);

	record_failed_op(&history);

	if ((failure_strategy == pcmk_on_fail_restart_container)
	&& cmp_on_fail(*on_fail, pcmk_on_fail_restart) <= 0) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);

	if (history.execution_status == PCMK_EXEC_ERROR_HARD) {
	uint8_t log_level = LOG_ERR;

	if (history.exit_status == PCMK_OCF_NOT_INSTALLED) {
	log_level = LOG_NOTICE;
	}
	do_crm_log(log_level,
	"Preventing %s from restarting on %s because "
	"of hard failure (%s%s%s) " QB_XS " %s",
	parent->id, pcmk__node_name(node),
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, node, -PCMK_SCORE_INFINITY,
	"hard-error", rsc->private->scheduler);

	} else if (history.execution_status == PCMK_EXEC_ERROR_FATAL) {
	pcmk__sched_err("Preventing %s from restarting anywhere because "
	"of fatal failure (%s%s%s) " QB_XS " %s",
	parent->id,
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, NULL, -PCMK_SCORE_INFINITY,
	"fatal-error", rsc->private->scheduler);
	}
	}

	done:
	pcmk__rsc_trace(rsc, "%s role on %s after %s is %s (next %s)",
	rsc->id, pcmk__node_name(node), history.id,
	pcmk_role_text(rsc->private->orig_role),
	pcmk_role_text(rsc->private->next_role));
	}

	static void
	add_node_attrs(const xmlNode xml_obj, pcmk_node_t node, bool overwrite,
	pcmk_scheduler_t *scheduler)
	{
	const char *cluster_name = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pcmk__insert_dup(node->private->attrs,
	CRM_ATTR_UNAME, node->private->name);

	pcmk__insert_dup(node->private->attrs, CRM_ATTR_ID, node->private->id);
	if (pcmk__str_eq(node->private->id, scheduler->dc_uuid, pcmk__str_casei)) {
	scheduler->dc_node = node;
	pcmk__insert_dup(node->private->attrs,
	CRM_ATTR_IS_DC, PCMK_VALUE_TRUE);
	} else {
	pcmk__insert_dup(node->private->attrs,
	CRM_ATTR_IS_DC, PCMK_VALUE_FALSE);
	}

	cluster_name = g_hash_table_lookup(scheduler->config_hash,
	PCMK_OPT_CLUSTER_NAME);
	if (cluster_name) {
	pcmk__insert_dup(node->private->attrs, CRM_ATTR_CLUSTER_NAME,
	cluster_name);
	}

	pe__unpack_dataset_nvpairs(xml_obj, PCMK_XE_INSTANCE_ATTRIBUTES, &rule_data,
	node->private->attrs, NULL, overwrite,
	scheduler);

	pe__unpack_dataset_nvpairs(xml_obj, PCMK_XE_UTILIZATION, &rule_data,
	node->private->utilization, NULL,
	FALSE, scheduler);

	if (pcmk__node_attr(node, CRM_ATTR_SITE_NAME, NULL,
	pcmk__rsc_node_current) == NULL) {
	const char *site_name = pcmk__node_attr(node, "site-name", NULL,
	pcmk__rsc_node_current);

	if (site_name) {
	pcmk__insert_dup(node->private->attrs,
	CRM_ATTR_SITE_NAME, site_name);

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	pcmk__insert_dup(node->private->attrs,
	CRM_ATTR_SITE_NAME, cluster_name);
	}
	}
	}

	static GList *
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GList *gIter = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry, NULL, NULL, NULL);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__xe_is(rsc_op, PCMK__XE_LRM_RSC_OP)) {
	crm_xml_add(rsc_op, PCMK_XA_RESOURCE, rsc);
	crm_xml_add(rsc_op, PCMK_XA_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", pcmk__xe_id(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", pcmk__xe_id(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GList *
	find_operations(const char rsc, const char node, gboolean active_filter,
	pcmk_scheduler_t *scheduler)
	{
	GList *output = NULL;
	GList *intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = pcmk__xe_first_child(scheduler->input, PCMK_XE_STATUS,
	NULL, NULL);

	pcmk_node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	CRM_CHECK(status != NULL, return NULL);

	for (node_state = pcmk__xe_first_child(status, NULL, NULL, NULL);
	node_state != NULL; node_state = pcmk__xe_next(node_state)) {

	if (pcmk__xe_is(node_state, PCMK__XE_NODE_STATE)) {
	const char *uname = crm_element_value(node_state, PCMK_XA_UNAME);

	if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
	continue;
	}

	this_node = pcmk_find_node(scheduler, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (pcmk__is_pacemaker_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);

	} else {
	determine_online_status(node_state, this_node, scheduler);
	}

	if (this_node->details->online
	\|\| pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = pcmk__xe_first_child(node_state, PCMK__XE_LRM, NULL,
	NULL);
	tmp = pcmk__xe_first_child(tmp, PCMK__XE_LRM_RESOURCES, NULL,
	NULL);

	for (lrm_rsc = pcmk__xe_first_child(tmp, NULL, NULL, NULL);
	lrm_rsc != NULL; lrm_rsc = pcmk__xe_next(lrm_rsc)) {

	if (pcmk__xe_is(lrm_rsc, PCMK__XE_LRM_RESOURCE)) {
	const char *rsc_id = crm_element_value(lrm_rsc,
	PCMK_XA_ID);

	if (rsc != NULL && !pcmk__str_eq(rsc_id, rsc, pcmk__str_casei)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}
	diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
	index 85cfd29bd4..466572c831 100644
	--- a/lib/pengine/utils.c
	+++ b/lib/pengine/utils.c
	@@ -1,942 +1,944 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>

	#include "pe_status_private.h"

	extern bool pcmk__is_daemon;

	gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);

	/*!
	* \internal
	* \brief Check whether we can fence a particular node
	*
	* \param[in] scheduler Scheduler data
	* \param[in] node Name of node to check
	*
	* \return true if node can be fenced, false otherwise
	*/
	bool
	pe_can_fence(const pcmk_scheduler_t scheduler, const pcmk_node_t node)
	{
	if (pcmk__is_guest_or_bundle_node(node)) {
	/* A guest or bundle node is fenced by stopping its launcher, which is
	* possible if the launcher's host is either online or fenceable.
	*/
	pcmk_resource_t *rsc = node->private->remote->private->launcher;

	for (GList *n = rsc->private->active_nodes; n != NULL; n = n->next) {
	pcmk_node_t *launcher_node = n->data;

	if (!launcher_node->details->online
	&& !pe_can_fence(scheduler, launcher_node)) {
	return false;
	}
	}
	return true;

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	return false; /* Turned off */

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {
	return false; /* No devices */

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	return true;

	} else if (scheduler->no_quorum_policy == pcmk_no_quorum_ignore) {
	return true;

	} else if(node == NULL) {
	return false;

	} else if(node->details->online) {
	crm_notice("We can fence %s without quorum because they're in our membership",
	pcmk__node_name(node));
	return true;
	}

	crm_trace("Cannot fence %s", pcmk__node_name(node));
	return false;
	}

	/*!
	* \internal
	* \brief Copy a node object
	*
	* \param[in] this_node Node object to copy
	*
	* \return Newly allocated shallow copy of this_node
	* \note This function asserts on errors and is guaranteed to return non-NULL.
	*/
	pcmk_node_t *
	pe__copy_node(const pcmk_node_t *this_node)
	{
	pcmk_node_t *new_node = NULL;

	CRM_ASSERT(this_node != NULL);

	new_node = pcmk__assert_alloc(1, sizeof(pcmk_node_t));
	+ new_node->assign = pcmk__assert_alloc(1,
	+ sizeof(struct pcmk__node_assignment));

	- new_node->rsc_discover_mode = this_node->rsc_discover_mode;
	- new_node->weight = this_node->weight;
	- new_node->fixed = this_node->fixed; // @COMPAT deprecated and unused
	- new_node->count = this_node->count;
	+ new_node->assign->probe_mode = this_node->assign->probe_mode;
	+ new_node->assign->score = this_node->assign->score;
	+ new_node->assign->count = this_node->assign->count;
	new_node->details = this_node->details;
	new_node->private = this_node->private;

	return new_node;
	}

	/*!
	* \internal
	* \brief Create a node hash table from a node list
	*
	* \param[in] list Node list
	*
	* \return Hash table equivalent of node list
	*/
	GHashTable *
	pe__node_list2table(const GList *list)
	{
	GHashTable *result = NULL;

	result = pcmk__strkey_table(NULL, free);
	for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *new_node = NULL;

	new_node = pe__copy_node((const pcmk_node_t *) gIter->data);
	g_hash_table_insert(result, (gpointer) new_node->private->id, new_node);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Compare two nodes by name, with numeric portions sorted numerically
	*
	* Sort two node names case-insensitively like strcasecmp(), but with any
	* numeric portions of the name sorted numerically. For example, "node10" will
	* sort higher than "node9" but lower than "remotenode9".
	*
	* \param[in] a First node to compare (can be \c NULL)
	* \param[in] b Second node to compare (can be \c NULL)
	*
	* \retval -1 \c a comes before \c b (or \c a is \c NULL and \c b is not)
	* \retval 0 \c a and \c b are equal (or both are \c NULL)
	* \retval 1 \c a comes after \c b (or \c b is \c NULL and \c a is not)
	*/
	gint
	pe__cmp_node_name(gconstpointer a, gconstpointer b)
	{
	const pcmk_node_t node1 = (const pcmk_node_t ) a;
	const pcmk_node_t node2 = (const pcmk_node_t ) b;

	if ((node1 == NULL) && (node2 == NULL)) {
	return 0;
	}

	if (node1 == NULL) {
	return -1;
	}

	if (node2 == NULL) {
	return 1;
	}

	return pcmk__numeric_strcasecmp(node1->private->name, node2->private->name);
	}

	/*!
	* \internal
	* \brief Output node weights to stdout
	*
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	pe__output_node_weights(const pcmk_resource_t rsc, const char comment,
	GHashTable nodes, pcmk_scheduler_t scheduler)
	{
	pcmk__output_t *out = scheduler->priv;

	// Sort the nodes so the output is consistent for regression tests
	GList *list = g_list_sort(g_hash_table_get_values(nodes),
	pe__cmp_node_name);

	for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	const pcmk_node_t node = (const pcmk_node_t ) gIter->data;

	out->message(out, "node-weight", rsc, comment, node->private->name,
	- pcmk_readable_score(node->weight));
	+ pcmk_readable_score(node->assign->score));
	}
	g_list_free(list);
	}

	/*!
	* \internal
	* \brief Log node weights at trace level
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] rsc If not NULL, include this resource's ID in logs
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes Nodes whose scores should be logged
	*/
	static void
	pe__log_node_weights(const char file, const char function, int line,
	const pcmk_resource_t rsc, const char comment,
	GHashTable *nodes)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	// Don't waste time if we're not tracing at this point
	pcmk__if_tracing({}, return);

	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (rsc) {
	qb_log_from_external_source(function, file,
	"%s: %s allocation score on %s: %s",
	LOG_TRACE, line, 0,
	comment, rsc->id,
	pcmk__node_name(node),
	- pcmk_readable_score(node->weight));
	+ pcmk_readable_score(node->assign->score));
	} else {
	qb_log_from_external_source(function, file, "%s: %s = %s",
	LOG_TRACE, line, 0,
	comment, pcmk__node_name(node),
	- pcmk_readable_score(node->weight));
	+ pcmk_readable_score(node->assign->score));
	}
	}
	}

	/*!
	* \internal
	* \brief Log or output node weights
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] to_log Log if true, otherwise output
	* \param[in] rsc If not NULL, use this resource's ID in logs,
	* and show scores recursively for any children
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes Nodes whose scores should be shown
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pe__show_node_scores_as(const char file, const char function, int line,
	bool to_log, const pcmk_resource_t *rsc,
	const char comment, GHashTable nodes,
	pcmk_scheduler_t *scheduler)
	{
	if ((rsc != NULL) && pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
	// Don't show allocation scores for orphans
	return;
	}
	if (nodes == NULL) {
	// Nothing to show
	return;
	}

	if (to_log) {
	pe__log_node_weights(file, function, line, rsc, comment, nodes);
	} else {
	pe__output_node_weights(rsc, comment, nodes, scheduler);
	}

	if (rsc == NULL) {
	return;
	}

	// If this resource has children, repeat recursively for each
	for (GList *gIter = rsc->private->children;
	gIter != NULL; gIter = gIter->next) {

	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	pe__show_node_scores_as(file, function, line, to_log, child, comment,
	child->private->allowed_nodes, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Compare two resources by priority
	*
	* \param[in] a First resource to compare (can be \c NULL)
	* \param[in] b Second resource to compare (can be \c NULL)
	*
	* \retval -1 a's priority > b's priority (or \c b is \c NULL and \c a is not)
	* \retval 0 a's priority == b's priority (or both \c a and \c b are \c NULL)
	* \retval 1 a's priority < b's priority (or \c a is \c NULL and \c b is not)
	*/
	gint
	pe__cmp_rsc_priority(gconstpointer a, gconstpointer b)
	{
	const pcmk_resource_t resource1 = (const pcmk_resource_t )a;
	const pcmk_resource_t resource2 = (const pcmk_resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->private->priority > resource2->private->priority) {
	return -1;
	}

	if (resource1->private->priority < resource2->private->priority) {
	return 1;
	}

	return 0;
	}

	static void
	resource_node_score(pcmk_resource_t rsc, const pcmk_node_t node, int score,
	const char *tag)
	{
	pcmk_node_t *match = NULL;

	if ((pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)
	- \|\| (node->rsc_discover_mode == pcmk_probe_never))
	+ \|\| (node->assign->probe_mode == pcmk__probe_never))
	&& pcmk__str_eq(tag, "symmetric_default", pcmk__str_casei)) {
	/* This string comparision may be fragile, but exclusive resources and
	* exclusive nodes should not have the symmetric_default constraint
	* applied to them.
	*/
	return;

	} else {
	for (GList *gIter = rsc->private->children;
	gIter != NULL; gIter = gIter->next) {

	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	resource_node_score(child_rsc, node, score, tag);
	}
	}

	match = g_hash_table_lookup(rsc->private->allowed_nodes, node->private->id);
	if (match == NULL) {
	match = pe__copy_node(node);
	g_hash_table_insert(rsc->private->allowed_nodes,
	(gpointer) match->private->id, match);
	}
	- match->weight = pcmk__add_scores(match->weight, score);
	+ match->assign->score = pcmk__add_scores(match->assign->score, score);
	pcmk__rsc_trace(rsc,
	"Enabling %s preference (%s) for %s on %s (now %s)",
	tag, pcmk_readable_score(score), rsc->id,
	- pcmk__node_name(node), pcmk_readable_score(match->weight));
	+ pcmk__node_name(node),
	+ pcmk_readable_score(match->assign->score));
	}

	void
	resource_location(pcmk_resource_t rsc, const pcmk_node_t node, int score,
	const char tag, pcmk_scheduler_t scheduler)
	{
	if (node != NULL) {
	resource_node_score(rsc, node, score, tag);

	} else if (scheduler != NULL) {
	GList *gIter = scheduler->nodes;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node_iter = (pcmk_node_t ) gIter->data;

	resource_node_score(rsc, node_iter, score, tag);
	}

	} else {
	GHashTableIter iter;
	pcmk_node_t *node_iter = NULL;

	g_hash_table_iter_init(&iter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
	resource_node_score(rsc, node_iter, score, tag);
	}
	}

	if ((node == NULL) && (score == -PCMK_SCORE_INFINITY)
	&& (rsc->private->assigned_node != NULL)) {

	// @TODO Should this be more like pcmk__unassign_resource()?
	crm_info("Unassigning %s from %s",
	rsc->id, pcmk__node_name(rsc->private->assigned_node));
	free(rsc->private->assigned_node);
	rsc->private->assigned_node = NULL;
	}
	}

	time_t
	get_effective_time(pcmk_scheduler_t *scheduler)
	{
	if(scheduler) {
	if (scheduler->now == NULL) {
	crm_trace("Recording a new 'now'");
	scheduler->now = crm_time_new(NULL);
	}
	return crm_time_get_seconds_since_epoch(scheduler->now);
	}

	crm_trace("Defaulting to 'now'");
	return time(NULL);
	}

	gboolean
	get_target_role(const pcmk_resource_t rsc, enum rsc_role_e role)
	{
	enum rsc_role_e local_role = pcmk_role_unknown;
	const char *value = g_hash_table_lookup(rsc->private->meta,
	PCMK_META_TARGET_ROLE);

	CRM_CHECK(role != NULL, return FALSE);

	if (pcmk__str_eq(value, PCMK_ROLE_STARTED,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	return FALSE;
	}
	if (pcmk__str_eq(PCMK_VALUE_DEFAULT, value, pcmk__str_casei)) {
	// @COMPAT Deprecated since 2.1.8
	pcmk__config_warn("Support for setting " PCMK_META_TARGET_ROLE
	" to the explicit value '" PCMK_VALUE_DEFAULT
	"' is deprecated and will be removed in a "
	"future release (just leave it unset)");
	return FALSE;
	}

	local_role = pcmk_parse_role(value);
	if (local_role == pcmk_role_unknown) {
	pcmk__config_err("Ignoring '" PCMK_META_TARGET_ROLE "' for %s "
	"because '%s' is not valid", rsc->id, value);
	return FALSE;

	} else if (local_role > pcmk_role_started) {
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk__rsc_promotable)) {
	if (local_role > pcmk_role_unpromoted) {
	/* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
	return FALSE;
	}

	} else {
	pcmk__config_err("Ignoring '" PCMK_META_TARGET_ROLE "' for %s "
	"because '%s' only makes sense for promotable "
	"clones", rsc->id, value);
	return FALSE;
	}
	}

	*role = local_role;
	return TRUE;
	}

	gboolean
	order_actions(pcmk_action_t lh_action, pcmk_action_t rh_action,
	uint32_t flags)
	{
	GList *gIter = NULL;
	pcmk__related_action_t *wrapper = NULL;
	GList *list = NULL;

	if (flags == pcmk__ar_none) {
	return FALSE;
	}

	if (lh_action == NULL \|\| rh_action == NULL) {
	return FALSE;
	}

	crm_trace("Creating action wrappers for ordering: %s then %s",
	lh_action->uuid, rh_action->uuid);

	/* Ensure we never create a dependency on ourselves... it's happened */
	CRM_ASSERT(lh_action != rh_action);

	/* Filter dups, otherwise update_action_states() has too much work to do */
	gIter = lh_action->actions_after;
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk__related_action_t *after = gIter->data;

	if (after->action == rh_action && (after->type & flags)) {
	return FALSE;
	}
	}

	wrapper = pcmk__assert_alloc(1, sizeof(pcmk__related_action_t));
	wrapper->action = rh_action;
	wrapper->type = flags;
	list = lh_action->actions_after;
	list = g_list_prepend(list, wrapper);
	lh_action->actions_after = list;

	wrapper = pcmk__assert_alloc(1, sizeof(pcmk__related_action_t));
	wrapper->action = lh_action;
	wrapper->type = flags;
	list = rh_action->actions_before;
	list = g_list_prepend(list, wrapper);
	rh_action->actions_before = list;
	return TRUE;
	}

	void
	destroy_ticket(gpointer data)
	{
	pcmk_ticket_t *ticket = data;

	if (ticket->state) {
	g_hash_table_destroy(ticket->state);
	}
	free(ticket->id);
	free(ticket);
	}

	pcmk_ticket_t *
	ticket_new(const char ticket_id, pcmk_scheduler_t scheduler)
	{
	pcmk_ticket_t *ticket = NULL;

	if (pcmk__str_empty(ticket_id)) {
	return NULL;
	}

	if (scheduler->tickets == NULL) {
	scheduler->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {

	ticket = calloc(1, sizeof(pcmk_ticket_t));
	if (ticket == NULL) {
	pcmk__sched_err("Cannot allocate ticket '%s'", ticket_id);
	return NULL;
	}

	crm_trace("Creating ticket entry for %s", ticket_id);

	ticket->id = strdup(ticket_id);
	ticket->granted = FALSE;
	ticket->last_granted = -1;
	ticket->standby = FALSE;
	ticket->state = pcmk__strkey_table(free, free);

	g_hash_table_insert(scheduler->tickets, strdup(ticket->id), ticket);
	}

	return ticket;
	}

	const char *
	rsc_printable_id(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
	return rsc->id;
	}
	return pcmk__xe_id(rsc->private->xml);
	}

	void
	pe__clear_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags)
	{
	pcmk__clear_rsc_flags(rsc, flags);

	for (GList *gIter = rsc->private->children;
	gIter != NULL; gIter = gIter->next) {

	pe__clear_resource_flags_recursive((pcmk_resource_t *) gIter->data,
	flags);
	}
	}

	void
	pe__clear_resource_flags_on_all(pcmk_scheduler_t *scheduler, uint64_t flag)
	{
	for (GList *lpc = scheduler->resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t r = (pcmk_resource_t ) lpc->data;
	pe__clear_resource_flags_recursive(r, flag);
	}
	}

	void
	pe__set_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags)
	{
	pcmk__set_rsc_flags(rsc, flags);

	for (GList *gIter = rsc->private->children;
	gIter != NULL; gIter = gIter->next) {

	pe__set_resource_flags_recursive((pcmk_resource_t *) gIter->data,
	flags);
	}
	}

	void
	trigger_unfencing(pcmk_resource_t rsc, pcmk_node_t node, const char *reason,
	pcmk_action_t dependency, pcmk_scheduler_t scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_enable_unfencing)) {
	/* No resources require it */
	return;

	} else if ((rsc != NULL)
	&& !pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)) {
	/* Wasn't a stonith device */
	return;

	} else if(node
	&& node->details->online
	&& node->details->unclean == FALSE
	&& node->details->shutdown == FALSE) {
	pcmk_action_t *unfence = pe_fence_op(node, PCMK_ACTION_ON, FALSE,
	reason, FALSE, scheduler);

	if(dependency) {
	order_actions(unfence, dependency, pcmk__ar_ordered);
	}

	} else if(rsc) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
	trigger_unfencing(rsc, node, reason, dependency, scheduler);
	}
	}
	}
	}

	gboolean
	add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref)
	{
	pcmk_tag_t *tag = NULL;
	GList *gIter = NULL;
	gboolean is_existing = FALSE;

	CRM_CHECK(tags && tag_name && obj_ref, return FALSE);

	tag = g_hash_table_lookup(tags, tag_name);
	if (tag == NULL) {
	tag = calloc(1, sizeof(pcmk_tag_t));
	if (tag == NULL) {
	pcmk__sched_err("Could not allocate memory for tag %s", tag_name);
	return FALSE;
	}
	tag->id = strdup(tag_name);
	tag->refs = NULL;
	g_hash_table_insert(tags, strdup(tag_name), tag);
	}

	for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
	const char existing_ref = (const char ) gIter->data;

	if (pcmk__str_eq(existing_ref, obj_ref, pcmk__str_none)){
	is_existing = TRUE;
	break;
	}
	}

	if (is_existing == FALSE) {
	tag->refs = g_list_append(tag->refs, strdup(obj_ref));
	crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Check whether shutdown has been requested for a node
	*
	* \param[in] node Node to check
	*
	* \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
	* \note This differs from simply using node->details->shutdown in that it can
	* be used before that has been determined (and in fact to determine it),
	* and it can also be used to distinguish requested shutdown from implicit
	* shutdown of remote nodes by virtue of their connection stopping.
	*/
	bool
	pe__shutdown_requested(const pcmk_node_t *node)
	{
	const char *shutdown = pcmk__node_attr(node, PCMK__NODE_ATTR_SHUTDOWN, NULL,
	pcmk__rsc_node_current);

	return !pcmk__str_eq(shutdown, "0", pcmk__str_null_matches);
	}

	/*!
	* \internal
	* \brief Update "recheck by" time in scheduler data
	*
	* \param[in] recheck Epoch time when recheck should happen
	* \param[in,out] scheduler Scheduler data
	* \param[in] reason What time is being updated for (for logs)
	*/
	void
	pe__update_recheck_time(time_t recheck, pcmk_scheduler_t *scheduler,
	const char *reason)
	{
	if ((recheck > get_effective_time(scheduler))
	&& ((scheduler->recheck_by == 0)
	\|\| (scheduler->recheck_by > recheck))) {
	scheduler->recheck_by = recheck;
	crm_debug("Updated next scheduler recheck to %s for %s",
	pcmk__trim(ctime(&recheck)), reason);
	}
	}

	/*!
	* \internal
	* \brief Extract nvpair blocks contained by a CIB XML element into a hash table
	*
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name If not NULL, only use blocks of this element
	* \param[in] rule_data Matching parameters to use when unpacking
	* \param[out] hash Where to store extracted name/value pairs
	* \param[in] always_first If not NULL, process block with this ID first
	* \param[in] overwrite Whether to replace existing values with same name
	* \param[in,out] scheduler Scheduler data containing \p xml_obj
	*/
	void
	pe__unpack_dataset_nvpairs(const xmlNode xml_obj, const char set_name,
	const pe_rule_eval_data_t *rule_data,
	GHashTable hash, const char always_first,
	gboolean overwrite, pcmk_scheduler_t *scheduler)
	{
	crm_time_t *next_change = crm_time_new_undefined();

	pe_eval_nvpairs(scheduler->input, xml_obj, set_name, rule_data, hash,
	always_first, overwrite, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, scheduler, "rule evaluation");
	}
	crm_time_free(next_change);
	}

	bool
	pe__resource_is_disabled(const pcmk_resource_t *rsc)
	{
	const char *target_role = NULL;

	CRM_CHECK(rsc != NULL, return false);
	target_role = g_hash_table_lookup(rsc->private->meta,
	PCMK_META_TARGET_ROLE);
	if (target_role) {
	// If invalid, we've already logged an error when unpacking
	enum rsc_role_e target_role_e = pcmk_parse_role(target_role);

	if ((target_role_e == pcmk_role_stopped)
	\|\| ((target_role_e == pcmk_role_unpromoted)
	&& pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk__rsc_promotable))) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a resource is running only on given node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p rsc is running only on \p node, otherwise false
	*/
	bool
	pe__rsc_running_on_only(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	return (rsc != NULL) && pcmk__list_of_1(rsc->private->active_nodes)
	&& pcmk__same_node((const pcmk_node_t *)
	rsc->private->active_nodes->data, node);
	}

	bool
	pe__rsc_running_on_any(pcmk_resource_t rsc, GList node_list)
	{
	if (rsc != NULL) {
	for (GList *ele = rsc->private->active_nodes; ele; ele = ele->next) {
	pcmk_node_t node = (pcmk_node_t ) ele->data;
	if (pcmk__str_in_list(node->private->name, node_list,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	return true;
	}
	}
	}
	return false;
	}

	bool
	pcmk__rsc_filtered_by_node(pcmk_resource_t rsc, GList only_node)
	{
	return rsc->private->fns->active(rsc, FALSE)
	&& !pe__rsc_running_on_any(rsc, only_node);
	}

	GList *
	pe__filter_rsc_list(GList rscs, GList filter)
	{
	GList *retval = NULL;

	for (GList *gIter = rscs; gIter; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	/* I think the second condition is safe here for all callers of this
	* function. If not, it needs to move into pe__node_text.
	*/
	if (pcmk__str_in_list(rsc_printable_id(rsc), filter, pcmk__str_star_matches) \|\|
	((rsc->private->parent != NULL)
	&& pcmk__str_in_list(rsc_printable_id(rsc->private->parent),
	filter, pcmk__str_star_matches))) {
	retval = g_list_prepend(retval, rsc);
	}
	}

	return retval;
	}

	GList *
	pe__build_node_name_list(pcmk_scheduler_t scheduler, const char s)
	{
	GList *nodes = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	/* Nothing was given so return a list of all node names. Or, '*' was
	* given. This would normally fall into the pe__unames_with_tag branch
	* where it will return an empty list. Catch it here instead.
	*/
	nodes = g_list_prepend(nodes, strdup("*"));
	} else {
	pcmk_node_t *node = pcmk_find_node(scheduler, s);

	if (node) {
	/* The given string was a valid uname for a node. Return a
	* singleton list containing just that uname.
	*/
	nodes = g_list_prepend(nodes, strdup(s));
	} else {
	/* The given string was not a valid uname. It's either a tag or
	* it's a typo or something. In the first case, we'll return a
	* list of all the unames of the nodes with the given tag. In the
	* second case, we'll return a NULL pointer and nothing will
	* get displayed.
	*/
	nodes = pe__unames_with_tag(scheduler, s);
	}
	}

	return nodes;
	}

	GList *
	pe__build_rsc_list(pcmk_scheduler_t scheduler, const char s)
	{
	GList *resources = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	resources = g_list_prepend(resources, strdup("*"));
	} else {
	const uint32_t flags = pcmk_rsc_match_history\|pcmk_rsc_match_basename;
	pcmk_resource_t *rsc = pe_find_resource_with_flags(scheduler->resources,
	s, flags);

	if (rsc) {
	/* A colon in the name we were given means we're being asked to filter
	* on a specific instance of a cloned resource. Put that exact string
	* into the filter list. Otherwise, use the printable ID of whatever
	* resource was found that matches what was asked for.
	*/
	if (strstr(s, ":") != NULL) {
	resources = g_list_prepend(resources, strdup(rsc->id));
	} else {
	resources = g_list_prepend(resources, strdup(rsc_printable_id(rsc)));
	}
	} else {
	/* The given string was not a valid resource name. It's a tag or a
	* typo or something. See pe__build_node_name_list() for more
	* detail.
	*/
	resources = pe__rscs_with_tag(scheduler, s);
	}
	}

	return resources;
	}

	xmlNode *
	pe__failed_probe_for_rsc(const pcmk_resource_t rsc, const char name)
	{
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
	const char *rsc_id = rsc->id;

	if (pcmk__is_clone(parent)) {
	rsc_id = pe__clone_child_id(parent);
	}

	for (xmlNode *xml_op = pcmk__xe_first_child(rsc->private->scheduler->failed,
	NULL, NULL, NULL);
	xml_op != NULL; xml_op = pcmk__xe_next(xml_op)) {

	const char *value = NULL;
	char *op_id = NULL;

	/* This resource operation is not a failed probe. */
	if (!pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	/* This resource operation was not run on the given node. Note that if name is
	* NULL, this will always succeed.
	*/
	value = crm_element_value(xml_op, PCMK__META_ON_NODE);
	if (value == NULL \|\| !pcmk__str_eq(value, name, pcmk__str_casei\|pcmk__str_null_matches)) {
	continue;
	}

	if (!parse_op_key(pcmk__xe_history_key(xml_op), &op_id, NULL, NULL)) {
	continue; // This history entry is missing an operation key
	}

	/* This resource operation's ID does not match the rsc_id we are looking for. */
	if (!pcmk__str_eq(op_id, rsc_id, pcmk__str_none)) {
	free(op_id);
	continue;
	}

	free(op_id);
	return xml_op;
	}

	return NULL;
	}
	diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
	index d2020e1a28..c35f4c2526 100644
	--- a/tools/crm_resource_runtime.c
	+++ b/tools/crm_resource_runtime.c
	@@ -1,2433 +1,2433 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <limits.h>
	#include <glib.h>
	#include <libxml/tree.h>

	#include <crm/common/ipc_attrd_internal.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/common/lists_internal.h>
	#include <crm/services_internal.h>

	#include <crm_resource.h>

	static GList *
	build_node_info_list(const pcmk_resource_t *rsc)
	{
	GList *retval = NULL;

	for (const GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;

	for (const GList *iter2 = child->private->active_nodes;
	iter2 != NULL; iter2 = iter2->next) {

	const pcmk_node_t node = (const pcmk_node_t ) iter2->data;
	node_info_t *ni = pcmk__assert_alloc(1, sizeof(node_info_t));

	ni->node_name = node->private->name;
	if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)
	&& (child->private->fns->state(child,
	TRUE) == pcmk_role_promoted)) {
	ni->promoted = true;
	}

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	GList *
	cli_resource_search(pcmk_resource_t rsc, const char requested_name,
	pcmk_scheduler_t *scheduler)
	{
	GList *retval = NULL;
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	if (pcmk__is_clone(rsc)) {
	retval = build_node_info_list(rsc);

	/* The anonymous clone children's common ID is supplied */
	} else if (pcmk__is_clone(parent)
	&& !pcmk_is_set(rsc->flags, pcmk__rsc_unique)
	&& (rsc->private->history_id != NULL)
	&& pcmk__str_eq(requested_name, rsc->private->history_id,
	pcmk__str_none)
	&& !pcmk__str_eq(requested_name, rsc->id, pcmk__str_none)) {

	retval = build_node_info_list(parent);

	} else {
	for (GList *iter = rsc->private->active_nodes;
	iter != NULL; iter = iter->next) {

	pcmk_node_t node = (pcmk_node_t ) iter->data;
	node_info_t *ni = pcmk__assert_alloc(1, sizeof(node_info_t));

	ni->node_name = node->private->name;
	if (rsc->private->fns->state(rsc, TRUE) == pcmk_role_promoted) {
	ni->promoted = true;
	}

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	// \return Standard Pacemaker return code
	static int
	find_resource_attr(pcmk__output_t out, cib_t the_cib, const char *attr,
	const char rsc, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, xmlNode **result)
	{
	xmlNode *xml_search;
	int rc = pcmk_rc_ok;
	GString *xpath = NULL;
	const char *xpath_base = NULL;

	if (result) {
	*result = NULL;
	}

	if(the_cib == NULL) {
	return ENOTCONN;
	}

	xpath_base = pcmk_cib_xpath_for(PCMK_XE_RESOURCES);
	if (xpath_base == NULL) {
	crm_err(PCMK_XE_RESOURCES " CIB element not known (bug?)");
	return ENOMSG;
	}

	xpath = g_string_sized_new(1024);
	pcmk__g_strcat(xpath,
	xpath_base, "//*[@" PCMK_XA_ID "=\"", rsc, "\"]", NULL);

	if (attr_set_type != NULL) {
	pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
	if (set_name != NULL) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "=\"", set_name, "\"]",
	NULL);
	}
	}

	g_string_append(xpath, "//" PCMK_XE_NVPAIR);

	if (attr_id != NULL && attr_name!= NULL) {
	pcmk__g_strcat(xpath,
	"[@" PCMK_XA_ID "='", attr_id, "' "
	"and @" PCMK_XA_NAME "='", attr_name, "']", NULL);

	} else if (attr_id != NULL) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", attr_id, "']", NULL);

	} else if (attr_name != NULL) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_NAME "='", attr_name, "']", NULL);
	}

	rc = the_cib->cmds->query(the_cib, xpath->str, &xml_search,
	cib_sync_call\|cib_xpath);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	crm_log_xml_debug(xml_search, "Match");
	if (xml_search->children != NULL) {
	rc = ENOTUNIQ;
	pcmk__warn_multiple_name_matches(out, xml_search, attr_name);
	out->spacer(out);
	}
	}

	if (result) {
	*result = xml_search;
	} else {
	pcmk__xml_free(xml_search);
	}

	g_string_free(xpath, TRUE);
	return rc;
	}

	/* PRIVATE. Use the find_matching_attr_resources instead. */
	static void
	find_matching_attr_resources_recursive(pcmk__output_t *out,
	GList /* <pcmk_resource_t> / **result,
	pcmk_resource_t rsc, const char attr_set,
	const char * attr_set_type, const char * attr_id,
	const char * attr_name, cib_t * cib, int depth)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = clone_strip(rsc->id);

	for (GList *gIter = rsc->private->children;
	gIter != NULL; gIter = gIter->next) {

	find_matching_attr_resources_recursive(out, result,
	(pcmk_resource_t *) gIter->data,
	attr_set, attr_set_type, attr_id,
	attr_name, cib, depth+1);
	/* do it only once for clones */
	if (pcmk__is_clone(rsc)) {
	break;
	}
	}

	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, NULL);
	/* Post-order traversal.
	* The root is always on the list and it is the last item. */
	if((0 == depth) \|\| (pcmk_rc_ok == rc)) {
	/* push the head */
	result = g_list_append(result, rsc);
	}

	free(lookup_id);
	}


	/* The result is a linearized pre-ordered tree of resources. */
	static GList/<pcmk_resource_t>/
	find_matching_attr_resources(pcmk__output_t out, pcmk_resource_t rsc,
	const char * rsc_id, const char * attr_set,
	const char * attr_set_type, const char * attr_id,
	const char * attr_name, cib_t * cib, const char * cmd,
	gboolean force)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = NULL;
	GList * result = NULL;

	/* If --force is used, update only the requested resource (clone or primitive).
	* Otherwise, if the primitive has the attribute, use that.
	* Otherwise use the clone. */
	if(force == TRUE) {
	return g_list_append(result, rsc);
	}
	if (pcmk__is_clone(rsc->private->parent)) {
	int rc = find_resource_attr(out, cib, PCMK_XA_ID, rsc_id, attr_set_type,
	attr_set, attr_id, attr_name, NULL);

	if(rc != pcmk_rc_ok) {
	rsc = rsc->private->parent;
	out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
	cmd, attr_name, rsc->id, rsc_id);
	}
	return g_list_append(result, rsc);

	} else if ((rsc->private->parent == NULL)
	&& (rsc->private->children != NULL) && pcmk__is_clone(rsc)) {

	pcmk_resource_t *child = rsc->private->children->data;

	if (pcmk__is_primitive(child)) {
	lookup_id = clone_strip(child->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id,
	attr_set_type, attr_set, attr_id, attr_name, NULL);

	if(rc == pcmk_rc_ok) {
	rsc = child;
	out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
	attr_name, lookup_id, cmd, rsc_id);
	}

	free(lookup_id);
	}
	return g_list_append(result, rsc);
	}
	/* If the resource is a group ==> children inherit the attribute if defined. */
	find_matching_attr_resources_recursive(out, &result, rsc, attr_set,
	attr_set_type, attr_id, attr_name,
	cib, 0);
	return result;
	}

	static int
	update_element_attribute(pcmk__output_t out, pcmk_resource_t rsc,
	cib_t cib, const char attr_name, const char *attr_value)
	{
	int rc = pcmk_rc_ok;

	if (cib == NULL) {
	return ENOTCONN;
	}

	crm_xml_add(rsc->private->xml, attr_name, attr_value);

	rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc->private->xml,
	cib_sync_call);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Set attribute: " PCMK_XA_NAME "=%s value=%s",
	attr_name, attr_value);
	}

	return rc;
	}

	static int
	resources_with_attr(pcmk__output_t out, cib_t cib, pcmk_resource_t *rsc,
	const char requested_name, const char attr_set,
	const char attr_set_type, const char attr_id,
	const char attr_name, const char top_id, gboolean force,
	GList **resources)
	{
	if (pcmk__str_eq(attr_set_type, PCMK_XE_INSTANCE_ATTRIBUTES,
	pcmk__str_casei)) {
	if (!force) {
	xmlNode *xml_search = NULL;
	int rc = pcmk_rc_ok;

	rc = find_resource_attr(out, cib, PCMK_XA_ID, top_id,
	PCMK_XE_META_ATTRIBUTES, attr_set, attr_id,
	attr_name, &xml_search);

	if (rc == pcmk_rc_ok \|\| rc == ENOTUNIQ) {
	char *found_attr_id = NULL;

	found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);

	if (!out->is_quiet(out)) {
	out->err(out,
	"WARNING: There is already a meta attribute "
	"for '%s' called '%s' (id=%s)",
	top_id, attr_name, found_attr_id);
	out->err(out,
	" Delete '%s' first or use the force option "
	"to override", found_attr_id);
	}

	free(found_attr_id);
	pcmk__xml_free(xml_search);
	return ENOTUNIQ;
	}

	pcmk__xml_free(xml_search);
	}

	resources = g_list_append(resources, rsc);

	} else {
	*resources = find_matching_attr_resources(out, rsc, requested_name,
	attr_set, attr_set_type,
	attr_id, attr_name, cib,
	"update", force);
	}

	/* If the user specified attr_set or attr_id, the intent is to modify a
	* single resource, which will be the last item in the list.
	*/
	if ((attr_set != NULL) \|\| (attr_id != NULL)) {
	GList last = g_list_last(resources);

	resources = g_list_remove_link(resources, last);
	g_list_free(*resources);
	*resources = last;
	}

	return pcmk_rc_ok;
	}

	static void
	free_attr_update_data(gpointer data)
	{
	attr_update_data_t *ud = data;

	if (ud == NULL) {
	return;
	}

	free(ud->attr_set_type);
	free(ud->attr_set_id);
	free(ud->attr_name);
	free(ud->attr_value);
	free(ud->given_rsc_id);
	free(ud->found_attr_id);
	free(ud);
	}

	static int
	update_attribute(pcmk_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	const char attr_value, gboolean recursive, cib_t cib,
	gboolean force, GList **results)
	{
	pcmk__output_t *out = rsc->private->scheduler->priv;
	int rc = pcmk_rc_ok;

	GList/<pcmk_resource_t>/ resources = NULL;
	const char *top_id = pe__const_top_resource(rsc, false)->id;

	if ((attr_id == NULL) && !force) {
	find_resource_attr(out, cib, PCMK_XA_ID, top_id, NULL, NULL, NULL,
	attr_name, NULL);
	}

	rc = resources_with_attr(out, cib, rsc, requested_name, attr_set, attr_set_type,
	attr_id, attr_name, top_id, force, &resources);

	if (rc != pcmk_rc_ok) {
	return rc;
	}

	for (GList *iter = resources; iter != NULL; iter = iter->next) {
	char *lookup_id = NULL;
	char *local_attr_set = NULL;
	char *found_attr_id = NULL;
	const char *rsc_attr_id = attr_id;
	const char *rsc_attr_set = attr_set;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_search = NULL;

	rsc = (pcmk_resource_t *) iter->data;

	lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &xml_search);

	switch (rc) {
	case pcmk_rc_ok:
	found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
	crm_debug("Found a match for " PCMK_XA_NAME "='%s': "
	PCMK_XA_ID "='%s'", attr_name, found_attr_id);
	rsc_attr_id = found_attr_id;
	break;

	case ENXIO:
	if (rsc_attr_set == NULL) {
	local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
	attr_set_type);
	rsc_attr_set = local_attr_set;
	}
	if (rsc_attr_id == NULL) {
	found_attr_id = crm_strdup_printf("%s-%s",
	rsc_attr_set, attr_name);
	rsc_attr_id = found_attr_id;
	}

	xml_top = pcmk__xe_create(NULL,
	(const char *)
	rsc->private->xml->name);
	crm_xml_add(xml_top, PCMK_XA_ID, lookup_id);

	xml_obj = pcmk__xe_create(xml_top, attr_set_type);
	crm_xml_add(xml_obj, PCMK_XA_ID, rsc_attr_set);
	break;

	default:
	free(lookup_id);
	free(found_attr_id);
	pcmk__xml_free(xml_search);
	g_list_free(resources);
	return rc;
	}

	xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
	attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, PCMK_XE_RESOURCES, xml_top, cib_sync_call);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	attr_update_data_t *ud = pcmk__assert_alloc(1, sizeof(attr_update_data_t));

	if (attr_set_type == NULL) {
	attr_set_type = (const char *) xml_search->parent->name;
	}

	if (rsc_attr_set == NULL) {
	rsc_attr_set = crm_element_value(xml_search->parent, PCMK_XA_ID);
	}

	ud->attr_set_type = pcmk__str_copy(attr_set_type);
	ud->attr_set_id = pcmk__str_copy(rsc_attr_set);
	ud->attr_name = pcmk__str_copy(attr_name);
	ud->attr_value = pcmk__str_copy(attr_value);
	ud->given_rsc_id = pcmk__str_copy(lookup_id);
	ud->found_attr_id = pcmk__str_copy(found_attr_id);
	ud->rsc = rsc;

	results = g_list_append(results, ud);
	}

	pcmk__xml_free(xml_top);
	pcmk__xml_free(xml_search);

	free(lookup_id);
	free(found_attr_id);
	free(local_attr_set);

	if (recursive
	&& pcmk__str_eq(attr_set_type, PCMK_XE_META_ATTRIBUTES,
	pcmk__str_casei)) {
	/* We want to set the attribute only on resources explicitly
	* colocated with this one, so we use
	* rsc->private->with_this_colocations directly rather than the
	* with_this_colocations() method.
	*/
	pcmk__set_rsc_flags(rsc, pcmk__rsc_detect_loop);
	for (GList *lpc = rsc->private->with_this_colocations;
	lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	crm_debug("Checking %s %d", cons->id, cons->score);

	if (pcmk_is_set(cons->dependent->flags, pcmk__rsc_detect_loop)
	\|\| (cons->score <= 0)) {
	continue;
	}

	crm_debug("Setting %s=%s for dependent resource %s",
	attr_name, attr_value, cons->dependent->id);
	update_attribute(cons->dependent, cons->dependent->id, NULL,
	attr_set_type, NULL, attr_name, attr_value,
	recursive, cib, force, results);
	}
	}
	}

	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_update_attribute(pcmk_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	const char *attr_value, gboolean recursive,
	cib_t *cib, gboolean force)
	{
	static bool need_init = true;
	int rc = pcmk_rc_ok;

	GList *results = NULL;
	pcmk__output_t *out = rsc->private->scheduler->priv;

	/* If we were asked to update the attribute in a resource element (for
	* instance, <primitive class="ocf">) there's really not much we need to do.
	*/
	if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
	return update_element_attribute(out, rsc, cib, attr_name, attr_value);
	}

	/* One time initialization - clear flags so we can detect loops */
	if (need_init) {
	need_init = false;
	pcmk__unpack_constraints(rsc->private->scheduler);
	pe__clear_resource_flags_on_all(rsc->private->scheduler,
	pcmk__rsc_detect_loop);
	}

	rc = update_attribute(rsc, requested_name, attr_set, attr_set_type,
	attr_id, attr_name, attr_value, recursive, cib,
	force, &results);

	if (rc == pcmk_rc_ok) {
	if (results == NULL) {
	return rc;
	}

	out->message(out, "attribute-changed-list", results);
	g_list_free_full(results, free_attr_update_data);
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete_attribute(pcmk_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	cib_t *cib, int cib_options, gboolean force)
	{
	pcmk__output_t *out = rsc->private->scheduler->priv;
	int rc = pcmk_rc_ok;
	GList/<pcmk_resource_t>/ resources = NULL;

	if ((attr_id == NULL) && !force) {
	find_resource_attr(out, cib, PCMK_XA_ID,
	pe__const_top_resource(rsc, false)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, PCMK_XE_META_ATTRIBUTES, pcmk__str_casei)) {
	resources = find_matching_attr_resources(out, rsc, requested_name,
	attr_set, attr_set_type,
	attr_id, attr_name, cib,
	"delete", force);

	} else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
	pcmk__xe_remove_attr(rsc->private->xml, attr_name);
	CRM_ASSERT(cib != NULL);
	rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc->private->xml,
	cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted attribute: %s", attr_name);
	}
	return rc;

	} else {
	resources = g_list_append(resources, rsc);
	}

	for (GList *iter = resources; iter != NULL; iter = iter->next) {
	char *lookup_id = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_search = NULL;
	char *found_attr_id = NULL;
	const char *rsc_attr_id = attr_id;

	rsc = (pcmk_resource_t *) iter->data;

	lookup_id = clone_strip(rsc->id);
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &xml_search);
	switch (rc) {
	case pcmk_rc_ok:
	found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
	pcmk__xml_free(xml_search);
	break;

	case ENXIO:
	free(lookup_id);
	pcmk__xml_free(xml_search);
	continue;

	default:
	free(lookup_id);
	pcmk__xml_free(xml_search);
	g_list_free(resources);
	return rc;
	}

	if (rsc_attr_id == NULL) {
	rsc_attr_id = found_attr_id;
	}

	xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
	crm_log_xml_debug(xml_obj, "Delete");

	CRM_ASSERT(cib);
	rc = cib->cmds->remove(cib, PCMK_XE_RESOURCES, xml_obj, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted '%s' option: " PCMK_XA_ID "=%s%s%s%s%s",
	lookup_id, found_attr_id,
	((attr_set == NULL)? "" : " set="),
	pcmk__s(attr_set, ""),
	((attr_name == NULL)? "" : " " PCMK_XA_NAME "="),
	pcmk__s(attr_name, ""));
	}

	free(lookup_id);
	pcmk__xml_free(xml_obj);
	free(found_attr_id);
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
	const char host_uname, const char rsc_id,
	pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	const char *router_node = host_uname;
	const char *rsc_api_id = NULL;
	const char *rsc_long_id = NULL;
	const char *rsc_class = NULL;
	const char *rsc_provider = NULL;
	const char *rsc_type = NULL;
	bool cib_only = false;
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	out->err(out, "Resource %s not found", rsc_id);
	return ENXIO;

	} else if (!pcmk__is_primitive(rsc)) {
	out->err(out, "We can only process primitive resources, not %s", rsc_id);
	return EINVAL;
	}

	rsc_class = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);
	rsc_provider = crm_element_value(rsc->private->xml, PCMK_XA_PROVIDER);
	rsc_type = crm_element_value(rsc->private->xml, PCMK_XA_TYPE);
	if ((rsc_class == NULL) \|\| (rsc_type == NULL)) {
	out->err(out, "Resource %s does not have a class and type", rsc_id);
	return EINVAL;
	}

	{
	pcmk_node_t *node = pcmk_find_node(scheduler, host_uname);

	if (node == NULL) {
	out->err(out, "Node %s not found", host_uname);
	return pcmk_rc_node_unknown;
	}

	if (!(node->details->online)) {
	if (do_fail_resource) {
	out->err(out, "Node %s is not online", host_uname);
	return ENOTCONN;
	} else {
	cib_only = true;
	}
	}
	if (!cib_only && pcmk__is_pacemaker_remote_node(node)) {
	node = pcmk__current_node(node->private->remote);
	if (node == NULL) {
	out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
	host_uname);
	return ENOTCONN;
	}
	router_node = node->private->name;
	}
	}

	if (rsc->private->history_id != NULL) {
	rsc_api_id = rsc->private->history_id;
	rsc_long_id = rsc->id;
	} else {
	rsc_api_id = rsc->id;
	}
	if (do_fail_resource) {
	return pcmk_controld_api_fail(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id,
	rsc_class, rsc_provider, rsc_type);
	} else {
	return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id, rsc_class,
	rsc_provider, rsc_type, cib_only);
	}
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(const pcmk_resource_t *rsc)
	{
	const char *name = pcmk__s(rsc->private->history_id, rsc->id);

	if (pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
	return strdup(name);
	}
	return clone_strip(name);
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_history(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pcmk_scheduler_t scheduler)
	{
	int rc = pcmk_rc_ok;

	/* Erase the resource's entire LRM history in the CIB, even if we're only
	* clearing a single operation's fail count. If we erased only entries for a
	* single operation, we might wind up with a wrong idea of the current
	* resource state, and we might not re-probe the resource.
	*/
	rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, scheduler);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_trace("Processing %d mainloop inputs",
	pcmk_controld_api_replies_expected(controld_api));
	while (g_main_context_iteration(NULL, FALSE)) {
	crm_trace("Processed mainloop input, %d still remaining",
	pcmk_controld_api_replies_expected(controld_api));
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_failures(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char node_name, const char rsc_id, const char *operation,
	const char interval_spec, pcmk_scheduler_t scheduler)
	{
	int rc = pcmk_rc_ok;
	const char *failed_value = NULL;
	const char *failed_id = NULL;
	char *interval_ms_s = NULL;
	GHashTable *rscs = NULL;
	GHashTableIter iter;

	/* Create a hash table to use as a set of resources to clean. This lets us
	* clean each resource only once (per node) regardless of how many failed
	* operations it has.
	*/
	rscs = pcmk__strkey_table(NULL, NULL);

	// Normalize interval to milliseconds for comparison to history entry
	if (operation) {
	guint interval_ms = 0U;

	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);
	}

	for (xmlNode *xml_op = pcmk__xe_first_child(scheduler->failed, NULL, NULL,
	NULL);
	xml_op != NULL; xml_op = pcmk__xe_next(xml_op)) {

	failed_id = crm_element_value(xml_op, PCMK__XA_RSC_ID);
	if (failed_id == NULL) {
	// Malformed history entry, should never happen
	continue;
	}

	// No resource specified means all resources match
	if (rsc_id) {
	pcmk_resource_t *fail_rsc = NULL;

	fail_rsc = pe_find_resource_with_flags(scheduler->resources,
	failed_id,
	pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename);
	if (!fail_rsc \|\| !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
	continue;
	}
	}

	// Host name should always have been provided by this point
	failed_value = crm_element_value(xml_op, PCMK_XA_UNAME);
	if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
	continue;
	}

	// No operation specified means all operations match
	if (operation) {
	failed_value = crm_element_value(xml_op, PCMK_XA_OPERATION);
	if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
	continue;
	}

	// Interval (if operation was specified) defaults to 0 (not all)
	failed_value = crm_element_value(xml_op, PCMK_META_INTERVAL);
	if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
	continue;
	}
	}

	g_hash_table_add(rscs, (gpointer) failed_id);
	}

	free(interval_ms_s);

	g_hash_table_iter_init(&iter, rscs);
	while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
	crm_debug("Erasing failures of %s on %s", failed_id, node_name);
	rc = clear_rsc_history(controld_api, node_name, failed_id, scheduler);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	g_hash_table_destroy(rscs);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_fail_attrs(const pcmk_resource_t rsc, const char operation,
	const char interval_spec, const pcmk_node_t node)
	{
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	char *rsc_name = rsc_fail_name(rsc);

	if (pcmk__is_pacemaker_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}

	rc = pcmk__attrd_api_clear_failures(NULL, node->private->name, rsc_name,
	operation, interval_spec, NULL,
	attr_options);
	free(rsc_name);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete(pcmk_ipc_api_t controld_api, const char host_uname,
	const pcmk_resource_t rsc, const char operation,
	const char *interval_spec, bool just_failures,
	pcmk_scheduler_t *scheduler, gboolean force)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	pcmk_node_t *node = NULL;

	if (rsc == NULL) {
	return ENXIO;

	} else if (rsc->private->children != NULL) {

	for (const GList *lpc = rsc->private->children;
	lpc != NULL; lpc = lpc->next) {

	const pcmk_resource_t child = (const pcmk_resource_t ) lpc->data;

	rc = cli_resource_delete(controld_api, host_uname, child, operation,
	interval_spec, just_failures, scheduler,
	force);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	return pcmk_rc_ok;

	} else if (host_uname == NULL) {
	GList *lpc = NULL;
	GList *nodes = g_hash_table_get_values(rsc->private->probed_nodes);

	if(nodes == NULL && force) {
	nodes = pcmk__copy_node_list(scheduler->nodes, false);

	} else if ((nodes == NULL)
	&& pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)) {
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
	- if(node->weight >= 0) {
	+ if (node->assign->score >= 0) {
	nodes = g_list_prepend(nodes, node);
	}
	}

	} else if(nodes == NULL) {
	nodes = g_hash_table_get_values(rsc->private->allowed_nodes);
	}

	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node = (pcmk_node_t *) lpc->data;

	if (node->details->online) {
	rc = cli_resource_delete(controld_api, node->private->name, rsc,
	operation, interval_spec, just_failures,
	scheduler, force);
	}
	if (rc != pcmk_rc_ok) {
	g_list_free(nodes);
	return rc;
	}
	}

	g_list_free(nodes);
	return pcmk_rc_ok;
	}

	node = pcmk_find_node(scheduler, host_uname);

	if (node == NULL) {
	out->err(out, "Unable to clean up %s because node %s not found",
	rsc->id, host_uname);
	return ENODEV;
	}

	if (!pcmk_is_set(node->private->flags, pcmk__node_probes_allowed)) {
	out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
	rsc->id, host_uname);
	return EOPNOTSUPP;
	}

	if (controld_api == NULL) {
	out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
	rsc->id, host_uname);
	return pcmk_rc_ok;
	}

	rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up %s failures on %s: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	return rc;
	}

	if (just_failures) {
	rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
	interval_spec, scheduler);
	} else {
	rc = clear_rsc_history(controld_api, host_uname, rsc->id, scheduler);
	}
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	} else {
	out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_cleanup_all(pcmk_ipc_api_t controld_api, const char node_name,
	const char operation, const char interval_spec,
	pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	const char *display_name = node_name? node_name : "all nodes";

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	display_name);
	return rc;
	}

	if (node_name) {
	pcmk_node_t *node = pcmk_find_node(scheduler, node_name);

	if (node == NULL) {
	out->err(out, "Unknown node: %s", node_name);
	return ENXIO;
	}
	if (pcmk__is_pacemaker_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
	interval_spec, NULL, attr_options);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up all failures on %s: %s",
	display_name, pcmk_rc_str(rc));
	return rc;
	}

	if (node_name) {
	rc = clear_rsc_failures(out, controld_api, node_name, NULL,
	operation, interval_spec, scheduler);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
	node_name, pcmk_rc_str(rc));
	return rc;
	}
	} else {
	for (GList *iter = scheduler->nodes; iter; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	rc = clear_rsc_failures(out, controld_api, node->private->name,
	NULL, operation, interval_spec, scheduler);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
	pcmk_rc_str(rc));
	return rc;
	}
	}
	}

	out->info(out, "Cleaned up all resources on %s", display_name);
	return rc;
	}

	static void
	check_role(resource_checks_t *checks)
	{
	const char *role_s = g_hash_table_lookup(checks->rsc->private->meta,
	PCMK_META_TARGET_ROLE);

	if (role_s == NULL) {
	return;
	}
	switch (pcmk_parse_role(role_s)) {
	case pcmk_role_stopped:
	checks->flags \|= rsc_remain_stopped;
	break;

	case pcmk_role_unpromoted:
	if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
	pcmk__rsc_promotable)) {
	checks->flags \|= rsc_unpromotable;
	}
	break;

	default:
	break;
	}
	}

	static void
	check_managed(resource_checks_t *checks)
	{
	const char *managed_s = g_hash_table_lookup(checks->rsc->private->meta,
	PCMK_META_IS_MANAGED);

	if ((managed_s != NULL) && !crm_is_true(managed_s)) {
	checks->flags \|= rsc_unmanaged;
	}
	}

	static void
	check_locked(resource_checks_t *checks)
	{
	const pcmk_node_t *lock_node = checks->rsc->private->lock_node;

	if (lock_node != NULL) {
	checks->flags \|= rsc_locked;
	checks->lock_node = lock_node->private->name;
	}
	}

	static bool
	node_is_unhealthy(pcmk_node_t *node)
	{
	switch (pe__health_strategy(node->private->scheduler)) {
	case pcmk__health_strategy_none:
	break;

	case pcmk__health_strategy_no_red:
	if (pe__node_health(node) < 0) {
	return true;
	}
	break;

	case pcmk__health_strategy_only_green:
	if (pe__node_health(node) <= 0) {
	return true;
	}
	break;

	case pcmk__health_strategy_progressive:
	case pcmk__health_strategy_custom:
	/* @TODO These are finite scores, possibly with rules, and possibly
	* combining with other scores, so attributing these as a cause is
	* nontrivial.
	*/
	break;
	}
	return false;
	}

	static void
	check_node_health(resource_checks_t checks, pcmk_node_t node)
	{
	if (node == NULL) {
	GHashTableIter iter;
	bool allowed = false;
	bool all_nodes_unhealthy = true;

	g_hash_table_iter_init(&iter, checks->rsc->private->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	allowed = true;
	if (!node_is_unhealthy(node)) {
	all_nodes_unhealthy = false;
	break;
	}
	}
	if (allowed && all_nodes_unhealthy) {
	checks->flags \|= rsc_node_health;
	}

	} else if (node_is_unhealthy(node)) {
	checks->flags \|= rsc_node_health;
	}
	}

	int
	cli_resource_check(pcmk__output_t out, pcmk_resource_t rsc, pcmk_node_t *node)
	{
	resource_checks_t checks = { .rsc = rsc };

	check_role(&checks);
	check_managed(&checks);
	check_locked(&checks);
	check_node_health(&checks, node);

	return out->message(out, "resource-check-list", &checks);
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_fail(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pcmk_scheduler_t scheduler)
	{
	crm_notice("Failing %s on %s", rsc_id, host_uname);
	return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, scheduler);
	}

	static GHashTable *
	generate_resource_params(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	combined = pcmk__strkey_table(free, free);

	params = pe_rsc_params(rsc, node, scheduler);
	if (params != NULL) {
	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	pcmk__insert_dup(combined, key, value);
	}
	}

	meta = pcmk__strkey_table(free, free);
	get_meta_attributes(meta, rsc, NULL, scheduler);
	if (meta != NULL) {
	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}

	bool resource_is_running_on(pcmk_resource_t rsc, const char host)
	{
	bool found = true;
	GList *hIter = NULL;
	GList *hosts = NULL;

	if (rsc == NULL) {
	return false;
	}

	rsc->private->fns->location(rsc, &hosts, TRUE);
	for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
	pcmk_node_t node = (pcmk_node_t ) hIter->data;

	if (pcmk__strcase_any_of(host, node->private->name, node->private->id,
	NULL)) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	}
	}

	if (host != NULL) {
	crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
	found = false;

	} else if(host == NULL && hosts == NULL) {
	crm_trace("Resource %s is not running\n", rsc->id);
	found = false;
	}

	done:
	g_list_free(hosts);
	return found;
	}

	/*!
	* \internal
	* \brief Create a list of all resources active on host from a given list
	*
	* \param[in] host Name of host to check whether resources are active
	* \param[in] rsc_list List of resources to check
	*
	* \return New list of resources from list that are active on host
	*/
	static GList *
	get_active_resources(const char host, GList rsc_list)
	{
	GList *rIter = NULL;
	GList *active = NULL;

	for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rIter->data;

	/* Expand groups to their members, because if we're restarting a member
	* other than the first, we can't otherwise tell which resources are
	* stopping and starting.
	*/
	if (pcmk__is_group(rsc)) {
	GList *member_active = NULL;

	member_active = get_active_resources(host, rsc->private->children);
	active = g_list_concat(active, member_active);
	} else if (resource_is_running_on(rsc, host)) {
	active = g_list_append(active, strdup(rsc->id));
	}
	}
	return active;
	}

	static void dump_list(GList items, const char tag)
	{
	int lpc = 0;
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
	lpc++;
	}
	}

	static void display_list(pcmk__output_t out, GList items, const char *tag)
	{
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	out->info(out, "%s%s", tag, (const char *)item->data);
	}
	}

	/*!
	* \internal
	* \brief Upgrade XML to latest schema version and use it as scheduler input
	*
	* This also updates the scheduler timestamp to the current time.
	*
	* \param[in,out] scheduler Scheduler data to update
	* \param[in,out] xml XML to use as input
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* scheduler->now.
	*/
	int
	update_scheduler_input(pcmk_scheduler_t scheduler, xmlNode *xml)
	{
	int rc = pcmk_update_configured_schema(xml, false);

	if (rc == pcmk_rc_ok) {
	scheduler->input = *xml;
	scheduler->now = crm_time_new(NULL);
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Update scheduler XML input based on a CIB query
	*
	* \param[in] scheduler Scheduler data to initialize
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* scheduler->input and scheduler->now.
	*/
	static int
	update_scheduler_input_to_cib(pcmk__output_t out, pcmk_scheduler_t scheduler,
	cib_t *cib)
	{
	xmlNode *cib_xml_copy = NULL;
	int rc = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_rc_str(rc), rc);
	return rc;
	}
	rc = update_scheduler_input(scheduler, &cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not upgrade the current CIB XML");
	pcmk__xml_free(cib_xml_copy);
	return rc;
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	update_dataset(cib_t cib, pcmk_scheduler_t scheduler, bool simulate)
	{
	char *pid = NULL;
	char *shadow_file = NULL;
	cib_t *shadow_cib = NULL;
	int rc = pcmk_rc_ok;

	pcmk__output_t *out = scheduler->priv;

	pe_reset_working_set(scheduler);
	pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);
	rc = update_scheduler_input_to_cib(out, scheduler, cib);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if(simulate) {
	bool prev_quiet = false;

	pid = pcmk__getpid_s();
	shadow_cib = cib_shadow_new(pid);
	shadow_file = get_shadow_file(pid);

	if (shadow_cib == NULL) {
	out->err(out, "Could not create shadow cib: '%s'", pid);
	rc = ENXIO;
	goto done;
	}

	rc = pcmk__xml_write_file(scheduler->input, shadow_file, false, NULL);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not populate shadow cib: %s", pcmk_rc_str(rc));
	goto done;
	}

	rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not connect to shadow cib: %s",
	pcmk_rc_str(rc));
	goto done;
	}

	pcmk__schedule_actions(scheduler->input,
	pcmk_sched_no_counts\|pcmk_sched_no_compat,
	scheduler);

	prev_quiet = out->is_quiet(out);
	out->quiet = true;
	pcmk__simulate_transition(scheduler, shadow_cib, NULL);
	out->quiet = prev_quiet;

	rc = update_dataset(shadow_cib, scheduler, false);

	} else {
	cluster_status(scheduler);
	}

	done:
	// Do not free scheduler->input because rsc->private->xml must remain valid
	cib_delete(shadow_cib);
	free(pid);

	if(shadow_file) {
	unlink(shadow_file);
	free(shadow_file);
	}

	return rc;
	}

	/*!
	* \internal
	* \brief Find the maximum stop timeout of a resource and its children (if any)
	*
	* \param[in,out] rsc Resource to get timeout for
	*
	* \return Maximum stop timeout for \p rsc (in milliseconds)
	*/
	static guint
	max_rsc_stop_timeout(pcmk_resource_t *rsc)
	{
	long long result_ll;
	guint max_delay = 0;
	xmlNode *config = NULL;
	GHashTable *meta = NULL;

	if (rsc == NULL) {
	return 0;
	}

	// If resource is collective, use maximum of its children's stop timeouts
	if (rsc->private->children != NULL) {

	for (GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t *child = iter->data;
	guint delay = max_rsc_stop_timeout(child);

	if (delay > max_delay) {
	pcmk__rsc_trace(rsc,
	"Maximum stop timeout for %s is now %s "
	"due to %s", rsc->id,
	pcmk__readable_interval(delay), child->id);
	max_delay = delay;
	}
	}
	return max_delay;
	}

	// Get resource's stop action configuration from CIB
	config = pcmk__find_action_config(rsc, PCMK_ACTION_STOP, 0, true);

	/* Get configured timeout for stop action (fully evaluated for rules,
	* defaults, etc.).
	*
	* @TODO This currently ignores node (which might matter for rules)
	*/
	meta = pcmk__unpack_action_meta(rsc, NULL, PCMK_ACTION_STOP, 0, config);
	if ((pcmk__scan_ll(g_hash_table_lookup(meta, PCMK_META_TIMEOUT),
	&result_ll, -1LL) == pcmk_rc_ok) && (result_ll >= 0)) {
	max_delay = (guint) QB_MIN(result_ll, UINT_MAX);
	}
	g_hash_table_destroy(meta);

	return max_delay;
	}

	/*!
	* \internal
	* \brief Find a reasonable waiting time for stopping any one resource in a list
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] resources List of names of resources that will be stopped
	*
	* \return Rough estimate of a reasonable time to wait (in seconds) to stop any
	* one resource in \p resources
	* \note This estimate is very rough, simply the maximum stop timeout of all
	* given resources and their children, plus a small fudge factor. It does
	* not account for children that must be stopped in sequence, action
	* throttling, or any demotions needed. It checks the stop timeout, even
	* if the resources in question are actually being started.
	*/
	static guint
	wait_time_estimate(pcmk_scheduler_t scheduler, const GList resources)
	{
	guint max_delay = 0U;

	// Find maximum stop timeout in milliseconds
	for (const GList *item = resources; item != NULL; item = item->next) {
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources,
	(const char *) item->data);
	guint delay = max_rsc_stop_timeout(rsc);

	if (delay > max_delay) {
	pcmk__rsc_trace(rsc,
	"Wait time is now %s due to %s",
	pcmk__readable_interval(delay), rsc->id);
	max_delay = delay;
	}
	}

	return (max_delay / 1000U) + 5U;
	}

	#define waiting_for_starts(d, r, h) ((d != NULL) \|\| \
	(!resource_is_running_on((r), (h))))

	/*!
	* \internal
	* \brief Restart a resource (on a particular host if requested).
	*
	* \param[in,out] out Output object
	* \param[in,out] rsc The resource to restart
	* \param[in] node Node to restart resource on (NULL for all)
	* \param[in] move_lifetime If not NULL, how long constraint should
	* remain in effect (as ISO 8601 string)
	* \param[in] timeout_ms Consider failed if actions do not complete
	* in this time (specified in milliseconds,
	* but a two-second granularity is actually
	* used; if 0, it will be calculated based on
	* the resource timeout)
	* \param[in,out] cib Connection to the CIB manager
	* \param[in] cib_options Group of enum cib_call_options flags to
	* use with CIB calls
	* \param[in] promoted_role_only If true, limit to promoted instances
	* \param[in] force If true, apply only to requested instance
	* if part of a collective resource
	*
	* \return Standard Pacemaker return code (exits on certain failures)
	*/
	int
	cli_resource_restart(pcmk__output_t out, pcmk_resource_t rsc,
	const pcmk_node_t node, const char move_lifetime,
	guint timeout_ms, cib_t *cib, int cib_options,
	gboolean promoted_role_only, gboolean force)
	{
	int rc = pcmk_rc_ok;
	int lpc = 0;
	int before = 0;
	guint step_timeout_s = 0;
	guint sleep_interval = 2U;
	guint timeout = timeout_ms / 1000U;

	bool stop_via_ban = false;
	char *rsc_id = NULL;
	char *lookup_id = NULL;
	char *orig_target_role = NULL;

	GList *list_delta = NULL;
	GList *target_active = NULL;
	GList *current_active = NULL;
	GList *restart_target_active = NULL;

	pcmk_scheduler_t *scheduler = NULL;
	pcmk_resource_t *parent = uber_parent(rsc);

	bool running = false;
	const char *id = pcmk__s(rsc->private->history_id, rsc->id);
	const char *host = node ? node->private->name : NULL;

	/* If the implicit resource or primitive resource of a bundle is given, operate on the
	* bundle itself instead.
	*/
	if (pcmk__is_bundled(rsc)) {
	rsc = parent->private->parent;
	}

	running = resource_is_running_on(rsc, host);

	if (pcmk__is_clone(parent) && !running) {
	if (pcmk__is_unique_clone(parent)) {
	lookup_id = strdup(rsc->id);
	} else {
	lookup_id = clone_strip(rsc->id);
	}

	rsc = parent->private->fns->find_rsc(parent, lookup_id, node,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node);
	free(lookup_id);
	running = resource_is_running_on(rsc, host);
	}

	if (!running) {
	if (host) {
	out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
	} else {
	out->err(out, "%s is not running anywhere and so cannot be restarted", id);
	}
	return ENXIO;
	}

	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	out->err(out, "Unmanaged resources cannot be restarted.");
	return EAGAIN;
	}

	rsc_id = strdup(rsc->id);

	if (pcmk__is_unique_clone(parent)) {
	lookup_id = strdup(rsc->id);
	} else {
	lookup_id = clone_strip(rsc->id);
	}

	if (host) {
	if (pcmk__is_clone(rsc) \|\| pe_bundle_replicas(rsc)) {
	stop_via_ban = true;
	} else if (pcmk__is_clone(parent)) {
	stop_via_ban = true;
	free(lookup_id);
	lookup_id = strdup(parent->id);
	}
	}

	/*
	grab full cib
	determine originally active resources
	disable or ban
	poll cib and watch for affected resources to get stopped
	without --timeout, calculate the stop timeout for each step and wait for that
	if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
	if everything stopped, re-enable or un-ban
	poll cib and watch for affected resources to get started
	without --timeout, calculate the start timeout for each step and wait for that
	if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
	report success

	Optimizations:
	- use constraints to determine ordered list of affected resources
	- Allow a --no-deps option (aka. --force-restart)
	*/

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	rc = errno;
	out->err(out, "Could not allocate scheduler data: %s", pcmk_rc_str(rc));
	goto done;
	}

	scheduler->priv = out;
	rc = update_dataset(cib, scheduler, false);

	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not get new resource list: %s (%d)", pcmk_rc_str(rc), rc);
	goto done;
	}

	restart_target_active = get_active_resources(host, scheduler->resources);
	current_active = get_active_resources(host, scheduler->resources);

	dump_list(current_active, "Origin");

	if (stop_via_ban) {
	/* Stop the clone or bundle instance by banning it from the host */
	out->quiet = true;
	rc = cli_resource_ban(out, lookup_id, host, move_lifetime, cib,
	cib_options, promoted_role_only,
	PCMK_ROLE_PROMOTED);
	} else {
	xmlNode *xml_search = NULL;

	/* Stop the resource by setting PCMK_META_TARGET_ROLE to Stopped.
	* Remember any existing PCMK_META_TARGET_ROLE so we can restore it
	* later (though it only makes any difference if it's Unpromoted).
	*/

	rc = find_resource_attr(out, cib, PCMK_XA_VALUE, lookup_id, NULL, NULL, NULL,
	PCMK_META_TARGET_ROLE, &xml_search);

	if (rc == pcmk_rc_ok) {
	orig_target_role = crm_element_value_copy(xml_search, PCMK_XA_VALUE);
	}

	pcmk__xml_free(xml_search);

	rc = cli_resource_update_attribute(rsc, rsc_id, NULL,
	PCMK_XE_META_ATTRIBUTES, NULL,
	PCMK_META_TARGET_ROLE,
	PCMK_ACTION_STOPPED, FALSE, cib,
	force);
	}
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not set " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
	rsc_id, pcmk_rc_str(rc), rc);
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	current_active = NULL;
	}
	if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	restart_target_active = NULL;
	}
	goto done;
	}

	rc = update_dataset(cib, scheduler, true);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources would be stopped");
	goto failure;
	}

	target_active = get_active_resources(host, scheduler->resources);
	dump_list(target_active, "Target");

	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (list_delta != NULL) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = wait_time_estimate(scheduler, list_delta)
	/ sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%us remaining", timeout);
	}
	rc = update_dataset(cib, scheduler, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were stopped");
	goto failure;
	}

	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(host, scheduler->resources);

	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);

	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
	if(before == g_list_length(list_delta)) {
	/* aborted during stop phase, print the contents of list_delta */
	out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	if (stop_via_ban) {
	rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);

	} else if (orig_target_role) {
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL,
	PCMK_XE_META_ATTRIBUTES, NULL,
	PCMK_META_TARGET_ROLE,
	orig_target_role, FALSE, cib, force);
	free(orig_target_role);
	orig_target_role = NULL;
	} else {
	rc = cli_resource_delete_attribute(rsc, rsc_id, NULL,
	PCMK_XE_META_ATTRIBUTES, NULL,
	PCMK_META_TARGET_ROLE, cib,
	cib_options, force);
	}

	if(rc != pcmk_rc_ok) {
	out->err(out,
	"Could not unset " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
	rsc_id, pcmk_rc_str(rc), rc);
	goto done;
	}

	if (target_active != NULL) {
	g_list_free_full(target_active, free);
	}
	target_active = restart_target_active;

	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (waiting_for_starts(list_delta, rsc, host)) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = wait_time_estimate(scheduler, list_delta)
	/ sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {

	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}

	rc = update_dataset(cib, scheduler, false);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were started");
	goto failure;
	}

	/* It's OK if dependent resources moved to a different node,
	* so we check active resources on all nodes.
	*/
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(NULL, scheduler->resources);

	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	if(before == g_list_length(list_delta)) {
	/* aborted during start phase, print the contents of list_delta */
	out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	rc = pcmk_rc_ok;
	goto done;

	failure:
	if (stop_via_ban) {
	cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
	} else if (orig_target_role) {
	cli_resource_update_attribute(rsc, rsc_id, NULL,
	PCMK_XE_META_ATTRIBUTES, NULL,
	PCMK_META_TARGET_ROLE, orig_target_role,
	FALSE, cib, force);
	free(orig_target_role);
	} else {
	cli_resource_delete_attribute(rsc, rsc_id, NULL,
	PCMK_XE_META_ATTRIBUTES, NULL,
	PCMK_META_TARGET_ROLE, cib, cib_options,
	force);
	}

	done:
	if (list_delta != NULL) {
	g_list_free(list_delta);
	}
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	if (target_active != NULL && (target_active != restart_target_active)) {
	g_list_free_full(target_active, free);
	}
	if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	}
	free(rsc_id);
	free(lookup_id);
	pe_free_working_set(scheduler);
	return rc;
	}

	static inline bool
	action_is_pending(const pcmk_action_t *action)
	{
	if (pcmk_any_flags_set(action->flags,
	pcmk_action_optional\|pcmk_action_pseudo)
	\|\| !pcmk_is_set(action->flags, pcmk_action_runnable)
	\|\| pcmk__str_eq(PCMK_ACTION_NOTIFY, action->task, pcmk__str_casei)) {
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether any actions in a list are pending
	*
	* \param[in] actions List of actions to check
	*
	* \return true if any actions in the list are pending, otherwise false
	*/
	static bool
	actions_are_pending(const GList *actions)
	{
	for (const GList *action = actions; action != NULL; action = action->next) {
	const pcmk_action_t a = (const pcmk_action_t ) action->data;

	if (action_is_pending(a)) {
	crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
	return true;
	}
	}
	return false;
	}

	static void
	print_pending_actions(pcmk__output_t out, GList actions)
	{
	GList *action;

	out->info(out, "Pending actions:");
	for (action = actions; action != NULL; action = action->next) {
	pcmk_action_t a = (pcmk_action_t ) action->data;

	if (!action_is_pending(a)) {
	continue;
	}

	if (a->node) {
	out->info(out, "\tAction %d: %s\ton %s",
	a->id, a->uuid, pcmk__node_name(a->node));
	} else {
	out->info(out, "\tAction %d: %s", a->id, a->uuid);
	}
	}
	}

	/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
	#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)

	/* For --wait, how long to sleep between cluster state checks */
	#define WAIT_SLEEP_S (2)

	/*!
	* \internal
	* \brief Wait until all pending cluster actions are complete
	*
	* This waits until either the CIB's transition graph is idle or a timeout is
	* reached.
	*
	* \param[in,out] out Output object
	* \param[in] timeout_ms Consider failed if actions do not complete in
	* this time (specified in milliseconds, but
	* one-second granularity is actually used; if 0, a
	* default will be used)
	* \param[in,out] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	*/
	int
	wait_till_stable(pcmk__output_t out, guint timeout_ms, cib_t cib)
	{
	pcmk_scheduler_t *scheduler = NULL;
	xmlXPathObjectPtr search;
	int rc = pcmk_rc_ok;
	bool pending_unknown_state_resources;
	time_t expire_time = time(NULL);
	time_t time_diff;
	bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
	char *xpath = NULL;

	if (timeout_ms == 0) {
	expire_time += WAIT_DEFAULT_TIMEOUT_S;
	} else {
	expire_time += (timeout_ms + 999) / 1000;
	}

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	return ENOMEM;
	}

	xpath = crm_strdup_printf("/" PCMK_XE_CIB "/" PCMK_XE_STATUS
	"/" PCMK__XE_NODE_STATE "/" PCMK__XE_LRM
	"/" PCMK__XE_LRM_RESOURCES
	"/" PCMK__XE_LRM_RESOURCE
	"/" PCMK__XE_LRM_RSC_OP
	"[@" PCMK__XA_RC_CODE "='%d']",
	PCMK_OCF_UNKNOWN);
	do {
	/* Abort if timeout is reached */
	time_diff = expire_time - time(NULL);
	if (time_diff <= 0) {
	print_pending_actions(out, scheduler->actions);
	rc = ETIME;
	break;
	}

	crm_info("Waiting up to %lld seconds for cluster actions to complete",
	(long long) time_diff);

	if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
	sleep(WAIT_SLEEP_S);
	}

	/* Get latest transition graph */
	pe_reset_working_set(scheduler);
	rc = update_scheduler_input_to_cib(out, scheduler, cib);
	if (rc != pcmk_rc_ok) {
	break;
	}
	pcmk__schedule_actions(scheduler->input,
	pcmk_sched_no_counts\|pcmk_sched_no_compat,
	scheduler);

	if (!printed_version_warning) {
	/* If the DC has a different version than the local node, the two
	* could come to different conclusions about what actions need to be
	* done. Warn the user in this case.
	*
	* @TODO A possible long-term solution would be to reimplement the
	* wait as a new controller operation that would be forwarded to the
	* DC. However, that would have potential problems of its own.
	*/
	const char *dc_version = g_hash_table_lookup(scheduler->config_hash,
	PCMK_OPT_DC_VERSION);

	if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
	out->info(out, "warning: wait option may not work properly in "
	"mixed-version cluster");
	printed_version_warning = true;
	}
	}

	search = xpath_search(scheduler->input, xpath);
	pending_unknown_state_resources = (numXpathResults(search) > 0);
	freeXpathObject(search);
	} while (actions_are_pending(scheduler->actions) \|\| pending_unknown_state_resources);

	pe_free_working_set(scheduler);
	free(xpath);
	return rc;
	}

	static const char *
	get_action(const char *rsc_action) {
	const char *action = NULL;

	if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
	action = PCMK_ACTION_VALIDATE_ALL;

	} else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
	action = PCMK_ACTION_MONITOR;

	} else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
	"force-demote", "force-promote", NULL)) {
	action = rsc_action+6;
	} else {
	action = rsc_action;
	}

	return action;
	}

	/*!
	* \brief Set up environment variables as expected by resource agents
	*
	* When the cluster executes resource agents, it adds certain environment
	* variables (directly or via resource meta-attributes) expected by some
	* resource agents. Add the essential ones that many resource agents expect, so
	* the behavior is the same for command-line execution.
	*
	* \param[in,out] params Resource parameters that will be passed to agent
	* \param[in] timeout_ms Action timeout (in milliseconds)
	* \param[in] check_level OCF check level
	* \param[in] verbosity Verbosity level
	*/
	static void
	set_agent_environment(GHashTable *params, guint timeout_ms, int check_level,
	int verbosity)
	{
	g_hash_table_insert(params, crm_meta_name(PCMK_META_TIMEOUT),
	crm_strdup_printf("%u", timeout_ms));

	pcmk__insert_dup(params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);

	if (check_level >= 0) {
	char *level = crm_strdup_printf("%d", check_level);

	setenv("OCF_CHECK_LEVEL", level, 1);
	free(level);
	}

	pcmk__set_env_option(PCMK__ENV_DEBUG, ((verbosity > 0)? "1" : "0"), true);
	if (verbosity > 1) {
	setenv("OCF_TRACE_RA", "1", 1);
	}

	/* A resource agent using the standard ocf-shellfuncs library will not print
	* messages to stderr if it doesn't have a controlling terminal (e.g. if
	* crm_resource is called via script or ssh). This forces it to do so.
	*/
	setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
	}

	/*!
	* \internal
	* \brief Apply command-line overrides to resource parameters
	*
	* \param[in,out] params Parameters to be passed to agent
	* \param[in] overrides Parameters to override (or NULL if none)
	*/
	static void
	apply_overrides(GHashTable params, GHashTable overrides)
	{
	if (overrides != NULL) {
	GHashTableIter iter;
	char *name = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer *) &name,
	(gpointer *) &value)) {
	pcmk__insert_dup(params, name, value);
	}
	}
	}

	crm_exit_t
	cli_resource_execute_from_params(pcmk__output_t out, const char rsc_name,
	const char rsc_class, const char rsc_prov,
	const char rsc_type, const char rsc_action,
	GHashTable params, GHashTable override_hash,
	guint timeout_ms, int resource_verbose,
	gboolean force, int check_level)
	{
	const char *class = rsc_class;
	const char *action = get_action(rsc_action);
	crm_exit_t exit_code = CRM_EX_OK;
	svc_action_t *op = NULL;

	// If no timeout was provided, use the same default as the cluster
	if (timeout_ms == 0U) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}

	set_agent_environment(params, timeout_ms, check_level, resource_verbose);
	apply_overrides(params, override_hash);

	op = services__create_resource_action(rsc_name? rsc_name : "test",
	rsc_class, rsc_prov, rsc_type, action,
	0, QB_MIN(timeout_ms, INT_MAX),
	params, 0);
	if (op == NULL) {
	out->err(out, "Could not execute %s using %s%s%s:%s: %s",
	action, rsc_class, (rsc_prov? ":" : ""),
	(rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
	g_hash_table_destroy(params);
	return CRM_EX_OSERR;
	}

	if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
	class = resources_find_service_class(rsc_type);
	}
	if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
	PCMK_RESOURCE_CLASS_LSB, NULL)) {
	services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
	"Manual execution of the %s standard is "
	"unsupported", pcmk__s(class, "unspecified"));
	}

	if (op->rc != PCMK_OCF_UNKNOWN) {
	exit_code = op->rc;
	goto done;
	}

	services_action_sync(op);

	// Map results to OCF codes for consistent reporting to user
	{
	enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);

	// Cast variable instead of function return to keep compilers happy
	exit_code = (crm_exit_t) ocf_code;
	}

	done:
	out->message(out, "resource-agent-action", resource_verbose, rsc_class,
	rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
	exit_code, op->status, services__exit_reason(op),
	op->stdout_data, op->stderr_data);
	services_action_free(op);
	return exit_code;
	}

	/*!
	* \internal
	* \brief Get the timeout the cluster would use for an action
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action Name of action
	*/
	static guint
	get_action_timeout(pcmk_resource_t rsc, const char action)
	{
	long long timeout_ms = -1LL;
	xmlNode *op = pcmk__find_action_config(rsc, action, 0, true);
	GHashTable *meta = pcmk__unpack_action_meta(rsc, NULL, action, 0, op);

	if ((pcmk__scan_ll(g_hash_table_lookup(meta, PCMK_META_TIMEOUT),
	&timeout_ms, -1LL) != pcmk_rc_ok)
	\|\| (timeout_ms <= 0LL)) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}
	g_hash_table_destroy(meta);
	return (guint) QB_MIN(timeout_ms, UINT_MAX);
	}

	crm_exit_t
	cli_resource_execute(pcmk_resource_t rsc, const char requested_name,
	const char rsc_action, GHashTable override_hash,
	guint timeout_ms, cib_t cib, pcmk_scheduler_t scheduler,
	int resource_verbose, gboolean force, int check_level)
	{
	pcmk__output_t *out = scheduler->priv;
	crm_exit_t exit_code = CRM_EX_OK;
	const char *rid = requested_name;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	GHashTable *params = NULL;

	if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
	"force-promote", NULL)) {
	if (pcmk__is_clone(rsc)) {
	GList *nodes = cli_resource_search(rsc, requested_name, scheduler);
	if(nodes != NULL && force == FALSE) {
	out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
	rsc_action, rsc->id);
	out->err(out,
	"Try setting "
	PCMK_META_TARGET_ROLE "=" PCMK_ROLE_STOPPED
	" first or specifying the force option");
	return CRM_EX_UNSAFE;
	}

	g_list_free_full(nodes, free);
	}
	}

	if (pcmk__is_clone(rsc)) {
	/* Grab the first child resource in the hope it's not a group */
	rsc = rsc->private->children->data;
	}

	if (pcmk__is_group(rsc)) {
	out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	} else if (pcmk__is_bundled(rsc)) {
	out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	}

	rclass = crm_element_value(rsc->private->xml, PCMK_XA_CLASS);
	rprov = crm_element_value(rsc->private->xml, PCMK_XA_PROVIDER);
	rtype = crm_element_value(rsc->private->xml, PCMK_XA_TYPE);

	params = generate_resource_params(rsc, NULL /* @TODO use local node */,
	scheduler);

	if (timeout_ms == 0U) {
	timeout_ms = get_action_timeout(rsc, get_action(rsc_action));
	}

	if (!pcmk__is_anonymous_clone(rsc->private->parent)) {
	rid = rsc->id;
	}

	exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
	params, override_hash, timeout_ms,
	resource_verbose, force, check_level);
	return exit_code;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_move(const pcmk_resource_t rsc, const char rsc_id,
	const char host_name, const char move_lifetime, cib_t *cib,
	int cib_options, pcmk_scheduler_t *scheduler,
	gboolean promoted_role_only, gboolean force)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	unsigned int count = 0;
	pcmk_node_t *current = NULL;
	pcmk_node_t *dest = pcmk_find_node(scheduler, host_name);
	bool cur_is_dest = false;

	if (dest == NULL) {
	return pcmk_rc_node_unknown;
	}

	if (promoted_role_only
	&& !pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {

	const pcmk_resource_t *p = pe__const_top_resource(rsc, false);

	if (pcmk_is_set(p->flags, pcmk__rsc_promotable)) {
	out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
	rsc_id = p->id;
	rsc = p;

	} else {
	out->info(out, "Ignoring --promoted option: %s is not promotable",
	rsc_id);
	promoted_role_only = FALSE;
	}
	}

	current = pe__find_active_requires(rsc, &count);

	if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
	unsigned int promoted_count = 0;
	pcmk_node_t *promoted_node = NULL;

	for (const GList *iter = rsc->private->children;
	iter != NULL; iter = iter->next) {

	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;
	enum rsc_role_e child_role = child->private->fns->state(child,
	TRUE);

	if (child_role == pcmk_role_promoted) {
	rsc = child;
	promoted_node = pcmk__current_node(child);
	promoted_count++;
	}
	}
	if (promoted_role_only \|\| (promoted_count != 0)) {
	count = promoted_count;
	current = promoted_node;
	}

	}

	if (count > 1) {
	if (pcmk__is_clone(rsc)) {
	current = NULL;
	} else {
	return pcmk_rc_multiple;
	}
	}

	if (pcmk__same_node(current, dest)) {
	cur_is_dest = true;
	if (force) {
	crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
	rsc_id, promoted_role_only?"promoted":"active",
	pcmk__node_name(dest));
	} else {
	return pcmk_rc_already;
	}
	}

	/* Clear any previous prefer constraints across all nodes. */
	cli_resource_clear(rsc_id, NULL, scheduler->nodes, cib, cib_options, false,
	force);

	/* Clear any previous ban constraints on 'dest'. */
	cli_resource_clear(rsc_id, dest->private->name, scheduler->nodes, cib,
	cib_options, TRUE, force);

	/* Record an explicit preference for 'dest' */
	rc = cli_resource_prefer(out, rsc_id, dest->private->name, move_lifetime,
	cib, cib_options, promoted_role_only,
	PCMK_ROLE_PROMOTED);

	crm_trace("%s%s now prefers %s%s",
	rsc->id, (promoted_role_only? " (promoted)" : ""),
	pcmk__node_name(dest), force?"(forced)":"");

	/* only ban the previous location if current location != destination location.
	* it is possible to use -M to enforce a location without regard of where the
	* resource is currently located */
	if (force && !cur_is_dest) {
	/* Ban the original location if possible */
	if(current) {
	(void)cli_resource_ban(out, rsc_id, current->private->name,
	move_lifetime, cib, cib_options,
	promoted_role_only, PCMK_ROLE_PROMOTED);
	} else if(count > 1) {
	out->info(out, "Resource '%s' is currently %s in %d locations. "
	"One may now move to %s",
	rsc_id, (promoted_role_only? "promoted" : "active"),
	count, pcmk__node_name(dest));
	out->info(out, "To prevent '%s' from being %s at a specific location, "
	"specify a node.",
	rsc_id, (promoted_role_only? "promoted" : "active"));

	} else {
	crm_trace("Not banning %s from its current location: not active", rsc_id);
	}
	}

	return rc;
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 8:08 PM (4 h, 35 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665674
Default Alt Text: (1 MB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions