No OneTemporary
Actions

Size

764 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/crm/common/nodes.h b/include/crm/common/nodes.h
	index b07fa1c9e3..503fe7680a 100644
	--- a/include/crm/common/nodes.h
	+++ b/include/crm/common/nodes.h
	@@ -1,171 +1,188 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_NODES__H
	# define PCMK__CRM_COMMON_NODES__H

	+#include <stdbool.h> // bool
	#include <glib.h> // gboolean, GList, GHashTable

	#include <crm/common/scheduler_types.h> // pcmk_resource_t, pcmk_scheduler_t

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*!
	* \file
	* \brief Scheduler API for nodes
	* \ingroup core
	*/

	// Special node attributes

	#define PCMK_NODE_ATTR_TERMINATE "terminate"


	//! Possible node types
	enum node_type {
	pcmk_node_variant_cluster = 1, //!< Cluster layer node
	pcmk_node_variant_remote = 2, //!< Pacemaker Remote node

	node_ping = 0, //!< \deprecated Do not use
	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	//! \deprecated Use pcmk_node_variant_cluster instead
	node_member = pcmk_node_variant_cluster,

	//! \deprecated Use pcmk_node_variant_remote instead
	node_remote = pcmk_node_variant_remote,
	#endif
	};

	//! When to probe a resource on a node (as specified in location constraints)
	enum pe_discover_e {
	pcmk_probe_always = 0, //! Always probe resource on node
	pcmk_probe_never = 1, //! Never probe resource on node
	pcmk_probe_exclusive = 2, //! Probe only on designated nodes

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	//! \deprecated Use pcmk_probe_always instead
	pe_discover_always = pcmk_probe_always,

	//! \deprecated Use pcmk_probe_never instead
	pe_discover_never = pcmk_probe_never,

	//! \deprecated Use pcmk_probe_exclusive instead
	pe_discover_exclusive = pcmk_probe_exclusive,
	#endif
	};

	//! Basic node information (all node objects for the same node share this)
	struct pe_node_shared_s {
	const char *id; //!< Node ID at the cluster layer
	const char *uname; //!< Node name in cluster
	enum node_type type; //!< Node variant

	// @TODO Convert these into a flag group
	gboolean online; //!< Whether online
	gboolean standby; //!< Whether in standby mode
	gboolean standby_onfail; //!< Whether in standby mode due to on-fail
	gboolean pending; //!< Whether controller membership is pending
	gboolean unclean; //!< Whether node requires fencing
	gboolean unseen; //!< Whether node has never joined cluster
	gboolean shutdown; //!< Whether shutting down
	gboolean expected_up; //!< Whether expected join state is member
	gboolean is_dc; //!< Whether node is cluster's DC
	gboolean maintenance; //!< Whether in maintenance mode
	gboolean rsc_discovery_enabled; //!< Whether probes are allowed on node

	/*!
	* Whether this is a guest node whose guest resource must be recovered or a
	* remote node that must be fenced
	*/
	gboolean remote_requires_reset;

	/*!
	* Whether this is a Pacemaker Remote node that was fenced since it was last
	* connected by the cluster
	*/
	gboolean remote_was_fenced;

	/*!
	* Whether this is a Pacemaker Remote node previously marked in its
	* node state as being in maintenance mode
	*/
	gboolean remote_maintenance;

	gboolean unpacked; //!< Whether node history has been unpacked

	/*!
	* Number of resources active on this node (valid after CIB status section
	* has been unpacked, as long as pcmk_sched_no_counts was not set)
	*/
	int num_resources;

	//! Remote connection resource for node, if it is a Pacemaker Remote node
	pcmk_resource_t *remote_rsc;

	GList *running_rsc; //!< List of resources active on node
	GList *allocated_rsc; //!< List of resources assigned to node
	GHashTable *attrs; //!< Node attributes
	GHashTable *utilization; //!< Node utilization attributes
	GHashTable *digest_cache; //!< Cache of calculated resource digests

	/*!
	* Sum of priorities of all resources active on node and on any guest nodes
	* connected to this node, with +1 for promoted instances (used to compare
	* nodes for PCMK_OPT_PRIORITY_FENCING_DELAY)
	*/
	int priority;

	pcmk_scheduler_t *data_set; //!< Cluster that node is part of
	};

	//! Implementation of pcmk_node_t
	struct pe_node_s {
	int weight; //!< Node score for a given resource
	gboolean fixed; //!< \deprecated Do not use
	int count; //!< Counter reused by assignment and promotion code
	struct pe_node_shared_s *details; //!< Basic node information

	// @COMPAT This should be enum pe_discover_e
	int rsc_discover_mode; //!< Probe mode (enum pe_discover_e)
	};

	/*!
	* \internal
	* \brief Return a string suitable for logging as a node name
	*
	* \param[in] node Node to return a node name string for
	*
	* \return Node name if available, otherwise node ID if available,
	* otherwise "unspecified node" if node is NULL or "unidentified node"
	* if node has neither a name nor ID.
	*/
	static inline const char *
	pcmk__node_name(const pcmk_node_t *node)
	{
	if (node == NULL) {
	return "unspecified node";

	} else if (node->details->uname != NULL) {
	return node->details->uname;

	} else if (node->details->id != NULL) {
	return node->details->id;

	} else {
	return "unidentified node";
	}
	}

	+/*!
	+ * \internal
	+ * \brief Check whether two node objects refer to the same node
	+ *
	+ * \param[in] node1 First node object to compare
	+ * \param[in] node2 Second node object to compare
	+ *
	+ * \return true if \p node1 and \p node2 refer to the same node
	+ */
	+static inline bool
	+pcmk__same_node(const pcmk_node_t node1, const pcmk_node_t node2)
	+{
	+ return (node1 != NULL) && (node2 != NULL)
	+ && (node1->details == node2->details);
	+}
	+
	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_NODES__H
	diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
	index a9eb45a3a3..bab1645e5e 100644
	--- a/include/crm/pengine/internal.h
	+++ b/include/crm/pengine/internal.h
	@@ -1,494 +1,478 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PE_INTERNAL__H
	# define PE_INTERNAL__H

	# include <stdbool.h>
	# include <stdint.h>
	# include <string.h>
	# include <crm/msg_xml.h>
	# include <crm/pengine/status.h>
	# include <crm/pengine/remote_internal.h>
	# include <crm/common/internal.h>
	# include <crm/common/options_internal.h>
	# include <crm/common/output_internal.h>
	# include <crm/common/scheduler_internal.h>

	const char pe__resource_description(const pcmk_resource_t rsc,
	uint32_t show_opts);

	bool pe__clone_is_ordered(const pcmk_resource_t *clone);
	int pe__set_clone_flag(pcmk_resource_t *clone, enum pcmk__clone_flags flag);
	bool pe__clone_flag_is_set(const pcmk_resource_t *clone, uint32_t flags);

	bool pe__group_flag_is_set(const pcmk_resource_t *group, uint32_t flags);
	pcmk_resource_t pe__last_group_member(const pcmk_resource_t group);

	const pcmk_resource_t pe__const_top_resource(const pcmk_resource_t rsc,
	bool include_bundle);

	int pe__clone_max(const pcmk_resource_t *clone);
	int pe__clone_node_max(const pcmk_resource_t *clone);
	int pe__clone_promoted_max(const pcmk_resource_t *clone);
	int pe__clone_promoted_node_max(const pcmk_resource_t *clone);
	void pe__create_clone_notifications(pcmk_resource_t *clone);
	void pe__free_clone_notification_data(pcmk_resource_t *clone);
	void pe__create_clone_notif_pseudo_ops(pcmk_resource_t *clone,
	pcmk_action_t *start,
	pcmk_action_t *started,
	pcmk_action_t *stop,
	pcmk_action_t *stopped);

	pcmk_action_t pe__new_rsc_pseudo_action(pcmk_resource_t rsc, const char *task,
	bool optional, bool runnable);

	void pe__create_promotable_pseudo_ops(pcmk_resource_t *clone,
	bool any_promoting, bool any_demoting);

	bool pe_can_fence(const pcmk_scheduler_t scheduler, const pcmk_node_t node);

	void add_hash_param(GHashTable * hash, const char name, const char value);

	char native_parameter(pcmk_resource_t rsc, pcmk_node_t *node, gboolean create,
	const char name, pcmk_scheduler_t scheduler);
	pcmk_node_t native_location(const pcmk_resource_t rsc, GList **list,
	int current);

	void pe_metadata(pcmk__output_t *out);
	void verify_pe_options(GHashTable * options);

	void native_add_running(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler, gboolean failed);

	gboolean native_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);
	gboolean group_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);
	gboolean clone_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);
	gboolean pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);

	pcmk_resource_t native_find_rsc(pcmk_resource_t rsc, const char *id,
	const pcmk_node_t *node, int flags);

	gboolean native_active(pcmk_resource_t *rsc, gboolean all);
	gboolean group_active(pcmk_resource_t *rsc, gboolean all);
	gboolean clone_active(pcmk_resource_t *rsc, gboolean all);
	gboolean pe__bundle_active(pcmk_resource_t *rsc, gboolean all);

	//! \deprecated This function will be removed in a future release
	void native_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void group_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void clone_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void pe__print_bundle(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	gchar pcmk__native_output_string(const pcmk_resource_t rsc, const char *name,
	const pcmk_node_t *node, uint32_t show_opts,
	const char *target_role, bool show_nodes);

	int pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...);
	char pe__node_display_name(pcmk_node_t node, bool print_detail);


	// Clone notifications (pe_notif.c)
	void pe__order_notifs_after_fencing(const pcmk_action_t *action,
	pcmk_resource_t *rsc,
	pcmk_action_t *stonith_op);


	// Resource output methods
	int pe__clone_xml(pcmk__output_t *out, va_list args);
	int pe__clone_default(pcmk__output_t *out, va_list args);
	int pe__group_xml(pcmk__output_t *out, va_list args);
	int pe__group_default(pcmk__output_t *out, va_list args);
	int pe__bundle_xml(pcmk__output_t *out, va_list args);
	int pe__bundle_html(pcmk__output_t *out, va_list args);
	int pe__bundle_text(pcmk__output_t *out, va_list args);
	int pe__node_html(pcmk__output_t *out, va_list args);
	int pe__node_text(pcmk__output_t *out, va_list args);
	int pe__node_xml(pcmk__output_t *out, va_list args);
	int pe__resource_xml(pcmk__output_t *out, va_list args);
	int pe__resource_html(pcmk__output_t *out, va_list args);
	int pe__resource_text(pcmk__output_t *out, va_list args);

	void native_free(pcmk_resource_t *rsc);
	void group_free(pcmk_resource_t *rsc);
	void clone_free(pcmk_resource_t *rsc);
	void pe__free_bundle(pcmk_resource_t *rsc);

	enum rsc_role_e native_resource_state(const pcmk_resource_t *rsc,
	gboolean current);
	enum rsc_role_e group_resource_state(const pcmk_resource_t *rsc,
	gboolean current);
	enum rsc_role_e clone_resource_state(const pcmk_resource_t *rsc,
	gboolean current);
	enum rsc_role_e pe__bundle_resource_state(const pcmk_resource_t *rsc,
	gboolean current);

	void pe__count_common(pcmk_resource_t *rsc);
	void pe__count_bundle(pcmk_resource_t *rsc);

	void common_free(pcmk_resource_t *rsc);

	pcmk_node_t pe__copy_node(const pcmk_node_t this_node);
	time_t get_effective_time(pcmk_scheduler_t *scheduler);

	/* Failure handling utilities (from failcounts.c) */

	int pe_get_failcount(const pcmk_node_t node, pcmk_resource_t rsc,
	time_t *last_failure, uint32_t flags,
	const xmlNode *xml_op);

	pcmk_action_t pe__clear_failcount(pcmk_resource_t rsc,
	const pcmk_node_t node, const char reason,
	pcmk_scheduler_t *scheduler);

	/* Functions for finding/counting a resource's active nodes */

	bool pe__count_active_node(const pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_node_t *active, unsigned int count_all,
	unsigned int *count_clean);

	pcmk_node_t pe__find_active_requires(const pcmk_resource_t rsc,
	unsigned int *count);

	/* Binary like operators for lists of nodes */
	GHashTable pe__node_list2table(const GList list);

	pcmk_action_t get_pseudo_op(const char name, pcmk_scheduler_t *scheduler);
	gboolean order_actions(pcmk_action_t lh_action, pcmk_action_t rh_action,
	uint32_t flags);

	void pe__show_node_scores_as(const char file, const char function,
	int line, bool to_log, const pcmk_resource_t *rsc,
	const char comment, GHashTable nodes,
	pcmk_scheduler_t *scheduler);

	#define pe__show_node_scores(level, rsc, text, nodes, scheduler) \
	pe__show_node_scores_as(__FILE__, __func__, __LINE__, \
	(level), (rsc), (text), (nodes), (scheduler))

	GHashTable pcmk__unpack_action_meta(pcmk_resource_t rsc,
	const pcmk_node_t *node,
	const char *action_name, guint interval_ms,
	const xmlNode *action_config);
	GHashTable pcmk__unpack_action_rsc_params(const xmlNode action_xml,
	GHashTable *node_attrs,
	pcmk_scheduler_t *data_set);
	xmlNode pcmk__find_action_config(const pcmk_resource_t rsc,
	const char *action_name, guint interval_ms,
	bool include_disabled);

	enum rsc_start_requirement pcmk__action_requires(const pcmk_resource_t *rsc,
	const char *action_name);

	enum action_fail_response pcmk__parse_on_fail(const pcmk_resource_t *rsc,
	const char *action_name,
	guint interval_ms,
	const char *value);

	enum rsc_role_e pcmk__role_after_failure(const pcmk_resource_t *rsc,
	const char *action_name,
	enum action_fail_response on_fail,
	GHashTable *meta);

	pcmk_action_t custom_action(pcmk_resource_t rsc, char key, const char task,
	const pcmk_node_t *on_node, gboolean optional,
	pcmk_scheduler_t *scheduler);

	# define delete_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_DELETE, 0)
	# define delete_action(rsc, node, optional) custom_action( \
	rsc, delete_key(rsc), PCMK_ACTION_DELETE, node, \
	optional, rsc->cluster);

	# define stop_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0)
	# define stop_action(rsc, node, optional) custom_action( \
	rsc, stop_key(rsc), PCMK_ACTION_STOP, node, \
	optional, rsc->cluster);

	# define reload_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_RELOAD_AGENT, 0)
	# define start_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_START, 0)
	# define start_action(rsc, node, optional) custom_action( \
	rsc, start_key(rsc), PCMK_ACTION_START, node, \
	optional, rsc->cluster)

	# define promote_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0)
	# define promote_action(rsc, node, optional) custom_action( \
	rsc, promote_key(rsc), PCMK_ACTION_PROMOTE, node, \
	optional, rsc->cluster)

	# define demote_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0)
	# define demote_action(rsc, node, optional) custom_action( \
	rsc, demote_key(rsc), PCMK_ACTION_DEMOTE, node, \
	optional, rsc->cluster)

	extern int pe_get_configured_timeout(pcmk_resource_t rsc, const char action,
	pcmk_scheduler_t *scheduler);

	pcmk_action_t find_first_action(const GList input, const char *uuid,
	const char task, const pcmk_node_t on_node);

	enum action_tasks get_complex_task(const pcmk_resource_t *rsc,
	const char *name);

	GList find_actions(GList input, const char key, const pcmk_node_t on_node);
	GList find_actions_exact(GList input, const char *key,
	const pcmk_node_t *on_node);
	GList pe__resource_actions(const pcmk_resource_t rsc, const pcmk_node_t *node,
	const char *task, bool require_node);

	extern void pe_free_action(pcmk_action_t *action);

	void resource_location(pcmk_resource_t rsc, const pcmk_node_t node, int score,
	const char tag, pcmk_scheduler_t scheduler);

	extern int pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b,
	bool same_node_default);
	extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
	gboolean get_target_role(const pcmk_resource_t rsc, enum rsc_role_e role);
	void pe__set_next_role(pcmk_resource_t *rsc, enum rsc_role_e role,
	const char *why);

	pcmk_resource_t find_clone_instance(const pcmk_resource_t rsc,
	const char *sub_id);

	extern void destroy_ticket(gpointer data);
	pcmk_ticket_t ticket_new(const char ticket_id, pcmk_scheduler_t *scheduler);

	// Resources for manipulating resource names
	const char pe_base_name_end(const char id);
	char clone_strip(const char last_rsc_id);
	char clone_zero(const char last_rsc_id);

	static inline bool
	pe_base_name_eq(const pcmk_resource_t rsc, const char id)
	{
	if (id && rsc && rsc->id) {
	// Number of characters in rsc->id before any clone suffix
	size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;

	return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
	}
	return false;
	}

	int pe__target_rc_from_xml(const xmlNode *xml_op);

	gint pe__cmp_node_name(gconstpointer a, gconstpointer b);
	bool is_set_recursive(const pcmk_resource_t *rsc, long long flag, bool any);

	pcmk__op_digest_t pe__calculate_digests(pcmk_resource_t rsc, const char *task,
	guint *interval_ms,
	const pcmk_node_t *node,
	const xmlNode *xml_op,
	GHashTable *overrides,
	bool calc_secure,
	pcmk_scheduler_t *scheduler);

	void pe__free_digests(gpointer ptr);

	pcmk__op_digest_t rsc_action_digest_cmp(pcmk_resource_t rsc,
	const xmlNode *xml_op,
	pcmk_node_t *node,
	pcmk_scheduler_t *scheduler);

	pcmk_action_t pe_fence_op(pcmk_node_t node, const char *op, bool optional,
	const char *reason, bool priority_delay,
	pcmk_scheduler_t *scheduler);
	void trigger_unfencing(pcmk_resource_t rsc, pcmk_node_t node,
	const char reason, pcmk_action_t dependency,
	pcmk_scheduler_t *scheduler);

	char pe__action2reason(const pcmk_action_t action, enum pe_action_flags flag);
	void pe_action_set_reason(pcmk_action_t action, const char reason,
	bool overwrite);
	void pe__add_action_expected_result(pcmk_action_t *action, int expected_result);

	void pe__set_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_on_all(pcmk_scheduler_t *scheduler,
	uint64_t flag);

	gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref);

	//! \deprecated This function will be removed in a future release
	void print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void * print_data, gboolean print_all);
	int pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, unsigned int options);
	void pe_fence_node(pcmk_scheduler_t scheduler, pcmk_node_t node,
	const char *reason, bool priority_delay);

	pcmk_node_t pe_create_node(const char id, const char uname, const char type,
	const char score, pcmk_scheduler_t scheduler);

	//! \deprecated This function will be removed in a future release
	void common_print(pcmk_resource_t rsc, const char pre_text, const char *name,
	const pcmk_node_t node, long options, void print_data);
	int pe__common_output_text(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	unsigned int options);
	int pe__common_output_html(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	unsigned int options);

	GList pe__bundle_containers(const pcmk_resource_t bundle);

	int pe__bundle_max(const pcmk_resource_t *rsc);
	bool pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node);
	pcmk_resource_t pe__bundled_resource(const pcmk_resource_t rsc);
	const pcmk_resource_t pe__get_rsc_in_container(const pcmk_resource_t instance);
	pcmk_resource_t pe__first_container(const pcmk_resource_t bundle);
	void pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data);
	void pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data);
	pcmk_resource_t pe__find_bundle_replica(const pcmk_resource_t bundle,
	const pcmk_node_t *node);
	bool pe__bundle_needs_remote_name(pcmk_resource_t *rsc);
	const char pe__add_bundle_remote_name(pcmk_resource_t rsc,
	pcmk_scheduler_t *scheduler,
	xmlNode xml, const char field);

	const char pe__node_attribute_calculated(const pcmk_node_t node,
	const char *name,
	const pcmk_resource_t *rsc,
	enum pcmk__rsc_node node_type,
	bool force_host);
	const char pe_node_attribute_raw(const pcmk_node_t node, const char *name);
	bool pe__is_universal_clone(const pcmk_resource_t *rsc,
	const pcmk_scheduler_t *scheduler);
	void pe__add_param_check(const xmlNode rsc_op, pcmk_resource_t rsc,
	pcmk_node_t *node, enum pcmk__check_parameters,
	pcmk_scheduler_t *scheduler);
	void pe__foreach_param_check(pcmk_scheduler_t *scheduler,
	void (cb)(pcmk_resource_t, pcmk_node_t*,
	const xmlNode*,
	enum pcmk__check_parameters));
	void pe__free_param_checks(pcmk_scheduler_t *scheduler);

	bool pe__shutdown_requested(const pcmk_node_t *node);
	void pe__update_recheck_time(time_t recheck, pcmk_scheduler_t *scheduler,
	const char *reason);

	/*!
	* \internal
	* \brief Register xml formatting message functions.
	*
	* \param[in,out] out Output object to register messages with
	*/
	void pe__register_messages(pcmk__output_t *out);

	void pe__unpack_dataset_nvpairs(const xmlNode xml_obj, const char set_name,
	const pe_rule_eval_data_t *rule_data,
	GHashTable hash, const char always_first,
	gboolean overwrite,
	pcmk_scheduler_t *scheduler);

	bool pe__resource_is_disabled(const pcmk_resource_t *rsc);
	void pe__clear_resource_history(pcmk_resource_t rsc, const pcmk_node_t node);

	GList pe__rscs_with_tag(pcmk_scheduler_t scheduler, const char *tag_name);
	GList pe__unames_with_tag(pcmk_scheduler_t scheduler, const char *tag_name);
	bool pe__rsc_has_tag(pcmk_scheduler_t scheduler, const char rsc,
	const char *tag);
	bool pe__uname_has_tag(pcmk_scheduler_t scheduler, const char node,
	const char *tag);

	bool pe__rsc_running_on_only(const pcmk_resource_t *rsc,
	const pcmk_node_t *node);
	bool pe__rsc_running_on_any(pcmk_resource_t rsc, GList node_list);
	GList pe__filter_rsc_list(GList rscs, GList *filter);
	GList * pe__build_node_name_list(pcmk_scheduler_t scheduler, const char s);
	GList * pe__build_rsc_list(pcmk_scheduler_t scheduler, const char s);

	bool pcmk__rsc_filtered_by_node(pcmk_resource_t rsc, GList only_node);

	gboolean pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);
	gboolean pe__clone_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);
	gboolean pe__group_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);
	gboolean pe__native_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);

	xmlNode pe__failed_probe_for_rsc(const pcmk_resource_t rsc, const char *name);

	const char pe__clone_child_id(const pcmk_resource_t rsc);

	int pe__sum_node_health_scores(const pcmk_node_t *node, int base_health);
	int pe__node_health(pcmk_node_t *node);

	static inline enum pcmk__health_strategy
	pe__health_strategy(pcmk_scheduler_t *scheduler)
	{
	return pcmk__parse_health_strategy(pe_pref(scheduler->config_hash,
	PCMK_OPT_NODE_HEALTH_STRATEGY));
	}

	static inline int
	pe__health_score(const char option, pcmk_scheduler_t scheduler)
	{
	return char2score(pe_pref(scheduler->config_hash, option));
	}

	-/*!
	- * \internal
	- * \brief Check whether two node objects refer to the same node
	- *
	- * \param[in] node1 First node object to compare
	- * \param[in] node2 Second node object to compare
	- *
	- * \return true if \p node1 and \p node2 refer to the same node
	- */
	-static inline bool
	-pe__same_node(const pcmk_node_t node1, const pcmk_node_t node2)
	-{
	- return (node1 != NULL) && (node2 != NULL)
	- && (node1->details == node2->details);
	-}
	-
	/*!
	* \internal
	* \brief Get the operation key from an action history entry
	*
	* \param[in] xml Action history entry
	*
	* \return Entry's operation key
	*/
	static inline const char *
	pe__xe_history_key(const xmlNode *xml)
	{
	if (xml == NULL) {
	return NULL;
	} else {
	/* @COMPAT Pacemaker <= 1.1.5 did not add the key, and used the ID
	* instead. Checking for that allows us to process old saved CIBs,
	* including some regression tests.
	*/
	const char *key = crm_element_value(xml, PCMK__XA_OPERATION_KEY);

	return pcmk__str_empty(key)? ID(xml) : key;
	}
	}

	#endif
	diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c
	index 784d147d0c..f5b486f589 100644
	--- a/lib/pacemaker/pcmk_graph_producer.c
	+++ b/lib/pacemaker/pcmk_graph_producer.c
	@@ -1,1096 +1,1096 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pcmk_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pcmk_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pcmk_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	/*!
	* \internal
	* \brief Add an XML node tag for a specified ID
	*
	* \param[in] id Node UUID to add
	* \param[in,out] xml Parent XML tag to add to
	*/
	static xmlNode*
	add_node_to_xml_by_id(const char id, xmlNode xml)
	{
	xmlNode *node_xml;

	node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
	crm_xml_add(node_xml, PCMK_XA_ID, id);

	return node_xml;
	}

	/*!
	* \internal
	* \brief Add an XML node tag for a specified node
	*
	* \param[in] node Node to add
	* \param[in,out] xml XML to add node to
	*/
	static void
	add_node_to_xml(const pcmk_node_t node, void xml)
	{
	add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
	}

	/*!
	* \internal
	* \brief Count (optionally add to XML) nodes needing maintenance state update
	*
	* \param[in,out] xml Parent XML tag to add to, if any
	* \param[in] scheduler Scheduler data
	*
	* \return Count of nodes added
	* \note Only Pacemaker Remote nodes are considered currently
	*/
	static int
	add_maintenance_nodes(xmlNode xml, const pcmk_scheduler_t scheduler)
	{
	xmlNode *maintenance = NULL;
	int count = 0;

	if (xml != NULL) {
	maintenance = create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE);
	}
	for (const GList *iter = scheduler->nodes;
	iter != NULL; iter = iter->next) {
	const pcmk_node_t *node = iter->data;

	if (pe__is_guest_or_remote_node(node) &&
	(node->details->maintenance != node->details->remote_maintenance)) {

	if (maintenance != NULL) {
	crm_xml_add(add_node_to_xml_by_id(node->details->id,
	maintenance),
	XML_NODE_IS_MAINTENANCE,
	(node->details->maintenance? "1" : "0"));
	}
	count++;
	}
	}
	crm_trace("%s %d nodes in need of maintenance mode update in state",
	((maintenance == NULL)? "Counted" : "Added"), count);
	return count;
	}

	/*!
	* \internal
	* \brief Add pseudo action with nodes needing maintenance state update
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	add_maintenance_update(pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *action = NULL;

	if (add_maintenance_nodes(NULL, scheduler) != 0) {
	action = get_pseudo_op(PCMK_ACTION_MAINTENANCE_NODES, scheduler);
	pcmk__set_action_flags(action, pcmk_action_always_in_graph);
	}
	}

	/*!
	* \internal
	* \brief Add XML with nodes that an action is expected to bring down
	*
	* If a specified action is expected to bring any nodes down, add an XML block
	* with their UUIDs. When a node is lost, this allows the controller to
	* determine whether it was expected.
	*
	* \param[in,out] xml Parent XML tag to add to
	* \param[in] action Action to check for downed nodes
	*/
	static void
	add_downed_nodes(xmlNode xml, const pcmk_action_t action)
	{
	CRM_CHECK((xml != NULL) && (action != NULL) && (action->node != NULL),
	return);

	if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {

	/* Shutdown makes the action's node down */
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {

	/* Fencing makes the action's node and any hosted guest nodes down */
	const char *fence = g_hash_table_lookup(action->meta, "stonith_action");

	if (pcmk__is_fencing_action(fence)) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);
	pe_foreach_guest_node(action->node->details->data_set,
	action->node, add_node_to_xml, downed);
	}

	} else if (action->rsc && action->rsc->is_remote_node
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {

	/* Stopping a remote connection resource makes connected node down,
	* unless it's part of a migration
	*/
	GList *iter;
	pcmk_action_t *input;
	bool migrating = false;

	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	input = ((pcmk__related_action_t *) iter->data)->action;
	if ((input->rsc != NULL)
	&& pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_none)
	&& pcmk__str_eq(input->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	migrating = true;
	break;
	}
	}
	if (!migrating) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->rsc->id, downed);
	}
	}
	}

	/*!
	* \internal
	* \brief Create a transition graph operation key for a clone action
	*
	* \param[in] action Clone action
	* \param[in] interval_ms Action interval in milliseconds
	*
	* \return Newly allocated string with transition graph operation key
	*/
	static char *
	clone_op_key(const pcmk_action_t *action, guint interval_ms)
	{
	if (pcmk__str_eq(action->task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
	const char *n_task = g_hash_table_lookup(action->meta,
	"notify_operation");

	CRM_LOG_ASSERT((n_type != NULL) && (n_task != NULL));
	return pcmk__notify_key(action->rsc->clone_name, n_type, n_task);

	} else if (action->cancel_task != NULL) {
	return pcmk__op_key(action->rsc->clone_name, action->cancel_task,
	interval_ms);
	} else {
	return pcmk__op_key(action->rsc->clone_name, action->task, interval_ms);
	}
	}

	/*!
	* \internal
	* \brief Add node details to transition graph action XML
	*
	* \param[in] action Scheduled action
	* \param[in,out] xml Transition graph action XML for \p action
	*/
	static void
	add_node_details(const pcmk_action_t action, xmlNode xml)
	{
	pcmk_node_t *router_node = pcmk__connection_host_for_action(action);

	crm_xml_add(xml, PCMK__META_ON_NODE, action->node->details->uname);
	crm_xml_add(xml, PCMK__META_ON_NODE_UUID, action->node->details->id);
	if (router_node != NULL) {
	crm_xml_add(xml, PCMK__XA_ROUTER_NODE, router_node->details->uname);
	}
	}

	/*!
	* \internal
	* \brief Add resource details to transition graph action XML
	*
	* \param[in] action Scheduled action
	* \param[in,out] action_xml Transition graph action XML for \p action
	*/
	static void
	add_resource_details(const pcmk_action_t action, xmlNode action_xml)
	{
	xmlNode *rsc_xml = NULL;
	const char *attr_list[] = {
	PCMK_XA_CLASS,
	PCMK_XA_PROVIDER,
	PCMK_XA_TYPE,
	};

	/* If a resource is locked to a node via PCMK_OPT_SHUTDOWN_LOCK, mark its
	* actions so the controller can preserve the lock when the action
	* completes.
	*/
	if (pcmk__action_locks_rsc_to_node(action)) {
	crm_xml_add_ll(action_xml, PCMK_OPT_SHUTDOWN_LOCK,
	(long long) action->rsc->lock_time);
	}

	// List affected resource

	rsc_xml = create_xml_node(action_xml,
	(const char *) action->rsc->xml->name);
	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_removed)
	&& (action->rsc->clone_name != NULL)) {
	/* Use the numbered instance name here, because if there is more
	* than one instance on a node, we need to make sure the command
	* goes to the right one.
	*
	* This is important even for anonymous clones, because the clone's
	* unique meta-attribute might have just been toggled from on to
	* off.
	*/
	crm_debug("Using orphan clone name %s instead of %s",
	action->rsc->id, action->rsc->clone_name);
	crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->clone_name);
	crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id);

	} else if (!pcmk_is_set(action->rsc->flags, pcmk_rsc_unique)) {
	const char *xml_id = ID(action->rsc->xml);

	crm_debug("Using anonymous clone name %s for %s (aka %s)",
	xml_id, action->rsc->id, action->rsc->clone_name);

	/* ID is what we'd like client to use
	* LONG_ID is what they might know it as instead
	*
	* LONG_ID is only strictly needed /here/ during the
	* transition period until all nodes in the cluster
	* are running the new software /and/ have rebooted
	* once (meaning that they've only ever spoken to a DC
	* supporting this feature).
	*
	* If anyone toggles the unique flag to 'on', the
	* 'instance free' name will correspond to an orphan
	* and fall into the clause above instead
	*/
	crm_xml_add(rsc_xml, PCMK_XA_ID, xml_id);
	if ((action->rsc->clone_name != NULL)
	&& !pcmk__str_eq(xml_id, action->rsc->clone_name,
	pcmk__str_none)) {
	crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->clone_name);
	} else {
	crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id);
	}

	} else {
	CRM_ASSERT(action->rsc->clone_name == NULL);
	crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->id);
	}

	for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
	crm_xml_add(rsc_xml, attr_list[lpc],
	g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
	}
	}

	/*!
	* \internal
	* \brief Add action attributes to transition graph action XML
	*
	* \param[in,out] action Scheduled action
	* \param[in,out] action_xml Transition graph action XML for \p action
	*/
	static void
	add_action_attributes(pcmk_action_t action, xmlNode action_xml)
	{
	xmlNode *args_xml = NULL;

	/* We create free-standing XML to start, so we can sort the attributes
	* before adding it to action_xml, which keeps the scheduler regression
	* test graphs comparable.
	*/
	args_xml = create_xml_node(NULL, XML_TAG_ATTRS);

	crm_xml_add(args_xml, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
	g_hash_table_foreach(action->extra, hash2field, args_xml);

	if ((action->rsc != NULL) && (action->node != NULL)) {
	// Get the resource instance attributes, evaluated properly for node
	GHashTable *params = pe_rsc_params(action->rsc, action->node,
	action->rsc->cluster);

	pcmk__substitute_remote_addr(action->rsc, params);

	g_hash_table_foreach(params, hash2smartfield, args_xml);

	} else if ((action->rsc != NULL)
	&& (action->rsc->variant <= pcmk_rsc_variant_primitive)) {
	GHashTable *params = pe_rsc_params(action->rsc, NULL,
	action->rsc->cluster);

	g_hash_table_foreach(params, hash2smartfield, args_xml);
	}

	g_hash_table_foreach(action->meta, hash2metafield, args_xml);
	if (action->rsc != NULL) {
	pcmk_resource_t *parent = action->rsc;

	while (parent != NULL) {
	parent->cmds->add_graph_meta(parent, args_xml);
	parent = parent->parent;
	}

	pcmk__add_guest_meta_to_xml(args_xml, action);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)
	&& (action->node != NULL)) {
	/* Pass the node's attributes as meta-attributes.
	*
	* @TODO: Determine whether it is still necessary to do this. It was
	* added in 33d99707, probably for the libfence-based implementation in
	* c9a90bd, which is no longer used.
	*/
	g_hash_table_foreach(action->node->details->attrs, hash2metafield,
	args_xml);
	}

	sorted_xml(args_xml, action_xml, FALSE);
	free_xml(args_xml);
	}

	/*!
	* \internal
	* \brief Create the transition graph XML for a scheduled action
	*
	* \param[in,out] parent Parent XML element to add action to
	* \param[in,out] action Scheduled action
	* \param[in] skip_details If false, add action details as sub-elements
	* \param[in] scheduler Scheduler data
	*/
	static void
	create_graph_action(xmlNode parent, pcmk_action_t action, bool skip_details,
	const pcmk_scheduler_t *scheduler)
	{
	bool needs_node_info = true;
	bool needs_maintenance_info = false;
	xmlNode *action_xml = NULL;

	if ((action == NULL) \|\| (scheduler == NULL)) {
	return;
	}

	// Create the top-level element based on task

	if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) {
	/* All fences need node info; guest node fences are pseudo-events */
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
	} else {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
	}

	} else if (pcmk__str_any_of(action->task,
	PCMK_ACTION_DO_SHUTDOWN,
	PCMK_ACTION_CLEAR_FAILCOUNT, NULL)) {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_LRM_DELETE,
	pcmk__str_none)) {
	// CIB-only clean-up for shutdown locks
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
	crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);

	} else if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	if (pcmk__str_eq(action->task, PCMK_ACTION_MAINTENANCE_NODES,
	pcmk__str_none)) {
	needs_maintenance_info = true;
	}
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
	needs_node_info = false;

	} else {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_RSC_OP);
	}

	crm_xml_add_int(action_xml, PCMK_XA_ID, action->id);
	crm_xml_add(action_xml, PCMK_XA_OPERATION, action->task);

	if ((action->rsc != NULL) && (action->rsc->clone_name != NULL)) {
	char *clone_key = NULL;
	guint interval_ms;

	if (pcmk__guint_from_hash(action->meta, PCMK_META_INTERVAL, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}
	clone_key = clone_op_key(action, interval_ms);
	crm_xml_add(action_xml, PCMK__XA_OPERATION_KEY, clone_key);
	crm_xml_add(action_xml, "internal_" PCMK__XA_OPERATION_KEY,
	action->uuid);
	free(clone_key);
	} else {
	crm_xml_add(action_xml, PCMK__XA_OPERATION_KEY, action->uuid);
	}

	if (needs_node_info && (action->node != NULL)) {
	add_node_details(action, action_xml);
	g_hash_table_insert(action->meta, strdup(PCMK__META_ON_NODE),
	strdup(action->node->details->uname));
	g_hash_table_insert(action->meta, strdup(PCMK__META_ON_NODE_UUID),
	strdup(action->node->details->id));
	}

	if (skip_details) {
	return;
	}

	if ((action->rsc != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_pseudo)) {

	// This is a real resource action, so add resource details
	add_resource_details(action, action_xml);
	}

	/* List any attributes in effect */
	add_action_attributes(action, action_xml);

	/* List any nodes this action is expected to make down */
	if (needs_node_info && (action->node != NULL)) {
	add_downed_nodes(action_xml, action);
	}

	if (needs_maintenance_info) {
	add_maintenance_nodes(action_xml, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Check whether an action should be added to the transition graph
	*
	* \param[in] action Action to check
	*
	* \return true if action should be added to graph, otherwise false
	*/
	static bool
	should_add_action_to_graph(const pcmk_action_t *action)
	{
	if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	crm_trace("Ignoring action %s (%d): unrunnable",
	action->uuid, action->id);
	return false;
	}

	if (pcmk_is_set(action->flags, pcmk_action_optional)
	&& !pcmk_is_set(action->flags, pcmk_action_always_in_graph)) {
	crm_trace("Ignoring action %s (%d): optional",
	action->uuid, action->id);
	return false;
	}

	/* Actions for unmanaged resources should be excluded from the graph,
	* with the exception of monitors and cancellation of recurring monitors.
	*/
	if ((action->rsc != NULL)
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
	&& !pcmk__str_eq(action->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {

	const char *interval_ms_s;

	/* A cancellation of a recurring monitor will get here because the task
	* is cancel rather than monitor, but the interval can still be used to
	* recognize it. The interval has been normalized to milliseconds by
	* this point, so a string comparison is sufficient.
	*/
	interval_ms_s = g_hash_table_lookup(action->meta, PCMK_META_INTERVAL);
	if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
	crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
	action->uuid, action->id, action->rsc->id);
	return false;
	}
	}

	/* Always add pseudo-actions, fence actions, and shutdown actions (already
	* determined to be required and runnable by this point)
	*/
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)
	\|\| pcmk__strcase_any_of(action->task, PCMK_ACTION_STONITH,
	PCMK_ACTION_DO_SHUTDOWN, NULL)) {
	return true;
	}

	if (action->node == NULL) {
	pcmk__sched_err("Skipping action %s (%d) "
	"because it was not assigned to a node (bug?)",
	action->uuid, action->id);
	pcmk__log_action("Unassigned", action, false);
	return false;
	}

	if (pcmk_is_set(action->flags, pcmk_action_on_dc)) {
	crm_trace("Action %s (%d) should be dumped: "
	"can run on DC instead of %s",
	action->uuid, action->id, pcmk__node_name(action->node));

	} else if (pe__is_guest_node(action->node)
	&& !action->node->details->remote_requires_reset) {
	crm_trace("Action %s (%d) should be dumped: "
	"assuming will be runnable on guest %s",
	action->uuid, action->id, pcmk__node_name(action->node));

	} else if (!action->node->details->online) {
	pcmk__sched_err("Skipping action %s (%d) "
	"because it was scheduled for offline node (bug?)",
	action->uuid, action->id);
	pcmk__log_action("Offline node", action, false);
	return false;

	} else if (action->node->details->unclean) {
	pcmk__sched_err("Skipping action %s (%d) "
	"because it was scheduled for unclean node (bug?)",
	action->uuid, action->id);
	pcmk__log_action("Unclean node", action, false);
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether an ordering's flags can change an action
	*
	* \param[in] ordering Ordering to check
	*
	* \return true if ordering has flags that can change an action, false otherwise
	*/
	static bool
	ordering_can_change_actions(const pcmk__related_action_t *ordering)
	{
	return pcmk_any_flags_set(ordering->type,
	~(pcmk__ar_then_implies_first_graphed
	\|pcmk__ar_first_implies_then_graphed
	\|pcmk__ar_ordered));
	}

	/*!
	* \internal
	* \brief Check whether an action input should be in the transition graph
	*
	* \param[in] action Action to check
	* \param[in,out] input Action input to check
	*
	* \return true if input should be in graph, false otherwise
	* \note This function may not only check an input, but disable it under certian
	* circumstances (load or anti-colocation orderings that are not needed).
	*/
	static bool
	should_add_input_to_graph(const pcmk_action_t *action,
	pcmk__related_action_t *input)
	{
	if (input->state == pe_link_dumped) {
	return true;
	}

	if ((uint32_t) input->type == pcmk__ar_none) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"ordering disabled",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable)
	&& !ordering_can_change_actions(input)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional and input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable)
	&& pcmk_is_set(input->type, pcmk__ar_min_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"minimum number of instances required but input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pcmk__ar_unmigratable_then_blocks)
	&& !pcmk_is_set(input->action->flags, pcmk_action_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"input blocked if 'then' unmigratable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pcmk__ar_if_first_unmigratable)
	&& pcmk_is_set(input->action->flags, pcmk_action_migratable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): ordering applies "
	"only if input is unmigratable, but it is migratable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (((uint32_t) input->type == pcmk__ar_ordered)
	&& pcmk_is_set(input->action->flags, pcmk_action_migratable)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional but stop in migration",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if ((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target) {
	pcmk_node_t *input_node = input->action->node;

	if ((action->rsc != NULL)
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {

	pcmk_node_t *assigned = action->rsc->allocated_to;

	/* For load_stopped -> migrate_to orderings, we care about where
	* the resource has been assigned, not where migrate_to will be
	* executed.
	*/
	- if (!pe__same_node(input_node, assigned)) {
	+ if (!pcmk__same_node(input_node, assigned)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"migration target %s is not same as input node %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(assigned? assigned->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;
	}

	- } else if (!pe__same_node(input_node, action->node)) {
	+ } else if (!pcmk__same_node(input_node, action->node)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"not on same node (%s vs %s)",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(action->node? action->node->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"ordering optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;
	}

	} else if ((uint32_t) input->type == pcmk__ar_if_required_on_same_node) {
	if (input->action->node && action->node
	- && !pe__same_node(input->action->node, action->node)) {
	+ && !pcmk__same_node(input->action->node, action->node)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"not on same node (%s vs %s)",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	pcmk__node_name(action->node),
	pcmk__node_name(input->action->node));
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;
	}

	} else if (input->action->rsc
	&& input->action->rsc != action->rsc
	&& pcmk_is_set(input->action->rsc->flags, pcmk_rsc_failed)
	&& !pcmk_is_set(input->action->rsc->flags, pcmk_rsc_managed)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")
	&& action->rsc && pe_rsc_is_clone(action->rsc)) {
	crm_warn("Ignoring requirement that %s complete before %s:"
	" unmanaged failed resources cannot prevent clone shutdown",
	input->action->uuid, action->uuid);
	return false;

	} else if (pcmk_is_set(input->action->flags, pcmk_action_optional)
	&& !pcmk_any_flags_set(input->action->flags,
	pcmk_action_always_in_graph
	\|pcmk_action_added_to_graph)
	&& !should_add_action_to_graph(input->action)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;
	}

	crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x",
	action->uuid, action->id, action_type_str(input->action->flags),
	input->action->uuid, input->action->id,
	action_node_str(input->action),
	action_runnable_str(input->action->flags),
	action_optional_str(input->action->flags), input->type);
	return true;
	}

	/*!
	* \internal
	* \brief Check whether an ordering creates an ordering loop
	*
	* \param[in] init_action "First" action in ordering
	* \param[in] action Callers should always set this the same as
	* \p init_action (this function may use a different
	* value for recursive calls)
	* \param[in,out] input Action wrapper for "then" action in ordering
	*
	* \return true if the ordering creates a loop, otherwise false
	*/
	bool
	pcmk__graph_has_loop(const pcmk_action_t *init_action,
	const pcmk_action_t action, pcmk__related_action_t input)
	{
	bool has_loop = false;

	if (pcmk_is_set(input->action->flags, pcmk_action_detect_loop)) {
	crm_trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	return false;
	}

	// Don't need to check inputs that won't be used
	if (!should_add_input_to_graph(action, input)) {
	return false;
	}

	if (input->action == init_action) {
	crm_debug("Input loop found in %s@%s ->...-> %s@%s",
	action->uuid,
	action->node? action->node->details->uname : "",
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");
	return true;
	}

	pcmk__set_action_flags(input->action, pcmk_action_detect_loop);

	crm_trace("Checking inputs of action %s@%s input %s@%s (%#.6x)"
	"for graph loop with %s@%s ",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	input->type,
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");

	// Recursively check input itself for loops
	for (GList *iter = input->action->actions_before;
	iter != NULL; iter = iter->next) {

	if (pcmk__graph_has_loop(init_action, input->action,
	(pcmk__related_action_t *) iter->data)) {
	// Recursive call already logged a debug message
	has_loop = true;
	break;
	}
	}

	pcmk__clear_action_flags(input->action, pcmk_action_detect_loop);

	if (!has_loop) {
	crm_trace("No input loop found in %s@%s -> %s@%s (%#.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	}
	return has_loop;
	}

	/*!
	* \internal
	* \brief Create a synapse XML element for a transition graph
	*
	* \param[in] action Action that synapse is for
	* \param[in,out] scheduler Scheduler data containing graph
	*
	* \return Newly added XML element for new graph synapse
	*/
	static xmlNode *
	create_graph_synapse(const pcmk_action_t action, pcmk_scheduler_t scheduler)
	{
	int synapse_priority = 0;
	xmlNode *syn = create_xml_node(scheduler->graph, "synapse");

	crm_xml_add_int(syn, PCMK_XA_ID, scheduler->num_synapse);
	scheduler->num_synapse++;

	if (action->rsc != NULL) {
	synapse_priority = action->rsc->priority;
	}
	if (action->priority > synapse_priority) {
	synapse_priority = action->priority;
	}
	if (synapse_priority > 0) {
	crm_xml_add_int(syn, PCMK__XA_PRIORITY, synapse_priority);
	}
	return syn;
	}

	/*!
	* \internal
	* \brief Add an action to the transition graph XML if appropriate
	*
	* \param[in,out] data Action to possibly add
	* \param[in,out] user_data Scheduler data
	*
	* \note This will de-duplicate the action inputs, meaning that the
	* pcmk__related_action_t:type flags can no longer be relied on to retain
	* their original settings. That means this MUST be called after
	* pcmk__apply_orderings() is complete, and nothing after this should rely
	* on those type flags. (For example, some code looks for type equal to
	* some flag rather than whether the flag is set, and some code looks for
	* particular combinations of flags -- such code must be done before
	* pcmk__create_graph().)
	*/
	static void
	add_action_to_graph(gpointer data, gpointer user_data)
	{
	pcmk_action_t action = (pcmk_action_t ) data;
	pcmk_scheduler_t scheduler = (pcmk_scheduler_t ) user_data;

	xmlNode *syn = NULL;
	xmlNode *set = NULL;
	xmlNode *in = NULL;

	/* If we haven't already, de-duplicate inputs (even if we won't be adding
	* the action to the graph, so that crm_simulate's dot graphs don't have
	* duplicates).
	*/
	if (!pcmk_is_set(action->flags, pcmk_action_inputs_deduplicated)) {
	pcmk__deduplicate_action_inputs(action);
	pcmk__set_action_flags(action, pcmk_action_inputs_deduplicated);
	}

	if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)
	\|\| !should_add_action_to_graph(action)) {
	return; // Already added, or shouldn't be
	}
	pcmk__set_action_flags(action, pcmk_action_added_to_graph);

	crm_trace("Adding action %d (%s%s%s) to graph",
	action->id, action->uuid,
	((action->node == NULL)? "" : " on "),
	((action->node == NULL)? "" : action->node->details->uname));

	syn = create_graph_synapse(action, scheduler);
	set = create_xml_node(syn, "action_set");
	in = create_xml_node(syn, "inputs");

	create_graph_action(set, action, false, scheduler);

	for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
	pcmk__related_action_t *input = lpc->data;

	if (should_add_input_to_graph(action, input)) {
	xmlNode *input_xml = create_xml_node(in, "trigger");

	input->state = pe_link_dumped;
	create_graph_action(input_xml, input->action, true, scheduler);
	}
	}
	}

	static int transition_id = -1;

	/*!
	* \internal
	* \brief Log a message after calculating a transition
	*
	* \param[in] filename Where transition input is stored
	*/
	void
	pcmk__log_transition_summary(const char *filename)
	{
	if (was_processing_error \|\| crm_config_error) {
	crm_err("Calculated transition %d (with errors)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else if (was_processing_warning \|\| crm_config_warning) {
	crm_warn("Calculated transition %d (with warnings)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else {
	crm_notice("Calculated transition %d%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);
	}
	if (crm_config_error) {
	crm_notice("Configuration errors found during scheduler processing,"
	" please run \"crm_verify -L\" to identify issues");
	}
	}

	/*!
	* \internal
	* \brief Add a resource's actions to the transition graph
	*
	* \param[in,out] rsc Resource whose actions should be added
	*/
	void
	pcmk__add_rsc_actions_to_graph(pcmk_resource_t *rsc)
	{
	GList *iter = NULL;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Adding actions for %s to graph", rsc->id);

	// First add the resource's own actions
	g_list_foreach(rsc->actions, add_action_to_graph, rsc->cluster);

	// Then recursively add its children's actions (appropriate to variant)
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) iter->data;

	child_rsc->cmds->add_actions_to_graph(child_rsc);
	}
	}

	/*!
	* \internal
	* \brief Create a transition graph with all cluster actions needed
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__create_graph(pcmk_scheduler_t *scheduler)
	{
	GList *iter = NULL;
	const char *value = NULL;
	long long limit = 0LL;

	transition_id++;
	crm_trace("Creating transition graph %d", transition_id);

	scheduler->graph = create_xml_node(NULL, XML_TAG_GRAPH);

	value = pe_pref(scheduler->config_hash, PCMK_OPT_CLUSTER_DELAY);
	crm_xml_add(scheduler->graph, PCMK_OPT_CLUSTER_DELAY, value);

	value = pe_pref(scheduler->config_hash, PCMK_OPT_STONITH_TIMEOUT);
	crm_xml_add(scheduler->graph, PCMK_OPT_STONITH_TIMEOUT, value);

	crm_xml_add(scheduler->graph, "failed-stop-offset", "INFINITY");

	if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
	crm_xml_add(scheduler->graph, "failed-start-offset", "INFINITY");
	} else {
	crm_xml_add(scheduler->graph, "failed-start-offset", "1");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_BATCH_LIMIT);
	crm_xml_add(scheduler->graph, PCMK_OPT_BATCH_LIMIT, value);

	crm_xml_add_int(scheduler->graph, "transition_id", transition_id);

	value = pe_pref(scheduler->config_hash, PCMK_OPT_MIGRATION_LIMIT);
	if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
	crm_xml_add(scheduler->graph, PCMK_OPT_MIGRATION_LIMIT, value);
	}

	if (scheduler->recheck_by > 0) {
	char *recheck_epoch = NULL;

	recheck_epoch = crm_strdup_printf("%llu",
	(long long) scheduler->recheck_by);
	crm_xml_add(scheduler->graph, "recheck-by", recheck_epoch);
	free(recheck_epoch);
	}

	/* The following code will de-duplicate action inputs, so nothing past this
	* should rely on the action input type flags retaining their original
	* values.
	*/

	// Add resource actions to graph
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	pcmk__rsc_trace(rsc, "Processing actions for %s", rsc->id);
	rsc->cmds->add_actions_to_graph(rsc);
	}

	// Add pseudo-action for list of nodes with maintenance state update
	add_maintenance_update(scheduler);

	// Add non-resource (node) actions
	for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;

	if ((action->rsc != NULL)
	&& (action->node != NULL)
	&& action->node->details->shutdown
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)
	&& !pcmk_any_flags_set(action->flags,
	pcmk_action_optional\|pcmk_action_runnable)
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	/* Eventually we should just ignore the 'fence' case, but for now
	* it's the best way to detect (in CTS) when CIB resource updates
	* are being lost.
	*/
	if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	\|\| (scheduler->no_quorum_policy == pcmk_no_quorum_ignore)) {
	const bool managed = pcmk_is_set(action->rsc->flags,
	pcmk_rsc_managed);
	const bool failed = pcmk_is_set(action->rsc->flags,
	pcmk_rsc_failed);

	crm_crit("Cannot %s %s because of %s:%s%s (%s)",
	action->node->details->unclean? "fence" : "shut down",
	pcmk__node_name(action->node), action->rsc->id,
	(managed? " blocked" : " unmanaged"),
	(failed? " failed" : ""), action->uuid);
	}
	}

	add_action_to_graph((gpointer) action, (gpointer) scheduler);
	}

	crm_log_xml_trace(scheduler->graph, "graph");
	}
	diff --git a/lib/pacemaker/pcmk_output.c b/lib/pacemaker/pcmk_output.c
	index 926e7e7bdf..63947a1d52 100644
	--- a/lib/pacemaker/pcmk_output.c
	+++ b/lib/pacemaker/pcmk_output.c
	@@ -1,2405 +1,2405 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/msg_xml.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/pengine/internal.h>
	#include <libxml/tree.h>
	#include <pacemaker-internal.h>

	#include <inttypes.h>
	#include <stdint.h>

	static char *
	colocations_header(pcmk_resource_t rsc, pcmk__colocation_t cons,
	bool dependents) {
	char *retval = NULL;

	if (cons->primary_role > pcmk_role_started) {
	retval = crm_strdup_printf("%s (score=%s, %s role=%s, id=%s)",
	rsc->id, pcmk_readable_score(cons->score),
	(dependents? "needs" : "with"),
	role2text(cons->primary_role), cons->id);
	} else {
	retval = crm_strdup_printf("%s (score=%s, id=%s)",
	rsc->id, pcmk_readable_score(cons->score),
	cons->id);
	}
	return retval;
	}

	static void
	colocations_xml_node(pcmk__output_t out, pcmk_resource_t rsc,
	pcmk__colocation_t *cons) {
	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, XML_CONS_TAG_RSC_DEPEND,
	PCMK_XA_ID, cons->id,
	"rsc", cons->dependent->id,
	"with-rsc", cons->primary->id,
	PCMK_XA_SCORE,
	pcmk_readable_score(cons->score),
	NULL);

	if (cons->node_attribute) {
	xmlSetProp(node, (pcmkXmlStr) "node-attribute",
	(pcmkXmlStr) cons->node_attribute);
	}

	if (cons->dependent_role != pcmk_role_unknown) {
	xmlSetProp(node, (pcmkXmlStr) "rsc-role",
	(pcmkXmlStr) role2text(cons->dependent_role));
	}

	if (cons->primary_role != pcmk_role_unknown) {
	xmlSetProp(node, (pcmkXmlStr) "with-rsc-role",
	(pcmkXmlStr) role2text(cons->primary_role));
	}
	}

	static int
	do_locations_list_xml(pcmk__output_t out, pcmk_resource_t rsc,
	bool add_header)
	{
	GList *lpc = NULL;
	GList *list = rsc->rsc_location;
	int rc = pcmk_rc_no_output;

	for (lpc = list; lpc != NULL; lpc = lpc->next) {
	pcmk__location_t *cons = lpc->data;

	GList *lpc2 = NULL;

	for (lpc2 = cons->nodes; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_node_t node = (pcmk_node_t ) lpc2->data;

	if (add_header) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "locations");
	}

	pcmk__output_create_xml_node(out, XML_CONS_TAG_RSC_LOCATION,
	"node", node->details->uname,
	"rsc", rsc->id,
	PCMK_XA_ID, cons->id,
	PCMK_XA_SCORE,
	pcmk_readable_score(node->weight),
	NULL);
	}
	}

	if (add_header) {
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("rsc-action-item", "const char ", "pcmk_resource_t ",
	"pcmk_node_t ", "pcmk_node_t ", "pcmk_action_t *",
	"pcmk_action_t *")
	static int
	rsc_action_item(pcmk__output_t *out, va_list args)
	{
	const char change = va_arg(args, const char );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t origin = va_arg(args, pcmk_node_t );
	pcmk_node_t destination = va_arg(args, pcmk_node_t );
	pcmk_action_t action = va_arg(args, pcmk_action_t );
	pcmk_action_t source = va_arg(args, pcmk_action_t );

	int len = 0;
	char *reason = NULL;
	char *details = NULL;
	bool same_host = false;
	bool same_role = false;
	bool need_role = false;

	static int rsc_width = 5;
	static int detail_width = 5;

	CRM_ASSERT(action);
	CRM_ASSERT(destination != NULL \|\| origin != NULL);

	if (source == NULL) {
	source = action;
	}

	len = strlen(rsc->id);
	if (len > rsc_width) {
	rsc_width = len + 2;
	}

	if ((rsc->role > pcmk_role_started)
	\|\| (rsc->next_role > pcmk_role_unpromoted)) {
	need_role = true;
	}

	- if (pe__same_node(origin, destination)) {
	+ if (pcmk__same_node(origin, destination)) {
	same_host = true;
	}

	if (rsc->role == rsc->next_role) {
	same_role = true;
	}

	if (need_role && (origin == NULL)) {
	/* Starting and promoting a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role),
	role2text(rsc->next_role),
	pcmk__node_name(destination));

	} else if (origin == NULL) {
	/* Starting a resource */
	details = crm_strdup_printf("%s", pcmk__node_name(destination));

	} else if (need_role && (destination == NULL)) {
	/* Stopping a promotable clone instance */
	details = crm_strdup_printf("%s %s", role2text(rsc->role),
	pcmk__node_name(origin));

	} else if (destination == NULL) {
	/* Stopping a resource */
	details = crm_strdup_printf("%s", pcmk__node_name(origin));

	} else if (need_role && same_role && same_host) {
	/* Recovering, restarting or re-promoting a promotable clone instance */
	details = crm_strdup_printf("%s %s", role2text(rsc->role),
	pcmk__node_name(origin));

	} else if (same_role && same_host) {
	/* Recovering or Restarting a normal resource */
	details = crm_strdup_printf("%s", pcmk__node_name(origin));

	} else if (need_role && same_role) {
	/* Moving a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", pcmk__node_name(origin),
	pcmk__node_name(destination),
	role2text(rsc->role));

	} else if (same_role) {
	/* Moving a normal resource */
	details = crm_strdup_printf("%s -> %s", pcmk__node_name(origin),
	pcmk__node_name(destination));

	} else if (same_host) {
	/* Promoting or demoting a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role),
	role2text(rsc->next_role),
	pcmk__node_name(origin));

	} else {
	/* Moving and promoting/demoting */
	details = crm_strdup_printf("%s %s -> %s %s", role2text(rsc->role),
	pcmk__node_name(origin),
	role2text(rsc->next_role),
	pcmk__node_name(destination));
	}

	len = strlen(details);
	if (len > detail_width) {
	detail_width = len;
	}

	if ((source->reason != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_runnable)) {
	reason = crm_strdup_printf("due to %s (blocked)", source->reason);

	} else if (source->reason) {
	reason = crm_strdup_printf("due to %s", source->reason);

	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	reason = strdup("blocked");

	}

	out->list_item(out, NULL, "%-8s %-s ( %s )%s%s",
	change, rsc_width, rsc->id, detail_width, details,
	((reason == NULL)? "" : " "), pcmk__s(reason, ""));

	free(details);
	free(reason);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("rsc-action-item", "const char ", "pcmk_resource_t ",
	"pcmk_node_t ", "pcmk_node_t ", "pcmk_action_t *",
	"pcmk_action_t *")
	static int
	rsc_action_item_xml(pcmk__output_t *out, va_list args)
	{
	const char change = va_arg(args, const char );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t origin = va_arg(args, pcmk_node_t );
	pcmk_node_t destination = va_arg(args, pcmk_node_t );
	pcmk_action_t action = va_arg(args, pcmk_action_t );
	pcmk_action_t source = va_arg(args, pcmk_action_t );

	char *change_str = NULL;

	bool same_host = false;
	bool same_role = false;
	bool need_role = false;
	xmlNode *xml = NULL;

	CRM_ASSERT(action);
	CRM_ASSERT(destination != NULL \|\| origin != NULL);

	if (source == NULL) {
	source = action;
	}

	if ((rsc->role > pcmk_role_started)
	\|\| (rsc->next_role > pcmk_role_unpromoted)) {
	need_role = true;
	}

	- if (pe__same_node(origin, destination)) {
	+ if (pcmk__same_node(origin, destination)) {
	same_host = true;
	}

	if (rsc->role == rsc->next_role) {
	same_role = true;
	}

	change_str = g_ascii_strdown(change, -1);
	xml = pcmk__output_create_xml_node(out, "rsc_action",
	"action", change_str,
	"resource", rsc->id,
	NULL);
	g_free(change_str);

	if (need_role && (origin == NULL)) {
	/* Starting and promoting a promotable clone instance */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE, role2text(rsc->role),
	"next-role", role2text(rsc->next_role),
	"dest", destination->details->uname,
	NULL);

	} else if (origin == NULL) {
	/* Starting a resource */
	crm_xml_add(xml, "node", destination->details->uname);

	} else if (need_role && (destination == NULL)) {
	/* Stopping a promotable clone instance */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE, role2text(rsc->role),
	"node", origin->details->uname,
	NULL);

	} else if (destination == NULL) {
	/* Stopping a resource */
	crm_xml_add(xml, "node", origin->details->uname);

	} else if (need_role && same_role && same_host) {
	/* Recovering, restarting or re-promoting a promotable clone instance */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE, role2text(rsc->role),
	"source", origin->details->uname,
	NULL);

	} else if (same_role && same_host) {
	/* Recovering or Restarting a normal resource */
	crm_xml_add(xml, "source", origin->details->uname);

	} else if (need_role && same_role) {
	/* Moving a promotable clone instance */
	pcmk__xe_set_props(xml,
	"source", origin->details->uname,
	"dest", destination->details->uname,
	PCMK_XA_ROLE, role2text(rsc->role),
	NULL);

	} else if (same_role) {
	/* Moving a normal resource */
	pcmk__xe_set_props(xml,
	"source", origin->details->uname,
	"dest", destination->details->uname,
	NULL);

	} else if (same_host) {
	/* Promoting or demoting a promotable clone instance */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE, role2text(rsc->role),
	"next-role", role2text(rsc->next_role),
	"source", origin->details->uname,
	NULL);

	} else {
	/* Moving and promoting/demoting */
	pcmk__xe_set_props(xml,
	PCMK_XA_ROLE, role2text(rsc->role),
	"source", origin->details->uname,
	"next-role", role2text(rsc->next_role),
	"dest", destination->details->uname,
	NULL);
	}

	if ((source->reason != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_runnable)) {
	pcmk__xe_set_props(xml,
	PCMK_XA_REASON, source->reason,
	"blocked", PCMK_VALUE_TRUE,
	NULL);

	} else if (source->reason != NULL) {
	crm_xml_add(xml, PCMK_XA_REASON, source->reason);

	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	pcmk__xe_set_bool_attr(xml, "blocked", true);

	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("rsc-is-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rsc_is_colocated_with_list(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use rsc->rsc_cons
	* directly rather than rsc->cmds->this_with_colocations().
	*/
	pcmk__set_rsc_flags(rsc, pcmk_rsc_detect_loop);
	for (GList *lpc = rsc->rsc_cons; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;
	char *hdr = NULL;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc,
	"Resources %s is colocated with", rsc->id);

	if (pcmk_is_set(cons->primary->flags, pcmk_rsc_detect_loop)) {
	out->list_item(out, NULL, "%s (id=%s - loop)",
	cons->primary->id, cons->id);
	continue;
	}

	hdr = colocations_header(cons->primary, cons, false);
	out->list_item(out, NULL, "%s", hdr);
	free(hdr);

	// Empty list header for indentation of information about this resource
	out->begin_list(out, NULL, NULL, NULL);

	out->message(out, "locations-list", cons->primary);
	if (recursive) {
	out->message(out, "rsc-is-colocated-with-list",
	cons->primary, recursive);
	}

	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("rsc-is-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rsc_is_colocated_with_list_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use rsc->rsc_cons
	* directly rather than rsc->cmds->this_with_colocations().
	*/
	pcmk__set_rsc_flags(rsc, pcmk_rsc_detect_loop);
	for (GList *lpc = rsc->rsc_cons; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	if (pcmk_is_set(cons->primary->flags, pcmk_rsc_detect_loop)) {
	colocations_xml_node(out, cons->primary, cons);
	continue;
	}

	colocations_xml_node(out, cons->primary, cons);
	do_locations_list_xml(out, cons->primary, false);

	if (recursive) {
	out->message(out, "rsc-is-colocated-with-list",
	cons->primary, recursive);
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("rscs-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rscs_colocated_with_list(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use
	* rsc->rsc_cons_lhs directly rather than
	* rsc->cmds->with_this_colocations().
	*/
	pcmk__set_rsc_flags(rsc, pcmk_rsc_detect_loop);
	for (GList *lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;
	char *hdr = NULL;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources colocated with %s",
	rsc->id);

	if (pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)) {
	out->list_item(out, NULL, "%s (id=%s - loop)",
	cons->dependent->id, cons->id);
	continue;
	}

	hdr = colocations_header(cons->dependent, cons, true);
	out->list_item(out, NULL, "%s", hdr);
	free(hdr);

	// Empty list header for indentation of information about this resource
	out->begin_list(out, NULL, NULL, NULL);

	out->message(out, "locations-list", cons->dependent);
	if (recursive) {
	out->message(out, "rscs-colocated-with-list",
	cons->dependent, recursive);
	}

	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("rscs-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rscs_colocated_with_list_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use
	* rsc->rsc_cons_lhs directly rather than
	* rsc->cmds->with_this_colocations().
	*/
	pcmk__set_rsc_flags(rsc, pcmk_rsc_detect_loop);
	for (GList *lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	if (pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)) {
	colocations_xml_node(out, cons->dependent, cons);
	continue;
	}

	colocations_xml_node(out, cons->dependent, cons);
	do_locations_list_xml(out, cons->dependent, false);

	if (recursive) {
	out->message(out, "rscs-colocated-with-list",
	cons->dependent, recursive);
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("locations-list", "pcmk_resource_t *")
	static int
	locations_list(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );

	GList *lpc = NULL;
	GList *list = rsc->rsc_location;
	int rc = pcmk_rc_no_output;

	for (lpc = list; lpc != NULL; lpc = lpc->next) {
	pcmk__location_t *cons = lpc->data;

	GList *lpc2 = NULL;

	for (lpc2 = cons->nodes; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_node_t node = (pcmk_node_t ) lpc2->data;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Locations");
	out->list_item(out, NULL, "Node %s (score=%s, id=%s, rsc=%s)",
	pcmk__node_name(node),
	pcmk_readable_score(node->weight), cons->id,
	rsc->id);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("locations-list", "pcmk_resource_t *")
	static int
	locations_list_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	return do_locations_list_xml(out, rsc, true);
	}

	PCMK__OUTPUT_ARGS("locations-and-colocations", "pcmk_resource_t *",
	"bool", "bool")
	static int
	locations_and_colocations(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);
	bool force = va_arg(args, int);

	pcmk__unpack_constraints(rsc->cluster);

	// Constraints apply to group/clone, not member/instance
	if (!force) {
	rsc = uber_parent(rsc);
	}

	out->message(out, "locations-list", rsc);

	pe__clear_resource_flags_on_all(rsc->cluster, pcmk_rsc_detect_loop);
	out->message(out, "rscs-colocated-with-list", rsc, recursive);

	pe__clear_resource_flags_on_all(rsc->cluster, pcmk_rsc_detect_loop);
	out->message(out, "rsc-is-colocated-with-list", rsc, recursive);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("locations-and-colocations", "pcmk_resource_t *",
	"bool", "bool")
	static int
	locations_and_colocations_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);
	bool force = va_arg(args, int);

	pcmk__unpack_constraints(rsc->cluster);

	// Constraints apply to group/clone, not member/instance
	if (!force) {
	rsc = uber_parent(rsc);
	}

	pcmk__output_xml_create_parent(out, "constraints", NULL);
	do_locations_list_xml(out, rsc, false);

	pe__clear_resource_flags_on_all(rsc->cluster, pcmk_rsc_detect_loop);
	out->message(out, "rscs-colocated-with-list", rsc, recursive);

	pe__clear_resource_flags_on_all(rsc->cluster, pcmk_rsc_detect_loop);
	out->message(out, "rsc-is-colocated-with-list", rsc, recursive);

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("health", "const char ", "const char ", "const char *",
	"const char *")
	static int
	health(pcmk__output_t *out, va_list args)
	{
	const char sys_from G_GNUC_UNUSED = va_arg(args, const char );
	const char host_from = va_arg(args, const char );
	const char fsa_state = va_arg(args, const char );
	const char result = va_arg(args, const char );

	return out->info(out, "Controller on %s in state %s: %s",
	pcmk__s(host_from, "unknown node"),
	pcmk__s(fsa_state, "unknown"),
	pcmk__s(result, "unknown result"));
	}

	PCMK__OUTPUT_ARGS("health", "const char ", "const char ", "const char *",
	"const char *")
	static int
	health_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return health(out, args);
	} else {
	const char sys_from G_GNUC_UNUSED = va_arg(args, const char );
	const char host_from G_GNUC_UNUSED = va_arg(args, const char );
	const char fsa_state = va_arg(args, const char );
	const char result G_GNUC_UNUSED = va_arg(args, const char );

	if (fsa_state != NULL) {
	pcmk__formatted_printf(out, "%s\n", fsa_state);
	return pcmk_rc_ok;
	}
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("health", "const char ", "const char ", "const char *",
	"const char *")
	static int
	health_xml(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	const char host_from = va_arg(args, const char );
	const char fsa_state = va_arg(args, const char );
	const char result = va_arg(args, const char );

	pcmk__output_create_xml_node(out, pcmk__s(sys_from, ""),
	"node_name", pcmk__s(host_from, ""),
	"state", pcmk__s(fsa_state, ""),
	PCMK_XA_RESULT, pcmk__s(result, ""),
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated = va_arg(args, time_t);

	char *last_updated_s = NULL;
	int rc = pcmk_rc_ok;

	if (sys_from == NULL) {
	if (state == pcmk_pacemakerd_state_remote) {
	sys_from = "pacemaker-remoted";
	} else {
	sys_from = CRM_SYSTEM_MCP;
	}
	}

	if (state_s == NULL) {
	state_s = pcmk__pcmkd_state_enum2friendly(state);
	}

	if (last_updated != 0) {
	last_updated_s = pcmk__epoch2str(&last_updated,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);
	}

	rc = out->info(out, "Status of %s: '%s' (last updated %s)",
	sys_from, state_s,
	pcmk__s(last_updated_s, "at unknown time"));

	free(last_updated_s);
	return rc;
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health_html(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated = va_arg(args, time_t);

	char *last_updated_s = NULL;
	char *msg = NULL;

	if (sys_from == NULL) {
	if (state == pcmk_pacemakerd_state_remote) {
	sys_from = "pacemaker-remoted";
	} else {
	sys_from = CRM_SYSTEM_MCP;
	}
	}

	if (state_s == NULL) {
	state_s = pcmk__pcmkd_state_enum2friendly(state);
	}

	if (last_updated != 0) {
	last_updated_s = pcmk__epoch2str(&last_updated,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);
	}

	msg = crm_strdup_printf("Status of %s: '%s' (last updated %s)",
	sys_from, state_s,
	pcmk__s(last_updated_s, "at unknown time"));
	pcmk__output_create_html_node(out, "li", NULL, NULL, msg);

	free(msg);
	free(last_updated_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return pacemakerd_health(out, args);
	} else {
	const char sys_from G_GNUC_UNUSED = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated G_GNUC_UNUSED = va_arg(args, time_t);

	if (state_s == NULL) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
	}
	pcmk__formatted_printf(out, "%s\n", state_s);
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health_xml(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated = va_arg(args, time_t);

	char *last_updated_s = NULL;

	if (sys_from == NULL) {
	if (state == pcmk_pacemakerd_state_remote) {
	sys_from = "pacemaker-remoted";
	} else {
	sys_from = CRM_SYSTEM_MCP;
	}
	}

	if (state_s == NULL) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
	}

	if (last_updated != 0) {
	last_updated_s = pcmk__epoch2str(&last_updated,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);
	}

	pcmk__output_create_xml_node(out, "pacemakerd",
	"sys_from", sys_from,
	"state", state_s,
	"last_updated", last_updated_s,
	NULL);
	free(last_updated_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("profile", "const char *", "clock_t", "clock_t")
	static int
	profile_default(pcmk__output_t *out, va_list args) {
	const char xml_file = va_arg(args, const char );
	clock_t start = va_arg(args, clock_t);
	clock_t end = va_arg(args, clock_t);

	out->list_item(out, NULL, "Testing %s ... %.2f secs", xml_file,
	(end - start) / (float) CLOCKS_PER_SEC);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("profile", "const char *", "clock_t", "clock_t")
	static int
	profile_xml(pcmk__output_t *out, va_list args) {
	const char xml_file = va_arg(args, const char );
	clock_t start = va_arg(args, clock_t);
	clock_t end = va_arg(args, clock_t);

	char *duration = pcmk__ftoa((end - start) / (float) CLOCKS_PER_SEC);

	pcmk__output_create_xml_node(out, "timing",
	"file", xml_file,
	"duration", duration,
	NULL);

	free(duration);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("dc", "const char *")
	static int
	dc(pcmk__output_t *out, va_list args)
	{
	const char dc = va_arg(args, const char );

	return out->info(out, "Designated Controller is: %s",
	pcmk__s(dc, "not yet elected"));
	}

	PCMK__OUTPUT_ARGS("dc", "const char *")
	static int
	dc_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return dc(out, args);
	} else {
	const char dc = va_arg(args, const char );

	if (dc != NULL) {
	pcmk__formatted_printf(out, "%s\n", pcmk__s(dc, ""));
	return pcmk_rc_ok;
	}
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("dc", "const char *")
	static int
	dc_xml(pcmk__output_t *out, va_list args)
	{
	const char dc = va_arg(args, const char );

	pcmk__output_create_xml_node(out, "dc",
	"node_name", pcmk__s(dc, ""),
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("crmadmin-node", "const char ", "const char ",
	"const char *", "bool")
	static int
	crmadmin_node(pcmk__output_t *out, va_list args)
	{
	const char type = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char id = va_arg(args, const char );
	bool bash_export = va_arg(args, int);

	if (bash_export) {
	return out->info(out, "export %s=%s",
	pcmk__s(name, "<null>"), pcmk__s(id, ""));
	} else {
	return out->info(out, "%s node: %s (%s)", type ? type : "cluster",
	pcmk__s(name, "<null>"), pcmk__s(id, "<null>"));
	}
	}

	PCMK__OUTPUT_ARGS("crmadmin-node", "const char ", "const char ",
	"const char *", "bool")
	static int
	crmadmin_node_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return crmadmin_node(out, args);
	} else {
	const char type G_GNUC_UNUSED = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char id G_GNUC_UNUSED = va_arg(args, const char );
	bool bash_export G_GNUC_UNUSED = va_arg(args, int);

	pcmk__formatted_printf(out, "%s\n", pcmk__s(name, "<null>"));
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("crmadmin-node", "const char ", "const char ",
	"const char *", "bool")
	static int
	crmadmin_node_xml(pcmk__output_t *out, va_list args)
	{
	const char type = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char id = va_arg(args, const char );
	bool bash_export G_GNUC_UNUSED = va_arg(args, int);

	pcmk__output_create_xml_node(out, "node",
	PCMK_XA_TYPE, pcmk__s(type, "cluster"),
	PCMK_XA_NAME, pcmk__s(name, ""),
	PCMK_XA_ID, pcmk__s(id, ""),
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("digests", "const pcmk_resource_t ", "const pcmk_node_t ",
	"const char ", "guint", "const pcmk__op_digest_t ")
	static int
	digests_text(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const pcmk_node_t node = va_arg(args, const pcmk_node_t );
	const char task = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);
	const pcmk__op_digest_t digests = va_arg(args, const pcmk__op_digest_t );

	char *action_desc = NULL;
	const char *rsc_desc = "unknown resource";
	const char *node_desc = "unknown node";

	if (interval_ms != 0) {
	action_desc = crm_strdup_printf("%ums-interval %s action", interval_ms,
	((task == NULL)? "unknown" : task));
	} else if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	action_desc = strdup("probe action");
	} else {
	action_desc = crm_strdup_printf("%s action",
	((task == NULL)? "unknown" : task));
	}
	if ((rsc != NULL) && (rsc->id != NULL)) {
	rsc_desc = rsc->id;
	}
	if ((node != NULL) && (node->details->uname != NULL)) {
	node_desc = node->details->uname;
	}
	out->begin_list(out, NULL, NULL, "Digests for %s %s on %s",
	rsc_desc, action_desc, node_desc);
	free(action_desc);

	if (digests == NULL) {
	out->list_item(out, NULL, "none");
	out->end_list(out);
	return pcmk_rc_ok;
	}
	if (digests->digest_all_calc != NULL) {
	out->list_item(out, NULL, "%s (all parameters)",
	digests->digest_all_calc);
	}
	if (digests->digest_secure_calc != NULL) {
	out->list_item(out, NULL, "%s (non-private parameters)",
	digests->digest_secure_calc);
	}
	if (digests->digest_restart_calc != NULL) {
	out->list_item(out, NULL, "%s (non-reloadable parameters)",
	digests->digest_restart_calc);
	}
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static void
	add_digest_xml(xmlNode parent, const char type, const char *digest,
	xmlNode *digest_source)
	{
	if (digest != NULL) {
	xmlNodePtr digest_xml = create_xml_node(parent, "digest");

	crm_xml_add(digest_xml, PCMK_XA_TYPE, pcmk__s(type, "unspecified"));
	crm_xml_add(digest_xml, "hash", digest);
	if (digest_source != NULL) {
	add_node_copy(digest_xml, digest_source);
	}
	}
	}

	PCMK__OUTPUT_ARGS("digests", "const pcmk_resource_t ", "const pcmk_node_t ",
	"const char ", "guint", "const pcmk__op_digest_t ")
	static int
	digests_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const pcmk_node_t node = va_arg(args, const pcmk_node_t );
	const char task = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);
	const pcmk__op_digest_t digests = va_arg(args, const pcmk__op_digest_t );

	char *interval_s = crm_strdup_printf("%ums", interval_ms);
	xmlNode *xml = NULL;

	xml = pcmk__output_create_xml_node(out, "digests",
	"resource", pcmk__s(rsc->id, ""),
	"node",
	pcmk__s(node->details->uname, ""),
	"task", pcmk__s(task, ""),
	"interval", interval_s,
	NULL);
	free(interval_s);
	if (digests != NULL) {
	add_digest_xml(xml, "all", digests->digest_all_calc,
	digests->params_all);
	add_digest_xml(xml, "nonprivate", digests->digest_secure_calc,
	digests->params_secure);
	add_digest_xml(xml, "nonreloadable", digests->digest_restart_calc,
	digests->params_restart);
	}
	return pcmk_rc_ok;
	}

	#define STOP_SANITY_ASSERT(lineno) do { \
	if ((current != NULL) && current->details->unclean) { \
	/* It will be a pseudo op */ \
	} else if (stop == NULL) { \
	crm_err("%s:%d: No stop action exists for %s", \
	__func__, lineno, rsc->id); \
	CRM_ASSERT(stop != NULL); \
	} else if (pcmk_is_set(stop->flags, pcmk_action_optional)) { \
	crm_err("%s:%d: Action %s is still optional", \
	__func__, lineno, stop->uuid); \
	CRM_ASSERT(!pcmk_is_set(stop->flags, pcmk_action_optional));\
	} \
	} while (0)

	PCMK__OUTPUT_ARGS("rsc-action", "pcmk_resource_t ", "pcmk_node_t ",
	"pcmk_node_t *")
	static int
	rsc_action_default(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t current = va_arg(args, pcmk_node_t );
	pcmk_node_t next = va_arg(args, pcmk_node_t );

	GList *possible_matches = NULL;
	char *key = NULL;
	int rc = pcmk_rc_no_output;
	bool moving = false;

	pcmk_node_t *start_node = NULL;
	pcmk_action_t *start = NULL;
	pcmk_action_t *stop = NULL;
	pcmk_action_t *promote = NULL;
	pcmk_action_t *demote = NULL;
	pcmk_action_t *reason_op = NULL;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)
	\|\| (current == NULL && next == NULL)) {
	const bool managed = pcmk_is_set(rsc->flags, pcmk_rsc_managed);

	pcmk__rsc_info(rsc, "Leave %s\t(%s%s)",
	rsc->id, role2text(rsc->role),
	(managed? "" : " unmanaged"));
	return rc;
	}

	moving = (current != NULL) && (next != NULL)
	- && !pe__same_node(current, next);
	+ && !pcmk__same_node(current, next);

	possible_matches = pe__resource_actions(rsc, next, PCMK_ACTION_START,
	false);
	if (possible_matches) {
	start = possible_matches->data;
	g_list_free(possible_matches);
	}

	if ((start == NULL)
	\|\| !pcmk_is_set(start->flags, pcmk_action_runnable)) {
	start_node = NULL;
	} else {
	start_node = current;
	}
	possible_matches = pe__resource_actions(rsc, start_node, PCMK_ACTION_STOP,
	false);
	if (possible_matches) {
	stop = possible_matches->data;
	g_list_free(possible_matches);
	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_stop_unexpected)) {
	/* The resource is multiply active with PCMK_META_MULTIPLE_ACTIVE set to
	* stop_unexpected, and not stopping on its current node, but it should
	* be stopping elsewhere.
	*/
	possible_matches = pe__resource_actions(rsc, NULL, PCMK_ACTION_STOP,
	false);
	if (possible_matches != NULL) {
	stop = possible_matches->data;
	g_list_free(possible_matches);
	}
	}

	possible_matches = pe__resource_actions(rsc, next, PCMK_ACTION_PROMOTE,
	false);
	if (possible_matches) {
	promote = possible_matches->data;
	g_list_free(possible_matches);
	}

	possible_matches = pe__resource_actions(rsc, next, PCMK_ACTION_DEMOTE,
	false);
	if (possible_matches) {
	demote = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (rsc->role == rsc->next_role) {
	pcmk_action_t *migrate_op = NULL;

	CRM_CHECK(next != NULL, return rc);

	possible_matches = pe__resource_actions(rsc, next,
	PCMK_ACTION_MIGRATE_FROM,
	false);
	if (possible_matches) {
	migrate_op = possible_matches->data;
	}

	if ((migrate_op != NULL) && (current != NULL)
	&& pcmk_is_set(migrate_op->flags, pcmk_action_runnable)) {
	rc = out->message(out, "rsc-action-item", "Migrate", rsc, current,
	next, start, NULL);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_reload)) {
	rc = out->message(out, "rsc-action-item", "Reload", rsc, current,
	next, start, NULL);

	} else if ((start == NULL)
	\|\| pcmk_is_set(start->flags, pcmk_action_optional)) {
	if ((demote != NULL) && (promote != NULL)
	&& !pcmk_is_set(demote->flags, pcmk_action_optional)
	&& !pcmk_is_set(promote->flags, pcmk_action_optional)) {
	rc = out->message(out, "rsc-action-item", "Re-promote", rsc,
	current, next, promote, demote);
	} else {
	pcmk__rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id,
	role2text(rsc->role), pcmk__node_name(next));
	}

	} else if (!pcmk_is_set(start->flags, pcmk_action_runnable)) {
	if ((stop == NULL) \|\| (stop->reason == NULL)) {
	reason_op = start;
	} else {
	reason_op = stop;
	}
	rc = out->message(out, "rsc-action-item", "Stop", rsc, current,
	NULL, stop, reason_op);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (moving && current) {
	const bool failed = pcmk_is_set(rsc->flags, pcmk_rsc_failed);

	rc = out->message(out, "rsc-action-item",
	(failed? "Recover" : "Move"), rsc, current, next,
	stop, NULL);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	rc = out->message(out, "rsc-action-item", "Recover", rsc, current,
	NULL, stop, NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else {
	rc = out->message(out, "rsc-action-item", "Restart", rsc, current,
	next, start, NULL);
	#if 0
	/* @TODO This can be reached in situations that should really be
	* "Start" (see for example the migrate-fail-7 regression test)
	*/
	STOP_SANITY_ASSERT(__LINE__);
	#endif
	}

	g_list_free(possible_matches);
	return rc;
	}

	if ((stop != NULL)
	&& ((rsc->next_role == pcmk_role_stopped)
	\|\| ((start != NULL)
	&& !pcmk_is_set(start->flags, pcmk_action_runnable)))) {

	key = stop_key(rsc);
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t *node = iter->data;
	pcmk_action_t *stop_op = NULL;

	reason_op = start;
	possible_matches = find_actions(rsc->actions, key, node);
	if (possible_matches) {
	stop_op = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (stop_op != NULL) {
	if (pcmk_is_set(stop_op->flags, pcmk_action_runnable)) {
	STOP_SANITY_ASSERT(__LINE__);
	}
	if (stop_op->reason != NULL) {
	reason_op = stop_op;
	}
	}

	if (out->message(out, "rsc-action-item", "Stop", rsc, node, NULL,
	stop_op, reason_op) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	free(key);

	} else if ((stop != NULL)
	&& pcmk_all_flags_set(rsc->flags,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed)) {
	/* 'stop' may be NULL if the failure was ignored */
	rc = out->message(out, "rsc-action-item", "Recover", rsc, current,
	next, stop, start);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (moving) {
	rc = out->message(out, "rsc-action-item", "Move", rsc, current, next,
	stop, NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_reload)) {
	rc = out->message(out, "rsc-action-item", "Reload", rsc, current, next,
	start, NULL);

	} else if ((stop != NULL)
	&& !pcmk_is_set(stop->flags, pcmk_action_optional)) {
	rc = out->message(out, "rsc-action-item", "Restart", rsc, current,
	next, start, NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (rsc->role == pcmk_role_promoted) {
	CRM_LOG_ASSERT(current != NULL);
	rc = out->message(out, "rsc-action-item", "Demote", rsc, current,
	next, demote, NULL);

	} else if (rsc->next_role == pcmk_role_promoted) {
	CRM_LOG_ASSERT(next);
	rc = out->message(out, "rsc-action-item", "Promote", rsc, current,
	next, promote, NULL);

	} else if ((rsc->role == pcmk_role_stopped)
	&& (rsc->next_role > pcmk_role_stopped)) {
	rc = out->message(out, "rsc-action-item", "Start", rsc, current, next,
	start, NULL);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("node-action", "const char ", "const char ", "const char *")
	static int
	node_action(pcmk__output_t *out, va_list args)
	{
	const char task = va_arg(args, const char );
	const char node_name = va_arg(args, const char );
	const char reason = va_arg(args, const char );

	if (task == NULL) {
	return pcmk_rc_no_output;
	} else if (reason) {
	out->list_item(out, NULL, "%s %s '%s'", task, node_name, reason);
	} else {
	crm_notice(" * %s %s", task, node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-action", "const char ", "const char ", "const char *")
	static int
	node_action_xml(pcmk__output_t *out, va_list args)
	{
	const char task = va_arg(args, const char );
	const char node_name = va_arg(args, const char );
	const char reason = va_arg(args, const char );

	if (task == NULL) {
	return pcmk_rc_no_output;
	} else if (reason) {
	pcmk__output_create_xml_node(out, "node_action",
	"task", task,
	"node", node_name,
	PCMK_XA_REASON, reason,
	NULL);
	} else {
	crm_notice(" * %s %s", task, node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-info", "uint32_t", "const char ", "const char ",
	"const char *", "bool", "bool")
	static int
	node_info_default(pcmk__output_t *out, va_list args)
	{
	uint32_t node_id = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );
	const char uuid = va_arg(args, const char );
	const char state = va_arg(args, const char );
	bool have_quorum = (bool) va_arg(args, int);
	bool is_remote = (bool) va_arg(args, int);

	return out->info(out,
	"Node %" PRIu32 ": %s "
	"(uuid=%s, state=%s, have_quorum=%s, is_remote=%s)",
	node_id, pcmk__s(node_name, "unknown"),
	pcmk__s(uuid, "unknown"), pcmk__s(state, "unknown"),
	pcmk__btoa(have_quorum), pcmk__btoa(is_remote));
	}

	PCMK__OUTPUT_ARGS("node-info", "uint32_t", "const char ", "const char ",
	"const char *", "bool", "bool")
	static int
	node_info_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t node_id = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );
	const char uuid = va_arg(args, const char );
	const char state = va_arg(args, const char );
	bool have_quorum = (bool) va_arg(args, int);
	bool is_remote = (bool) va_arg(args, int);

	char *id_s = crm_strdup_printf("%" PRIu32, node_id);

	pcmk__output_create_xml_node(out, "node-info",
	"nodeid", id_s,
	PCMK_XA_UNAME, node_name,
	PCMK_XA_ID, uuid,
	PCMK__XA_CRMD, state,
	PCMK_XA_HAVE_QUORUM, pcmk__btoa(have_quorum),
	XML_NODE_IS_REMOTE, pcmk__btoa(is_remote),
	NULL);
	free(id_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-cluster-action", "const char ", "const char ",
	"xmlNodePtr")
	static int
	inject_cluster_action(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );
	xmlNodePtr rsc = va_arg(args, xmlNodePtr);

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (rsc != NULL) {
	out->list_item(out, NULL, "Cluster action: %s for %s on %s",
	task, ID(rsc), node);
	} else {
	out->list_item(out, NULL, "Cluster action: %s on %s", task, node);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-cluster-action", "const char ", "const char ",
	"xmlNodePtr")
	static int
	inject_cluster_action_xml(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );
	xmlNodePtr rsc = va_arg(args, xmlNodePtr);

	xmlNodePtr xml_node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	xml_node = pcmk__output_create_xml_node(out, "cluster_action",
	"task", task,
	"node", node,
	NULL);

	if (rsc) {
	crm_xml_add(xml_node, PCMK_XA_ID, ID(rsc));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-fencing-action", "const char ", "const char ")
	static int
	inject_fencing_action(pcmk__output_t *out, va_list args)
	{
	const char target = va_arg(args, const char );
	const char op = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->list_item(out, NULL, "Fencing %s (%s)", target, op);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-fencing-action", "const char ", "const char ")
	static int
	inject_fencing_action_xml(pcmk__output_t *out, va_list args)
	{
	const char target = va_arg(args, const char );
	const char op = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, "fencing_action",
	PCMK_XA_TARGET, target,
	PCMK_XA_OP, op,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-attr", "const char ", "const char ", "xmlNodePtr")
	static int
	inject_attr(pcmk__output_t *out, va_list args)
	{
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	xmlNodePtr cib_node = va_arg(args, xmlNodePtr);

	xmlChar *node_path = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	node_path = xmlGetNodePath(cib_node);

	out->list_item(out, NULL, "Injecting attribute %s=%s into %s '%s'",
	name, value, node_path, ID(cib_node));

	free(node_path);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-attr", "const char ", "const char ", "xmlNodePtr")
	static int
	inject_attr_xml(pcmk__output_t *out, va_list args)
	{
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	xmlNodePtr cib_node = va_arg(args, xmlNodePtr);

	xmlChar *node_path = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	node_path = xmlGetNodePath(cib_node);

	pcmk__output_create_xml_node(out, "inject_attr",
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	"node_path", node_path,
	"cib_node", ID(cib_node),
	NULL);
	free(node_path);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-spec", "const char *")
	static int
	inject_spec(pcmk__output_t *out, va_list args)
	{
	const char spec = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->list_item(out, NULL, "Injecting %s into the configuration", spec);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-spec", "const char *")
	static int
	inject_spec_xml(pcmk__output_t *out, va_list args)
	{
	const char spec = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, "inject_spec",
	"spec", spec,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-config", "const char ", "const char ")
	static int
	inject_modify_config(pcmk__output_t *out, va_list args)
	{
	const char quorum = va_arg(args, const char );
	const char watchdog = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Performing Requested Modifications");

	if (quorum) {
	out->list_item(out, NULL, "Setting quorum: %s", quorum);
	}

	if (watchdog) {
	out->list_item(out, NULL, "Setting watchdog: %s", watchdog);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-config", "const char ", "const char ")
	static int
	inject_modify_config_xml(pcmk__output_t *out, va_list args)
	{
	const char quorum = va_arg(args, const char );
	const char watchdog = va_arg(args, const char );

	xmlNodePtr node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	node = pcmk__output_xml_create_parent(out, "modifications", NULL);

	if (quorum) {
	crm_xml_add(node, "quorum", quorum);
	}

	if (watchdog) {
	crm_xml_add(node, "watchdog", watchdog);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-node", "const char ", "const char ")
	static int
	inject_modify_node(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char node = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (pcmk__str_eq(action, "Online", pcmk__str_none)) {
	out->list_item(out, NULL, "Bringing node %s online", node);
	return pcmk_rc_ok;
	} else if (pcmk__str_eq(action, "Offline", pcmk__str_none)) {
	out->list_item(out, NULL, "Taking node %s offline", node);
	return pcmk_rc_ok;
	} else if (pcmk__str_eq(action, "Failing", pcmk__str_none)) {
	out->list_item(out, NULL, "Failing node %s", node);
	return pcmk_rc_ok;
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("inject-modify-node", "const char ", "const char ")
	static int
	inject_modify_node_xml(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char node = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, "modify_node",
	"action", action,
	"node", node,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-ticket", "const char ", "const char ")
	static int
	inject_modify_ticket(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char ticket = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (pcmk__str_eq(action, "Standby", pcmk__str_none)) {
	out->list_item(out, NULL, "Making ticket %s standby", ticket);
	} else {
	out->list_item(out, NULL, "%s ticket %s", action, ticket);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-ticket", "const char ", "const char ")
	static int
	inject_modify_ticket_xml(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char ticket = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, "modify_ticket",
	"action", action,
	"ticket", ticket,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-pseudo-action", "const char ", "const char ")
	static int
	inject_pseudo_action(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->list_item(out, NULL, "Pseudo action: %s%s%s",
	task, ((node == NULL)? "" : " on "), pcmk__s(node, ""));
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-pseudo-action", "const char ", "const char ")
	static int
	inject_pseudo_action_xml(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );

	xmlNodePtr xml_node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	xml_node = pcmk__output_create_xml_node(out, "pseudo_action",
	"task", task,
	NULL);
	if (node) {
	crm_xml_add(xml_node, "node", node);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-rsc-action", "const char ", "const char ",
	"const char *", "guint")
	static int
	inject_rsc_action(pcmk__output_t *out, va_list args)
	{
	const char rsc = va_arg(args, const char );
	const char operation = va_arg(args, const char );
	const char node = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (interval_ms) {
	out->list_item(out, NULL, "Resource action: %-15s %s=%u on %s",
	rsc, operation, interval_ms, node);
	} else {
	out->list_item(out, NULL, "Resource action: %-15s %s on %s",
	rsc, operation, node);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-rsc-action", "const char ", "const char ",
	"const char *", "guint")
	static int
	inject_rsc_action_xml(pcmk__output_t *out, va_list args)
	{
	const char rsc = va_arg(args, const char );
	const char operation = va_arg(args, const char );
	const char node = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);

	xmlNodePtr xml_node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	xml_node = pcmk__output_create_xml_node(out, "rsc_action",
	"resource", rsc,
	PCMK_XA_OP, operation,
	"node", node,
	NULL);

	if (interval_ms) {
	char *interval_s = pcmk__itoa(interval_ms);

	crm_xml_add(xml_node, "interval", interval_s);
	free(interval_s);
	}

	return pcmk_rc_ok;
	}

	#define CHECK_RC(retcode, retval) \
	if (retval == pcmk_rc_ok) { \
	retcode = pcmk_rc_ok; \
	}

	PCMK__OUTPUT_ARGS("cluster-status", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "crm_exit_t",
	"stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
	"uint32_t", "const char ", "GList ", "GList *")
	int
	pcmk__cluster_status_text(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t stonith_history = va_arg(args, stonith_history_t );
	enum pcmk__fence_history fence_history = va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	const char prefix = va_arg(args, const char );
	GList unames = va_arg(args, GList );
	GList resources = va_arg(args, GList );

	int rc = pcmk_rc_no_output;
	bool already_printed_failure = false;

	CHECK_RC(rc, out->message(out, "cluster-summary", scheduler, pcmkd_state,
	section_opts, show_opts));

	if (pcmk_is_set(section_opts, pcmk_section_nodes) && unames) {
	CHECK_RC(rc, out->message(out, "node-list", scheduler->nodes, unames,
	resources, show_opts, rc == pcmk_rc_ok));
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(section_opts, pcmk_section_resources)) {
	CHECK_RC(rc, out->message(out, "resource-list", scheduler, show_opts,
	true, unames, resources, rc == pcmk_rc_ok));
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(section_opts, pcmk_section_attributes)) {
	CHECK_RC(rc, out->message(out, "node-attribute-list", scheduler,
	show_opts, (rc == pcmk_rc_ok), unames,
	resources));
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_any_flags_set(section_opts,
	pcmk_section_operations\|pcmk_section_failcounts)) {
	CHECK_RC(rc, out->message(out, "node-summary", scheduler, unames,
	resources, section_opts, show_opts,
	(rc == pcmk_rc_ok)));
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(section_opts, pcmk_section_failures)
	&& (scheduler->failed != NULL)
	&& (scheduler->failed->children != NULL)) {

	CHECK_RC(rc, out->message(out, "failed-action-list", scheduler, unames,
	resources, show_opts, rc == pcmk_rc_ok));
	}

	/* Print failed stonith actions */
	if (pcmk_is_set(section_opts, pcmk_section_fence_failed) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc == 0) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_eq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	CHECK_RC(rc, out->message(out, "failed-fencing-list",
	stonith_history, unames, section_opts,
	show_opts, rc == pcmk_rc_ok));
	}
	} else {
	PCMK__OUTPUT_SPACER_IF(out, rc == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);

	already_printed_failure = true;
	}
	}

	/* Print tickets if requested */
	if (pcmk_is_set(section_opts, pcmk_section_tickets)) {
	CHECK_RC(rc, out->message(out, "ticket-list", scheduler,
	(rc == pcmk_rc_ok)));
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(section_opts, pcmk_section_bans)) {
	CHECK_RC(rc, out->message(out, "ban-list", scheduler, prefix, resources,
	show_opts, rc == pcmk_rc_ok));
	}

	/* Print stonith history */
	if (pcmk_any_flags_set(section_opts, pcmk_section_fencing_all) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc != 0) {
	if (!already_printed_failure) {
	PCMK__OUTPUT_SPACER_IF(out, rc == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_worked)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_neq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	CHECK_RC(rc, out->message(out, "fencing-list", hp, unames,
	section_opts, show_opts,
	rc == pcmk_rc_ok));
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_pending)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_pending,
	NULL);
	if (hp) {
	CHECK_RC(rc, out->message(out, "pending-fencing-list", hp,
	unames, section_opts, show_opts,
	rc == pcmk_rc_ok));
	}
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-status", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "crm_exit_t",
	"stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
	"uint32_t", "const char ", "GList ", "GList *")
	static int
	cluster_status_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t stonith_history = va_arg(args, stonith_history_t );
	enum pcmk__fence_history fence_history = va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	const char prefix = va_arg(args, const char );
	GList unames = va_arg(args, GList );
	GList resources = va_arg(args, GList );

	out->message(out, "cluster-summary", scheduler, pcmkd_state, section_opts,
	show_opts);

	/* NODES */
	if (pcmk_is_set(section_opts, pcmk_section_nodes)) {
	out->message(out, "node-list", scheduler->nodes, unames, resources,
	show_opts, false);
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(section_opts, pcmk_section_resources)) {
	/* XML output always displays full details. */
	uint32_t full_show_opts = show_opts & ~pcmk_show_brief;

	out->message(out, "resource-list", scheduler, full_show_opts,
	false, unames, resources, false);
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(section_opts, pcmk_section_attributes)) {
	out->message(out, "node-attribute-list", scheduler, show_opts, false,
	unames, resources);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_any_flags_set(section_opts,
	pcmk_section_operations\|pcmk_section_failcounts)) {
	out->message(out, "node-summary", scheduler, unames,
	resources, section_opts, show_opts, false);
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(section_opts, pcmk_section_failures)
	&& (scheduler->failed != NULL)
	&& (scheduler->failed->children != NULL)) {

	out->message(out, "failed-action-list", scheduler, unames, resources,
	show_opts, false);
	}

	/* Print stonith history */
	if (pcmk_is_set(section_opts, pcmk_section_fencing_all) &&
	fence_history != pcmk__fence_history_none) {
	out->message(out, "full-fencing-list", history_rc, stonith_history,
	unames, section_opts, show_opts, false);
	}

	/* Print tickets if requested */
	if (pcmk_is_set(section_opts, pcmk_section_tickets)) {
	out->message(out, "ticket-list", scheduler, false);
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(section_opts, pcmk_section_bans)) {
	out->message(out, "ban-list", scheduler, prefix, resources, show_opts,
	false);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-status", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "crm_exit_t",
	"stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
	"uint32_t", "const char ", "GList ", "GList *")
	static int
	cluster_status_html(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t stonith_history = va_arg(args, stonith_history_t );
	enum pcmk__fence_history fence_history = va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	const char prefix = va_arg(args, const char );
	GList unames = va_arg(args, GList );
	GList resources = va_arg(args, GList );
	bool already_printed_failure = false;

	out->message(out, "cluster-summary", scheduler, pcmkd_state, section_opts,
	show_opts);

	/* NODE LIST */
	if (pcmk_is_set(section_opts, pcmk_section_nodes) && unames) {
	out->message(out, "node-list", scheduler->nodes, unames, resources,
	show_opts, false);
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(section_opts, pcmk_section_resources)) {
	out->message(out, "resource-list", scheduler, show_opts, true, unames,
	resources, false);
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(section_opts, pcmk_section_attributes)) {
	out->message(out, "node-attribute-list", scheduler, show_opts, false,
	unames, resources);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_any_flags_set(section_opts,
	pcmk_section_operations\|pcmk_section_failcounts)) {
	out->message(out, "node-summary", scheduler, unames,
	resources, section_opts, show_opts, false);
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(section_opts, pcmk_section_failures)
	&& (scheduler->failed != NULL)
	&& (scheduler->failed->children != NULL)) {

	out->message(out, "failed-action-list", scheduler, unames, resources,
	show_opts, false);
	}

	/* Print failed stonith actions */
	if (pcmk_is_set(section_opts, pcmk_section_fence_failed) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc == 0) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_eq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	out->message(out, "failed-fencing-list", stonith_history,
	unames, section_opts, show_opts, false);
	}
	} else {
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);
	}
	}

	/* Print stonith history */
	if (pcmk_any_flags_set(section_opts, pcmk_section_fencing_all) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc != 0) {
	if (!already_printed_failure) {
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_worked)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_neq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	out->message(out, "fencing-list", hp, unames, section_opts,
	show_opts, false);
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_pending)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_pending,
	NULL);
	if (hp) {
	out->message(out, "pending-fencing-list", hp, unames,
	section_opts, show_opts, false);
	}
	}
	}

	/* Print tickets if requested */
	if (pcmk_is_set(section_opts, pcmk_section_tickets)) {
	out->message(out, "ticket-list", scheduler, false);
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(section_opts, pcmk_section_bans)) {
	out->message(out, "ban-list", scheduler, prefix, resources, show_opts,
	false);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute", "const char ", "const char ", "const char *",
	"const char ", "const char ")
	static int
	attribute_default(pcmk__output_t *out, va_list args)
	{
	const char scope = va_arg(args, const char );
	const char instance = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	const char host = va_arg(args, const char );

	GString *s = g_string_sized_new(50);

	if (!pcmk__str_empty(scope)) {
	pcmk__g_strcat(s, "scope=\"", scope, "\" ", NULL);
	}

	if (!pcmk__str_empty(instance)) {
	pcmk__g_strcat(s, "id=\"", instance, "\" ", NULL);
	}

	pcmk__g_strcat(s, "name=\"", pcmk__s(name, ""), "\" ", NULL);

	if (!pcmk__str_empty(host)) {
	pcmk__g_strcat(s, "host=\"", host, "\" ", NULL);
	}

	pcmk__g_strcat(s, "value=\"", pcmk__s(value, ""), "\"", NULL);

	out->info(out, "%s", s->str);
	g_string_free(s, TRUE);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute", "const char ", "const char ", "const char *",
	"const char ", "const char ")
	static int
	attribute_xml(pcmk__output_t *out, va_list args)
	{
	const char scope = va_arg(args, const char );
	const char instance = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	const char host = va_arg(args, const char );

	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, "attribute",
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, pcmk__s(value, ""),
	NULL);

	if (!pcmk__str_empty(scope)) {
	crm_xml_add(node, "scope", scope);
	}

	if (!pcmk__str_empty(instance)) {
	crm_xml_add(node, PCMK_XA_ID, instance);
	}

	if (!pcmk__str_empty(host)) {
	crm_xml_add(node, "host", host);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("rule-check", "const char ", "int", "const char ")
	static int
	rule_check_default(pcmk__output_t *out, va_list args)
	{
	const char rule_id = va_arg(args, const char );
	int result = va_arg(args, int);
	const char error = va_arg(args, const char );

	switch (result) {
	case pcmk_rc_within_range:
	return out->info(out, "Rule %s is still in effect", rule_id);
	case pcmk_rc_ok:
	return out->info(out, "Rule %s satisfies conditions", rule_id);
	case pcmk_rc_after_range:
	return out->info(out, "Rule %s is expired", rule_id);
	case pcmk_rc_before_range:
	return out->info(out, "Rule %s has not yet taken effect", rule_id);
	case pcmk_rc_op_unsatisfied:
	return out->info(out, "Rule %s does not satisfy conditions",
	rule_id);
	default:
	out->err(out,
	"Could not determine whether rule %s is in effect: %s",
	rule_id, ((error != NULL)? error : "unexpected error"));
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("rule-check", "const char ", "int", "const char ")
	static int
	rule_check_xml(pcmk__output_t *out, va_list args)
	{
	const char rule_id = va_arg(args, const char );
	int result = va_arg(args, int);
	const char error = va_arg(args, const char );

	char *rc_str = pcmk__itoa(pcmk_rc2exitc(result));

	pcmk__output_create_xml_node(out, "rule-check",
	"rule-id", rule_id,
	"rc", rc_str,
	NULL);
	free(rc_str);

	switch (result) {
	case pcmk_rc_within_range:
	case pcmk_rc_ok:
	case pcmk_rc_after_range:
	case pcmk_rc_before_range:
	case pcmk_rc_op_unsatisfied:
	return pcmk_rc_ok;
	default:
	out->err(out,
	"Could not determine whether rule %s is in effect: %s",
	rule_id, ((error != NULL)? error : "unexpected error"));
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("result-code", "int", "const char ", "const char ")
	static int
	result_code_none(pcmk__output_t *out, va_list args)
	{
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("result-code", "int", "const char ", "const char ")
	static int
	result_code_text(pcmk__output_t *out, va_list args)
	{
	int code = va_arg(args, int);
	const char name = va_arg(args, const char );
	const char desc = va_arg(args, const char );

	static int code_width = 0;

	if (out->is_quiet(out)) {
	/* If out->is_quiet(), don't print the code. Print name and/or desc in a
	* compact format for text output, or print nothing at all for none-type
	* output.
	*/
	if ((name != NULL) && (desc != NULL)) {
	pcmk__formatted_printf(out, "%s - %s\n", name, desc);

	} else if ((name != NULL) \|\| (desc != NULL)) {
	pcmk__formatted_printf(out, "%s\n", ((name != NULL)? name : desc));
	}
	return pcmk_rc_ok;
	}

	/* Get length of longest (most negative) standard Pacemaker return code
	* This should be longer than all the values of any other type of return
	* code.
	*/
	if (code_width == 0) {
	long long most_negative = pcmk_rc_error - (long long) pcmk__n_rc + 1;
	code_width = (int) snprintf(NULL, 0, "%lld", most_negative);
	}

	if ((name != NULL) && (desc != NULL)) {
	static int name_width = 0;

	if (name_width == 0) {
	// Get length of longest standard Pacemaker return code name
	for (int lpc = 0; lpc < pcmk__n_rc; lpc++) {
	int len = (int) strlen(pcmk_rc_name(pcmk_rc_error - lpc));
	name_width = QB_MAX(name_width, len);
	}
	}
	return out->info(out, "% d: %-s %s", code_width, code, name_width,
	name, desc);
	}

	if ((name != NULL) \|\| (desc != NULL)) {
	return out->info(out, "% *d: %s", code_width, code,
	((name != NULL)? name : desc));
	}

	return out->info(out, "% *d", code_width, code);
	}

	PCMK__OUTPUT_ARGS("result-code", "int", "const char ", "const char ")
	static int
	result_code_xml(pcmk__output_t *out, va_list args)
	{
	int code = va_arg(args, int);
	const char name = va_arg(args, const char );
	const char desc = va_arg(args, const char );

	char *code_str = pcmk__itoa(code);

	pcmk__output_create_xml_node(out, "result-code",
	"code", code_str,
	PCMK_XA_NAME, name,
	PCMK_XA_DESCRIPTION, desc,
	NULL);
	free(code_str);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "attribute", "default", attribute_default },
	{ "attribute", "xml", attribute_xml },
	{ "cluster-status", "default", pcmk__cluster_status_text },
	{ "cluster-status", "html", cluster_status_html },
	{ "cluster-status", "xml", cluster_status_xml },
	{ "crmadmin-node", "default", crmadmin_node },
	{ "crmadmin-node", "text", crmadmin_node_text },
	{ "crmadmin-node", "xml", crmadmin_node_xml },
	{ "dc", "default", dc },
	{ "dc", "text", dc_text },
	{ "dc", "xml", dc_xml },
	{ "digests", "default", digests_text },
	{ "digests", "xml", digests_xml },
	{ "health", "default", health },
	{ "health", "text", health_text },
	{ "health", "xml", health_xml },
	{ "inject-attr", "default", inject_attr },
	{ "inject-attr", "xml", inject_attr_xml },
	{ "inject-cluster-action", "default", inject_cluster_action },
	{ "inject-cluster-action", "xml", inject_cluster_action_xml },
	{ "inject-fencing-action", "default", inject_fencing_action },
	{ "inject-fencing-action", "xml", inject_fencing_action_xml },
	{ "inject-modify-config", "default", inject_modify_config },
	{ "inject-modify-config", "xml", inject_modify_config_xml },
	{ "inject-modify-node", "default", inject_modify_node },
	{ "inject-modify-node", "xml", inject_modify_node_xml },
	{ "inject-modify-ticket", "default", inject_modify_ticket },
	{ "inject-modify-ticket", "xml", inject_modify_ticket_xml },
	{ "inject-pseudo-action", "default", inject_pseudo_action },
	{ "inject-pseudo-action", "xml", inject_pseudo_action_xml },
	{ "inject-rsc-action", "default", inject_rsc_action },
	{ "inject-rsc-action", "xml", inject_rsc_action_xml },
	{ "inject-spec", "default", inject_spec },
	{ "inject-spec", "xml", inject_spec_xml },
	{ "locations-list", "default", locations_list },
	{ "locations-list", "xml", locations_list_xml },
	{ "node-action", "default", node_action },
	{ "node-action", "xml", node_action_xml },
	{ "node-info", "default", node_info_default },
	{ "node-info", "xml", node_info_xml },
	{ "pacemakerd-health", "default", pacemakerd_health },
	{ "pacemakerd-health", "html", pacemakerd_health_html },
	{ "pacemakerd-health", "text", pacemakerd_health_text },
	{ "pacemakerd-health", "xml", pacemakerd_health_xml },
	{ "profile", "default", profile_default, },
	{ "profile", "xml", profile_xml },
	{ "result-code", "none", result_code_none },
	{ "result-code", "text", result_code_text },
	{ "result-code", "xml", result_code_xml },
	{ "rsc-action", "default", rsc_action_default },
	{ "rsc-action-item", "default", rsc_action_item },
	{ "rsc-action-item", "xml", rsc_action_item_xml },
	{ "rsc-is-colocated-with-list", "default", rsc_is_colocated_with_list },
	{ "rsc-is-colocated-with-list", "xml", rsc_is_colocated_with_list_xml },
	{ "rscs-colocated-with-list", "default", rscs_colocated_with_list },
	{ "rscs-colocated-with-list", "xml", rscs_colocated_with_list_xml },
	{ "rule-check", "default", rule_check_default },
	{ "rule-check", "xml", rule_check_xml },
	{ "locations-and-colocations", "default", locations_and_colocations },
	{ "locations-and-colocations", "xml", locations_and_colocations_xml },

	{ NULL, NULL, NULL }
	};

	void
	pcmk__register_lib_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/lib/pacemaker/pcmk_sched_actions.c b/lib/pacemaker/pcmk_sched_actions.c
	index 5a1f77ee05..efb06c62e7 100644
	--- a/lib/pacemaker/pcmk_sched_actions.c
	+++ b/lib/pacemaker/pcmk_sched_actions.c
	@@ -1,1932 +1,1932 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/param.h>
	#include <glib.h>

	#include <crm/lrmd_internal.h>
	#include <crm/common/scheduler_internal.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Get the action flags relevant to ordering constraints
	*
	* \param[in,out] action Action to check
	* \param[in] node Node that other action in the ordering is on
	* (used only for clone resource actions)
	*
	* \return Action flags that should be used for orderings
	*/
	static uint32_t
	action_flags_for_ordering(pcmk_action_t action, const pcmk_node_t node)
	{
	bool runnable = false;
	uint32_t flags;

	// For non-resource actions, return the action flags
	if (action->rsc == NULL) {
	return action->flags;
	}

	/* For non-clone resources, or a clone action not assigned to a node,
	* return the flags as determined by the resource method without a node
	* specified.
	*/
	flags = action->rsc->cmds->action_flags(action, NULL);
	if ((node == NULL) \|\| !pe_rsc_is_clone(action->rsc)) {
	return flags;
	}

	/* Otherwise (i.e., for clone resource actions on a specific node), first
	* remember whether the non-node-specific action is runnable.
	*/
	runnable = pcmk_is_set(flags, pcmk_action_runnable);

	// Then recheck the resource method with the node
	flags = action->rsc->cmds->action_flags(action, node);

	/* For clones in ordering constraints, the node-specific "runnable" doesn't
	* matter, just the non-node-specific setting (i.e., is the action runnable
	* anywhere).
	*
	* This applies only to runnable, and only for ordering constraints. This
	* function shouldn't be used for other types of constraints without
	* changes. Not very satisfying, but it's logical and appears to work well.
	*/
	if (runnable && !pcmk_is_set(flags, pcmk_action_runnable)) {
	pcmk__set_raw_action_flags(flags, action->rsc->id,
	pcmk_action_runnable);
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Get action UUID that should be used with a resource ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the UUID and resource of the first action in an
	* ordering, this returns the UUID of the action that should actually be used
	* for ordering (for example, "CLONE_started_0" instead of "CLONE_start_0").
	*
	* \param[in] first_uuid UUID of first action in ordering
	* \param[in] first_rsc Resource of first action in ordering
	*
	* \return Newly allocated copy of UUID to use with ordering
	* \note It is the caller's responsibility to free the return value.
	*/
	static char *
	action_uuid_for_ordering(const char *first_uuid,
	const pcmk_resource_t *first_rsc)
	{
	guint interval_ms = 0;
	char *uuid = NULL;
	char *rid = NULL;
	char *first_task_str = NULL;
	enum action_tasks first_task = pcmk_action_unspecified;
	enum action_tasks remapped_task = pcmk_action_unspecified;

	// Only non-notify actions for collective resources need remapping
	if ((strstr(first_uuid, PCMK_ACTION_NOTIFY) != NULL)
	\|\| (first_rsc->variant < pcmk_rsc_variant_group)) {
	goto done;
	}

	// Only non-recurring actions need remapping
	CRM_ASSERT(parse_op_key(first_uuid, &rid, &first_task_str, &interval_ms));
	if (interval_ms > 0) {
	goto done;
	}

	first_task = text2task(first_task_str);
	switch (first_task) {
	case pcmk_action_stop:
	case pcmk_action_start:
	case pcmk_action_notify:
	case pcmk_action_promote:
	case pcmk_action_demote:
	remapped_task = first_task + 1;
	break;
	case pcmk_action_stopped:
	case pcmk_action_started:
	case pcmk_action_notified:
	case pcmk_action_promoted:
	case pcmk_action_demoted:
	remapped_task = first_task;
	break;
	case pcmk_action_monitor:
	case pcmk_action_shutdown:
	case pcmk_action_fence:
	break;
	default:
	crm_err("Unknown action '%s' in ordering", first_task_str);
	break;
	}

	if (remapped_task != pcmk_action_unspecified) {
	/* If a clone or bundle has notifications enabled, the ordering will be
	* relative to when notifications have been sent for the remapped task.
	*/
	if (pcmk_is_set(first_rsc->flags, pcmk_rsc_notify)
	&& (pe_rsc_is_clone(first_rsc) \|\| pe_rsc_is_bundled(first_rsc))) {
	uuid = pcmk__notify_key(rid, "confirmed-post",
	task2text(remapped_task));
	} else {
	uuid = pcmk__op_key(rid, task2text(remapped_task), 0);
	}
	pcmk__rsc_trace(first_rsc,
	"Remapped action UUID %s to %s for ordering purposes",
	first_uuid, uuid);
	}

	done:
	if (uuid == NULL) {
	uuid = strdup(first_uuid);
	CRM_ASSERT(uuid != NULL);
	}
	free(first_task_str);
	free(rid);
	return uuid;
	}

	/*!
	* \internal
	* \brief Get actual action that should be used with an ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the first action in an ordering, this returns the
	* the action that should actually be used for ordering (for example, the
	* started action instead of the start action).
	*
	* \param[in] action First action in an ordering
	*
	* \return Actual action that should be used for the ordering
	*/
	static pcmk_action_t *
	action_for_ordering(pcmk_action_t *action)
	{
	pcmk_action_t *result = action;
	pcmk_resource_t *rsc = action->rsc;

	if ((rsc != NULL) && (rsc->variant >= pcmk_rsc_variant_group)
	&& (action->uuid != NULL)) {
	char *uuid = action_uuid_for_ordering(action->uuid, rsc);

	result = find_first_action(rsc->actions, uuid, NULL, NULL);
	if (result == NULL) {
	crm_warn("Not remapping %s to %s because %s does not have "
	"remapped action", action->uuid, uuid, rsc->id);
	result = action;
	}
	free(uuid);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Wrapper for update_ordered_actions() method for readability
	*
	* \param[in,out] rsc Resource to call method for
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this
	* node (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates
	* (may include pcmk_action_optional to affect only
	* mandatory actions, and pe_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static inline uint32_t
	update(pcmk_resource_t rsc, pcmk_action_t first, pcmk_action_t *then,
	const pcmk_node_t *node, uint32_t flags, uint32_t filter, uint32_t type,
	pcmk_scheduler_t *scheduler)
	{
	return rsc->cmds->update_ordered_actions(first, then, node, flags, filter,
	type, scheduler);
	}

	/*!
	* \internal
	* \brief Update flags for ordering's actions appropriately for ordering's flags
	*
	* \param[in,out] first First action in an ordering
	* \param[in,out] then Then action in an ordering
	* \param[in] first_flags Action flags for \p first for ordering purposes
	* \param[in] then_flags Action flags for \p then for ordering purposes
	* \param[in,out] order Action wrapper for \p first in ordering
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags
	*/
	static uint32_t
	update_action_for_ordering_flags(pcmk_action_t first, pcmk_action_t then,
	uint32_t first_flags, uint32_t then_flags,
	pcmk__related_action_t *order,
	pcmk_scheduler_t *scheduler)
	{
	uint32_t changed = pcmk__updated_none;

	/* The node will only be used for clones. If interleaved, node will be NULL,
	* otherwise the ordering scope will be limited to the node. Normally, the
	* whole 'then' clone should restart if 'first' is restarted, so then->node
	* is needed.
	*/
	pcmk_node_t *node = then->node;

	if (pcmk_is_set(order->type, pcmk__ar_first_implies_same_node_then)) {
	/* For unfencing, only instances of 'then' on the same node as 'first'
	* (the unfencing operation) should restart, so reset node to
	* first->node, at which point this case is handled like a normal
	* pcmk__ar_first_implies_then.
	*/
	pcmk__clear_relation_flags(order->type,
	pcmk__ar_first_implies_same_node_then);
	pcmk__set_relation_flags(order->type, pcmk__ar_first_implies_then);
	node = first->node;
	pcmk__rsc_trace(then->rsc,
	"%s then %s: mapped "
	"pcmk__ar_first_implies_same_node_then to "
	"pcmk__ar_first_implies_then on %s",
	first->uuid, then->uuid, pcmk__node_name(node));
	}

	if (pcmk_is_set(order->type, pcmk__ar_first_implies_then)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node,
	first_flags & pcmk_action_optional,
	pcmk_action_optional, pcmk__ar_first_implies_then,
	scheduler);
	} else if (!pcmk_is_set(first_flags, pcmk_action_optional)
	&& pcmk_is_set(then->flags, pcmk_action_optional)) {
	pcmk__clear_action_flags(then, pcmk_action_optional);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_first_implies_then",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_intermediate_stop)
	&& (then->rsc != NULL)) {
	enum pe_action_flags restart = pcmk_action_optional
	\|pcmk_action_runnable;

	changed \|= update(then->rsc, first, then, node, first_flags, restart,
	pcmk__ar_intermediate_stop, scheduler);
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_intermediate_stop",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_then_implies_first)) {
	if (first->rsc != NULL) {
	changed \|= update(first->rsc, first, then, node, first_flags,
	pcmk_action_optional, pcmk__ar_then_implies_first,
	scheduler);
	} else if (!pcmk_is_set(first_flags, pcmk_action_optional)
	&& pcmk_is_set(first->flags, pcmk_action_runnable)) {
	pcmk__clear_action_flags(first, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_then_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_promoted_then_implies_first)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node,
	first_flags & pcmk_action_optional,
	pcmk_action_optional,
	pcmk__ar_promoted_then_implies_first, scheduler);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after "
	"pcmk__ar_promoted_then_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_min_runnable)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pcmk__ar_min_runnable,
	scheduler);

	} else if (pcmk_is_set(first_flags, pcmk_action_runnable)) {
	// We have another runnable instance of "first"
	then->runnable_before++;

	/* Mark "then" as runnable if it requires a certain number of
	* "before" instances to be runnable, and they now are.
	*/
	if ((then->runnable_before >= then->required_runnable_before)
	&& !pcmk_is_set(then->flags, pcmk_action_runnable)) {

	pcmk__set_action_flags(then, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	}
	pcmk__rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_min_runnable",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_nested_remote_probe)
	&& (then->rsc != NULL)) {

	if (!pcmk_is_set(first_flags, pcmk_action_runnable)
	&& (first->rsc != NULL) && (first->rsc->running_on != NULL)) {

	pcmk__rsc_trace(then->rsc,
	"%s then %s: ignoring because first is stopping",
	first->uuid, then->uuid);
	order->type = (enum pe_ordering) pcmk__ar_none;
	} else {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable,
	pcmk__ar_unrunnable_first_blocks, scheduler);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_nested_remote_probe",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_unrunnable_first_blocks)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable,
	pcmk__ar_unrunnable_first_blocks, scheduler);

	} else if (!pcmk_is_set(first_flags, pcmk_action_runnable)
	&& pcmk_is_set(then->flags, pcmk_action_runnable)) {

	pcmk__clear_action_flags(then, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_unrunnable_first_blocks",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_unmigratable_then_blocks)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_optional,
	pcmk__ar_unmigratable_then_blocks, scheduler);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after "
	"pcmk__ar_unmigratable_then_blocks",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_first_else_then)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_optional, pcmk__ar_first_else_then,
	scheduler);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_first_else_then",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_ordered)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pcmk__ar_ordered,
	scheduler);
	}
	pcmk__rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_ordered",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_asymmetric)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pcmk__ar_asymmetric,
	scheduler);
	}
	pcmk__rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_asymmetric",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(first->flags, pcmk_action_runnable)
	&& pcmk_is_set(order->type, pcmk__ar_first_implies_then_graphed)
	&& !pcmk_is_set(first_flags, pcmk_action_optional)) {

	pcmk__rsc_trace(then->rsc, "%s will be in graph because %s is required",
	then->uuid, first->uuid);
	pcmk__set_action_flags(then, pcmk_action_always_in_graph);
	// Don't bother marking 'then' as changed just for this
	}

	if (pcmk_is_set(order->type, pcmk__ar_then_implies_first_graphed)
	&& !pcmk_is_set(then_flags, pcmk_action_optional)) {

	pcmk__rsc_trace(then->rsc, "%s will be in graph because %s is required",
	first->uuid, then->uuid);
	pcmk__set_action_flags(first, pcmk_action_always_in_graph);
	// Don't bother marking 'first' as changed just for this
	}

	if (pcmk_any_flags_set(order->type, pcmk__ar_first_implies_then
	\|pcmk__ar_then_implies_first
	\|pcmk__ar_intermediate_stop)
	&& (first->rsc != NULL)
	&& !pcmk_is_set(first->rsc->flags, pcmk_rsc_managed)
	&& pcmk_is_set(first->rsc->flags, pcmk_rsc_blocked)
	&& !pcmk_is_set(first->flags, pcmk_action_runnable)
	&& pcmk__str_eq(first->task, PCMK_ACTION_STOP, pcmk__str_none)) {

	if (pcmk_is_set(then->flags, pcmk_action_runnable)) {
	pcmk__clear_action_flags(then, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after checking whether first "
	"is blocked, unmanaged, unrunnable stop",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	return changed;
	}

	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pcmk_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pcmk_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pcmk_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	/*!
	* \internal
	* \brief Update an action's flags for all orderings where it is "then"
	*
	* \param[in,out] then Action to update
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__update_action_for_orderings(pcmk_action_t *then,
	pcmk_scheduler_t *scheduler)
	{
	GList *lpc = NULL;
	uint32_t changed = pcmk__updated_none;
	int last_flags = then->flags;

	pcmk__rsc_trace(then->rsc, "Updating %s %s (%s %s) on %s",
	action_type_str(then->flags), then->uuid,
	action_optional_str(then->flags),
	action_runnable_str(then->flags), action_node_str(then));

	if (pcmk_is_set(then->flags, pcmk_action_min_runnable)) {
	/* Initialize current known "runnable before" actions. As
	* update_action_for_ordering_flags() is called for each of then's
	* before actions, this number will increment as runnable 'first'
	* actions are encountered.
	*/
	then->runnable_before = 0;

	if (then->required_runnable_before == 0) {
	/* @COMPAT This ordering constraint uses the deprecated
	* "require-all=false" attribute. Treat it like
	* PCMK_META_CLONE_MIN=1.
	*/
	then->required_runnable_before = 1;
	}

	/* The pcmk__ar_min_runnable clause of
	* update_action_for_ordering_flags() (called below)
	* will reset runnable if appropriate.
	*/
	pcmk__clear_action_flags(then, pcmk_action_runnable);
	}

	for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
	pcmk__related_action_t *other = lpc->data;
	pcmk_action_t *first = other->action;

	pcmk_node_t *then_node = then->node;
	pcmk_node_t *first_node = first->node;

	if ((first->rsc != NULL)
	&& (first->rsc->variant == pcmk_rsc_variant_group)
	&& pcmk__str_eq(first->task, PCMK_ACTION_START, pcmk__str_none)) {

	first_node = first->rsc->fns->location(first->rsc, NULL, FALSE);
	if (first_node != NULL) {
	pcmk__rsc_trace(first->rsc, "Found %s for 'first' %s",
	pcmk__node_name(first_node), first->uuid);
	}
	}

	if ((then->rsc != NULL)
	&& (then->rsc->variant == pcmk_rsc_variant_group)
	&& pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)) {

	then_node = then->rsc->fns->location(then->rsc, NULL, FALSE);
	if (then_node != NULL) {
	pcmk__rsc_trace(then->rsc, "Found %s for 'then' %s",
	pcmk__node_name(then_node), then->uuid);
	}
	}

	// Disable constraint if it only applies when on same node, but isn't
	if (pcmk_is_set(other->type, pcmk__ar_if_on_same_node)
	&& (first_node != NULL) && (then_node != NULL)
	- && !pe__same_node(first_node, then_node)) {
	+ && !pcmk__same_node(first_node, then_node)) {

	pcmk__rsc_trace(then->rsc,
	"Disabled ordering %s on %s then %s on %s: "
	"not same node",
	other->action->uuid, pcmk__node_name(first_node),
	then->uuid, pcmk__node_name(then_node));
	other->type = (enum pe_ordering) pcmk__ar_none;
	continue;
	}

	pcmk__clear_updated_flags(changed, then, pcmk__updated_first);

	if ((first->rsc != NULL)
	&& pcmk_is_set(other->type, pcmk__ar_then_cancels_first)
	&& !pcmk_is_set(then->flags, pcmk_action_optional)) {

	/* 'then' is required, so we must abandon 'first'
	* (e.g. a required stop cancels any agent reload).
	*/
	pcmk__set_action_flags(other->action, pcmk_action_optional);
	if (!strcmp(first->task, PCMK_ACTION_RELOAD_AGENT)) {
	pcmk__clear_rsc_flags(first->rsc, pcmk_rsc_reload);
	}
	}

	if ((first->rsc != NULL) && (then->rsc != NULL)
	&& (first->rsc != then->rsc) && !is_parent(then->rsc, first->rsc)) {
	first = action_for_ordering(first);
	}
	if (first != other->action) {
	pcmk__rsc_trace(then->rsc, "Ordering %s after %s instead of %s",
	then->uuid, first->uuid, other->action->uuid);
	}

	pcmk__rsc_trace(then->rsc,
	"%s (%#.6x) then %s (%#.6x): type=%#.6x node=%s",
	first->uuid, first->flags, then->uuid, then->flags,
	other->type, action_node_str(first));

	if (first == other->action) {
	/* 'first' was not remapped (e.g. from 'start' to 'running'), which
	* could mean it is a non-resource action, a primitive resource
	* action, or already expanded.
	*/
	uint32_t first_flags, then_flags;

	first_flags = action_flags_for_ordering(first, then_node);
	then_flags = action_flags_for_ordering(then, first_node);

	changed \|= update_action_for_ordering_flags(first, then,
	first_flags, then_flags,
	other, scheduler);

	/* 'first' was for a complex resource (clone, group, etc),
	* create a new dependency if necessary
	*/
	} else if (order_actions(first, then, other->type)) {
	/* This was the first time 'first' and 'then' were associated,
	* start again to get the new actions_before list
	*/
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	pcmk__rsc_trace(then->rsc,
	"Disabled ordering %s then %s in favor of %s "
	"then %s",
	other->action->uuid, then->uuid, first->uuid,
	then->uuid);
	other->type = (enum pe_ordering) pcmk__ar_none;
	}


	if (pcmk_is_set(changed, pcmk__updated_first)) {
	crm_trace("Re-processing %s and its 'after' actions "
	"because it changed", first->uuid);
	for (GList *lpc2 = first->actions_after; lpc2 != NULL;
	lpc2 = lpc2->next) {
	pcmk__related_action_t *other = lpc2->data;

	pcmk__update_action_for_orderings(other->action, scheduler);
	}
	pcmk__update_action_for_orderings(first, scheduler);
	}
	}

	if (pcmk_is_set(then->flags, pcmk_action_min_runnable)) {
	if (last_flags == then->flags) {
	pcmk__clear_updated_flags(changed, then, pcmk__updated_then);
	} else {
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	}
	}

	if (pcmk_is_set(changed, pcmk__updated_then)) {
	crm_trace("Re-processing %s and its 'after' actions because it changed",
	then->uuid);
	if (pcmk_is_set(last_flags, pcmk_action_runnable)
	&& !pcmk_is_set(then->flags, pcmk_action_runnable)) {
	pcmk__block_colocation_dependents(then);
	}
	pcmk__update_action_for_orderings(then, scheduler);
	for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
	pcmk__related_action_t *other = lpc->data;

	pcmk__update_action_for_orderings(other->action, scheduler);
	}
	}
	}

	static inline bool
	is_primitive_action(const pcmk_action_t *action)
	{
	return (action != NULL) && (action->rsc != NULL)
	&& (action->rsc->variant == pcmk_rsc_variant_primitive);
	}

	/*!
	* \internal
	* \brief Clear a single action flag and set reason text
	*
	* \param[in,out] action Action whose flag should be cleared
	* \param[in] flag Action flag that should be cleared
	* \param[in] reason Action that is the reason why flag is being cleared
	*/
	#define clear_action_flag_because(action, flag, reason) do { \
	if (pcmk_is_set((action)->flags, (flag))) { \
	pcmk__clear_action_flags(action, flag); \
	if ((action)->rsc != (reason)->rsc) { \
	char *reason_text = pe__action2reason((reason), (flag)); \
	pe_action_set_reason((action), reason_text, false); \
	free(reason_text); \
	} \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Update actions in an asymmetric ordering
	*
	* If the "first" action in an asymmetric ordering is unrunnable, make the
	* "second" action unrunnable as well, if appropriate.
	*
	* \param[in] first 'First' action in an asymmetric ordering
	* \param[in,out] then 'Then' action in an asymmetric ordering
	*/
	static void
	handle_asymmetric_ordering(const pcmk_action_t first, pcmk_action_t then)
	{
	/* Only resource actions after an unrunnable 'first' action need updates for
	* asymmetric ordering.
	*/
	if ((then->rsc == NULL)
	\|\| pcmk_is_set(first->flags, pcmk_action_runnable)) {
	return;
	}

	// Certain optional 'then' actions are unaffected by unrunnable 'first'
	if (pcmk_is_set(then->flags, pcmk_action_optional)) {
	enum rsc_role_e then_rsc_role = then->rsc->fns->state(then->rsc, TRUE);

	if ((then_rsc_role == pcmk_role_stopped)
	&& pcmk__str_eq(then->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	/* If 'then' should stop after 'first' but is already stopped, the
	* ordering is irrelevant.
	*/
	return;
	} else if ((then_rsc_role >= pcmk_role_started)
	&& pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)
	&& pe__rsc_running_on_only(then->rsc, then->node)) {
	/* Similarly if 'then' should start after 'first' but is already
	* started on a single node.
	*/
	return;
	}
	}

	// 'First' can't run, so 'then' can't either
	clear_action_flag_because(then, pcmk_action_optional, first);
	clear_action_flag_because(then, pcmk_action_runnable, first);
	}

	/*!
	* \internal
	* \brief Set action bits appropriately when pe_restart_order is used
	*
	* \param[in,out] first 'First' action in an ordering with pe_restart_order
	* \param[in,out] then 'Then' action in an ordering with pe_restart_order
	* \param[in] filter What action flags to care about
	*
	* \note pe_restart_order is set for "stop resource before starting it" and
	* "stop later group member before stopping earlier group member"
	*/
	static void
	handle_restart_ordering(pcmk_action_t first, pcmk_action_t then,
	uint32_t filter)
	{
	const char *reason = NULL;

	CRM_ASSERT(is_primitive_action(first));
	CRM_ASSERT(is_primitive_action(then));

	// We need to update the action in two cases:

	// ... if 'then' is required
	if (pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(then->flags, pcmk_action_optional)) {
	reason = "restart";
	}

	/* ... if 'then' is unrunnable action on same resource (if a resource
	* should restart but can't start, we still want to stop)
	*/
	if (pcmk_is_set(filter, pcmk_action_runnable)
	&& !pcmk_is_set(then->flags, pcmk_action_runnable)
	&& pcmk_is_set(then->rsc->flags, pcmk_rsc_managed)
	&& (first->rsc == then->rsc)) {
	reason = "stop";
	}

	if (reason == NULL) {
	return;
	}

	pcmk__rsc_trace(first->rsc, "Handling %s -> %s for %s",
	first->uuid, then->uuid, reason);

	// Make 'first' required if it is runnable
	if (pcmk_is_set(first->flags, pcmk_action_runnable)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}

	// Make 'first' required if 'then' is required
	if (!pcmk_is_set(then->flags, pcmk_action_optional)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}

	// Make 'first' unmigratable if 'then' is unmigratable
	if (!pcmk_is_set(then->flags, pcmk_action_migratable)) {
	clear_action_flag_because(first, pcmk_action_migratable, then);
	}

	// Make 'then' unrunnable if 'first' is required but unrunnable
	if (!pcmk_is_set(first->flags, pcmk_action_optional)
	&& !pcmk_is_set(first->flags, pcmk_action_runnable)) {
	clear_action_flag_because(then, pcmk_action_runnable, first);
	}
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions' flags
	* (and runnable_before members if appropriate) as appropriate for the ordering.
	* Effects may cascade to other orderings involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (ignored)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__update_ordered_actions(pcmk_action_t first, pcmk_action_t then,
	const pcmk_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pcmk_scheduler_t *scheduler)
	{
	uint32_t changed = pcmk__updated_none;
	uint32_t then_flags = 0U;
	uint32_t first_flags = 0U;

	CRM_ASSERT((first != NULL) && (then != NULL) && (scheduler != NULL));

	then_flags = then->flags;
	first_flags = first->flags;
	if (pcmk_is_set(type, pcmk__ar_asymmetric)) {
	handle_asymmetric_ordering(first, then);
	}

	if (pcmk_is_set(type, pcmk__ar_then_implies_first)
	&& !pcmk_is_set(then_flags, pcmk_action_optional)) {
	// Then is required, and implies first should be, too

	if (pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(flags, pcmk_action_optional)
	&& pcmk_is_set(first_flags, pcmk_action_optional)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}

	if (pcmk_is_set(flags, pcmk_action_migratable)
	&& !pcmk_is_set(then->flags, pcmk_action_migratable)) {
	clear_action_flag_because(first, pcmk_action_migratable, then);
	}
	}

	if (pcmk_is_set(type, pcmk__ar_promoted_then_implies_first)
	&& (then->rsc != NULL) && (then->rsc->role == pcmk_role_promoted)
	&& pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(then->flags, pcmk_action_optional)) {

	clear_action_flag_because(first, pcmk_action_optional, then);

	if (pcmk_is_set(first->flags, pcmk_action_migratable)
	&& !pcmk_is_set(then->flags, pcmk_action_migratable)) {
	clear_action_flag_because(first, pcmk_action_migratable, then);
	}
	}

	if (pcmk_is_set(type, pcmk__ar_unmigratable_then_blocks)
	&& pcmk_is_set(filter, pcmk_action_optional)) {

	if (!pcmk_all_flags_set(then->flags, pcmk_action_migratable
	\|pcmk_action_runnable)) {
	clear_action_flag_because(first, pcmk_action_runnable, then);
	}

	if (!pcmk_is_set(then->flags, pcmk_action_optional)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}
	}

	if (pcmk_is_set(type, pcmk__ar_first_else_then)
	&& pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(first->flags, pcmk_action_runnable)) {

	clear_action_flag_because(then, pcmk_action_migratable, first);
	pcmk__clear_action_flags(then, pcmk_action_pseudo);
	}

	if (pcmk_is_set(type, pcmk__ar_unrunnable_first_blocks)
	&& pcmk_is_set(filter, pcmk_action_runnable)
	&& pcmk_is_set(then->flags, pcmk_action_runnable)
	&& !pcmk_is_set(flags, pcmk_action_runnable)) {

	clear_action_flag_because(then, pcmk_action_runnable, first);
	clear_action_flag_because(then, pcmk_action_migratable, first);
	}

	if (pcmk_is_set(type, pcmk__ar_first_implies_then)
	&& pcmk_is_set(filter, pcmk_action_optional)
	&& pcmk_is_set(then->flags, pcmk_action_optional)
	&& !pcmk_is_set(flags, pcmk_action_optional)
	&& !pcmk_is_set(first->flags, pcmk_action_migratable)) {

	clear_action_flag_because(then, pcmk_action_optional, first);
	}

	if (pcmk_is_set(type, pcmk__ar_intermediate_stop)) {
	handle_restart_ordering(first, then, filter);
	}

	if (then_flags != then->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	pcmk__rsc_trace(then->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'first' %s (%#.6x)",
	then->uuid, pcmk__node_name(then->node),
	then->flags, then_flags, first->uuid, first->flags);

	if ((then->rsc != NULL) && (then->rsc->parent != NULL)) {
	// Required to handle "X_stop then X_start" for cloned groups
	pcmk__update_action_for_orderings(then, scheduler);
	}
	}

	if (first_flags != first->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	pcmk__rsc_trace(first->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'then' %s (%#.6x)",
	first->uuid, pcmk__node_name(first->node),
	first->flags, first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	/*!
	* \internal
	* \brief Trace-log an action (optionally with its dependent actions)
	*
	* \param[in] pre_text If not NULL, prefix the log with this plus ": "
	* \param[in] action Action to log
	* \param[in] details If true, recursively log dependent actions
	*/
	void
	pcmk__log_action(const char pre_text, const pcmk_action_t action,
	bool details)
	{
	const char *node_uname = NULL;
	const char *node_uuid = NULL;
	const char *desc = NULL;

	CRM_CHECK(action != NULL, return);

	if (!pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	if (action->node != NULL) {
	node_uname = action->node->details->uname;
	node_uuid = action->node->details->id;
	} else {
	node_uname = "<none>";
	}
	}

	switch (text2task(action->task)) {
	case pcmk_action_fence:
	case pcmk_action_shutdown:
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	desc = "Pseudo ";
	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	desc = "Optional ";
	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	default:
	if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	desc = "Optional ";
	} else if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	desc = "Pseudo ";
	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(action->rsc? action->rsc->id : "<none>"),
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	}

	if (details) {
	const GList *iter = NULL;
	const pcmk__related_action_t *other = NULL;

	crm_trace("\t\t====== Preceding Actions");
	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	other = (const pcmk__related_action_t *) iter->data;
	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== Subsequent Actions");
	for (iter = action->actions_after; iter != NULL; iter = iter->next) {
	other = (const pcmk__related_action_t *) iter->data;
	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== End");

	} else {
	crm_trace("\t\t(before=%d, after=%d)",
	g_list_length(action->actions_before),
	g_list_length(action->actions_after));
	}
	}

	/*!
	* \internal
	* \brief Create a new shutdown action for a node
	*
	* \param[in,out] node Node being shut down
	*
	* \return Newly created shutdown action for \p node
	*/
	pcmk_action_t *
	pcmk__new_shutdown_action(pcmk_node_t *node)
	{
	char *shutdown_id = NULL;
	pcmk_action_t *shutdown_op = NULL;

	CRM_ASSERT(node != NULL);

	shutdown_id = crm_strdup_printf("%s-%s", PCMK_ACTION_DO_SHUTDOWN,
	node->details->uname);

	shutdown_op = custom_action(NULL, shutdown_id, PCMK_ACTION_DO_SHUTDOWN,
	node, FALSE, node->details->data_set);

	pcmk__order_stops_before_shutdown(node, shutdown_op);
	add_hash_param(shutdown_op->meta, PCMK__META_OP_NO_WAIT, PCMK_VALUE_TRUE);
	return shutdown_op;
	}

	/*!
	* \internal
	* \brief Calculate and add an operation digest to XML
	*
	* Calculate an operation digest, which enables us to later determine when a
	* restart is needed due to the resource's parameters being changed, and add it
	* to given XML.
	*
	* \param[in] op Operation result from executor
	* \param[in,out] update XML to add digest to
	*/
	static void
	add_op_digest_to_xml(const lrmd_event_data_t op, xmlNode update)
	{
	char *digest = NULL;
	xmlNode *args_xml = NULL;

	if (op->params == NULL) {
	return;
	}
	args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
	g_hash_table_foreach(op->params, hash2field, args_xml);
	pcmk__filter_op_for_digest(args_xml);
	digest = calculate_operation_digest(args_xml, NULL);
	crm_xml_add(update, PCMK__XA_OP_DIGEST, digest);
	free_xml(args_xml);
	free(digest);
	}

	#define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"

	/*!
	* \internal
	* \brief Create XML for resource operation history update
	*
	* \param[in,out] parent Parent XML node to add to
	* \param[in,out] op Operation event data
	* \param[in] caller_version DC feature set
	* \param[in] target_rc Expected result of operation
	* \param[in] node Name of node on which operation was performed
	* \param[in] origin Arbitrary description of update source
	*
	* \return Newly created XML node for history update
	*/
	xmlNode *
	pcmk__create_history_xml(xmlNode parent, lrmd_event_data_t op,
	const char *caller_version, int target_rc,
	const char node, const char origin)
	{
	char *key = NULL;
	char *magic = NULL;
	char *op_id = NULL;
	char *op_id_additional = NULL;
	char *local_user_data = NULL;
	const char *exit_reason = NULL;

	xmlNode *xml_op = NULL;
	const char *task = NULL;

	CRM_CHECK(op != NULL, return NULL);
	crm_trace("Creating history XML for %s-interval %s action for %s on %s "
	"(DC version: %s, origin: %s)",
	pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
	((node == NULL)? "no node" : node), caller_version, origin);

	task = op->op_type;

	/* Record a successful agent reload as a start, and a failed one as a
	* monitor, to make life easier for the scheduler when determining the
	* current state.
	*
	* @COMPAT We should check "reload" here only if the operation was for a
	* pre-OCF-1.1 resource agent, but we don't know that here, and we should
	* only ever get results for actions scheduled by us, so we can reasonably
	* assume any "reload" is actually a pre-1.1 agent reload.
	*/
	if (pcmk__str_any_of(task, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT,
	NULL)) {
	if (op->op_status == PCMK_EXEC_DONE) {
	task = PCMK_ACTION_START;
	} else {
	task = PCMK_ACTION_MONITOR;
	}
	}

	key = pcmk__op_key(op->rsc_id, task, op->interval_ms);
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = crm_meta_value(op->params, "notify_type");
	const char *n_task = crm_meta_value(op->params, "notify_operation");

	CRM_LOG_ASSERT(n_type != NULL);
	CRM_LOG_ASSERT(n_task != NULL);
	op_id = pcmk__notify_key(op->rsc_id, n_type, n_task);

	if (op->op_status != PCMK_EXEC_PENDING) {
	/* Ignore notify errors.
	*
	* @TODO It might be better to keep the correct result here, and
	* ignore it in process_graph_event().
	*/
	lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	}

	/* Migration history is preserved separately, which usually matters for
	* multiple nodes and is important for future cluster transitions.
	*/
	} else if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	op_id = strdup(key);

	} else if (did_rsc_op_fail(op, target_rc)) {
	op_id = pcmk__op_key(op->rsc_id, "last_failure", 0);
	if (op->interval_ms == 0) {
	/* Ensure 'last' gets updated, in case PCMK_META_RECORD_PENDING is
	* true
	*/
	op_id_additional = pcmk__op_key(op->rsc_id, "last", 0);
	}
	exit_reason = op->exit_reason;

	} else if (op->interval_ms > 0) {
	op_id = strdup(key);

	} else {
	op_id = pcmk__op_key(op->rsc_id, "last", 0);
	}

	again:
	xml_op = pcmk__xe_match(parent, XML_LRM_TAG_RSC_OP, PCMK_XA_ID, op_id);
	if (xml_op == NULL) {
	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
	}

	if (op->user_data == NULL) {
	crm_debug("Generating fake transition key for: " PCMK__OP_FMT
	" %d from %s", op->rsc_id, op->op_type, op->interval_ms,
	op->call_id, origin);
	local_user_data = pcmk__transition_key(-1, op->call_id, target_rc,
	FAKE_TE_ID);
	op->user_data = local_user_data;
	}

	if (magic == NULL) {
	magic = crm_strdup_printf("%d:%d;%s", op->op_status, op->rc,
	(const char *) op->user_data);
	}

	crm_xml_add(xml_op, PCMK_XA_ID, op_id);
	crm_xml_add(xml_op, PCMK__XA_OPERATION_KEY, key);
	crm_xml_add(xml_op, PCMK_XA_OPERATION, task);
	crm_xml_add(xml_op, PCMK_XA_CRM_DEBUG_ORIGIN, origin);
	crm_xml_add(xml_op, PCMK_XA_CRM_FEATURE_SET, caller_version);
	crm_xml_add(xml_op, PCMK__XA_TRANSITION_KEY, op->user_data);
	crm_xml_add(xml_op, PCMK__XA_TRANSITION_MAGIC, magic);
	crm_xml_add(xml_op, PCMK_XA_EXIT_REASON, pcmk__s(exit_reason, ""));
	crm_xml_add(xml_op, PCMK__META_ON_NODE, node); // For context during triage

	crm_xml_add_int(xml_op, PCMK__XA_CALL_ID, op->call_id);
	crm_xml_add_int(xml_op, PCMK__XA_RC_CODE, op->rc);
	crm_xml_add_int(xml_op, PCMK__XA_OP_STATUS, op->op_status);
	crm_xml_add_ms(xml_op, PCMK_META_INTERVAL, op->interval_ms);

	if (compare_version("2.1", caller_version) <= 0) {
	if (op->t_run \|\| op->t_rcchange \|\| op->exec_time \|\| op->queue_time) {
	crm_trace("Timing data (" PCMK__OP_FMT
	"): last=%u change=%u exec=%u queue=%u",
	op->rsc_id, op->op_type, op->interval_ms,
	op->t_run, op->t_rcchange, op->exec_time, op->queue_time);

	if ((op->interval_ms != 0) && (op->t_rcchange != 0)) {
	// Recurring ops may have changed rc after initial run
	crm_xml_add_ll(xml_op, PCMK_XA_LAST_RC_CHANGE,
	(long long) op->t_rcchange);
	} else {
	crm_xml_add_ll(xml_op, PCMK_XA_LAST_RC_CHANGE,
	(long long) op->t_run);
	}

	crm_xml_add_int(xml_op, PCMK_XA_EXEC_TIME, op->exec_time);
	crm_xml_add_int(xml_op, PCMK_XA_QUEUE_TIME, op->queue_time);
	}
	}

	if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* Record PCMK__META_MIGRATE_SOURCE and PCMK__META_MIGRATE_TARGET always
	* for migrate ops.
	*/
	const char *name = PCMK__META_MIGRATE_SOURCE;

	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));

	name = PCMK__META_MIGRATE_TARGET;
	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
	}

	add_op_digest_to_xml(op, xml_op);

	if (op_id_additional) {
	free(op_id);
	op_id = op_id_additional;
	op_id_additional = NULL;
	goto again;
	}

	if (local_user_data) {
	free(local_user_data);
	op->user_data = NULL;
	}
	free(magic);
	free(op_id);
	free(key);
	return xml_op;
	}

	/*!
	* \internal
	* \brief Check whether an action shutdown-locks a resource to a node
	*
	* If the PCMK_OPT_SHUTDOWN_LOCK cluster property is set, resources will not be
	* recovered on a different node if cleanly stopped, and may start only on that
	* same node. This function checks whether that applies to a given action, so
	* that the transition graph can be marked appropriately.
	*
	* \param[in] action Action to check
	*
	* \return true if \p action locks its resource to the action's node,
	* otherwise false
	*/
	bool
	pcmk__action_locks_rsc_to_node(const pcmk_action_t *action)
	{
	// Only resource actions taking place on resource's lock node are locked
	if ((action == NULL) \|\| (action->rsc == NULL)
	- \|\| !pe__same_node(action->node, action->rsc->lock_node)) {
	+ \|\| !pcmk__same_node(action->node, action->rsc->lock_node)) {
	return false;
	}

	/* During shutdown, only stops are locked (otherwise, another action such as
	* a demote would cause the controller to clear the lock)
	*/
	if (action->node->details->shutdown && (action->task != NULL)
	&& (strcmp(action->task, PCMK_ACTION_STOP) != 0)) {
	return false;
	}

	return true;
	}

	/* lowest to highest */
	static gint
	sort_action_id(gconstpointer a, gconstpointer b)
	{
	const pcmk__related_action_t *action_wrapper2 = a;
	const pcmk__related_action_t *action_wrapper1 = b;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}
	if (action_wrapper1->action->id < action_wrapper2->action->id) {
	return 1;
	}
	if (action_wrapper1->action->id > action_wrapper2->action->id) {
	return -1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Remove any duplicate action inputs, merging action flags
	*
	* \param[in,out] action Action whose inputs should be checked
	*/
	void
	pcmk__deduplicate_action_inputs(pcmk_action_t *action)
	{
	GList *item = NULL;
	GList *next = NULL;
	pcmk__related_action_t *last_input = NULL;

	action->actions_before = g_list_sort(action->actions_before,
	sort_action_id);
	for (item = action->actions_before; item != NULL; item = next) {
	pcmk__related_action_t *input = item->data;

	next = item->next;
	if ((last_input != NULL)
	&& (input->action->id == last_input->action->id)) {
	crm_trace("Input %s (%d) duplicate skipped for action %s (%d)",
	input->action->uuid, input->action->id,
	action->uuid, action->id);

	/* For the purposes of scheduling, the ordering flags no longer
	* matter, but crm_simulate looks at certain ones when creating a
	* dot graph. Combining the flags is sufficient for that purpose.
	*/
	last_input->type \|= input->type;
	if (input->state == pe_link_dumped) {
	last_input->state = pe_link_dumped;
	}

	free(item->data);
	action->actions_before = g_list_delete_link(action->actions_before,
	item);
	} else {
	last_input = input;
	input->state = pe_link_not_dumped;
	}
	}
	}

	/*!
	* \internal
	* \brief Output all scheduled actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__output_actions(pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;

	// Output node (non-resource) actions
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	char *node_name = NULL;
	char *task = NULL;
	pcmk_action_t action = (pcmk_action_t ) iter->data;

	if (action->rsc != NULL) {
	continue; // Resource actions will be output later

	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	continue; // This action was not scheduled
	}

	if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_none)) {
	task = strdup("Shutdown");

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {
	const char *op = g_hash_table_lookup(action->meta,
	"stonith_action");

	task = crm_strdup_printf("Fence (%s)", op);

	} else {
	continue; // Don't display other node action types
	}

	if (pe__is_guest_node(action->node)) {
	const pcmk_resource_t *remote = action->node->details->remote_rsc;

	node_name = crm_strdup_printf("%s (resource: %s)",
	pcmk__node_name(action->node),
	remote->container->id);
	} else if (action->node != NULL) {
	node_name = crm_strdup_printf("%s", pcmk__node_name(action->node));
	}

	out->message(out, "node-action", task, node_name, action->reason);

	free(node_name);
	free(task);
	}

	// Output resource actions
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->output_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Get action name needed to compare digest for configuration changes
	*
	* \param[in] task Action name from history
	* \param[in] interval_ms Action interval (in milliseconds)
	*
	* \return Action name whose digest should be compared
	*/
	static const char *
	task_for_digest(const char *task, guint interval_ms)
	{
	/* Certain actions need to be compared against the parameters used to start
	* the resource.
	*/
	if ((interval_ms == 0)
	&& pcmk__str_any_of(task, PCMK_ACTION_MONITOR, PCMK_ACTION_MIGRATE_FROM,
	PCMK_ACTION_PROMOTE, NULL)) {
	task = PCMK_ACTION_START;
	}
	return task;
	}

	/*!
	* \internal
	* \brief Check whether only sanitized parameters to an action changed
	*
	* When collecting CIB files for troubleshooting, crm_report will mask
	* sensitive resource parameters. If simulations were run using that, affected
	* resources would appear to need a restart, which would complicate
	* troubleshooting. To avoid that, we save a "secure digest" of non-sensitive
	* parameters. This function used that digest to check whether only masked
	* parameters are different.
	*
	* \param[in] xml_op Resource history entry with secure digest
	* \param[in] digest_data Operation digest information being compared
	* \param[in] scheduler Scheduler data
	*
	* \return true if only sanitized parameters changed, otherwise false
	*/
	static bool
	only_sanitized_changed(const xmlNode *xml_op,
	const pcmk__op_digest_t *digest_data,
	const pcmk_scheduler_t *scheduler)
	{
	const char *digest_secure = NULL;

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_sanitized)) {
	// The scheduler is not being run as a simulation
	return false;
	}

	digest_secure = crm_element_value(xml_op, PCMK__XA_OP_SECURE_DIGEST);

	return (digest_data->rc != pcmk__digest_match) && (digest_secure != NULL)
	&& (digest_data->digest_secure_calc != NULL)
	&& (strcmp(digest_data->digest_secure_calc, digest_secure) == 0);
	}

	/*!
	* \internal
	* \brief Force a restart due to a configuration change
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] task Name of action whose configuration changed
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in,out] node Node where resource should be restarted
	*/
	static void
	force_restart(pcmk_resource_t rsc, const char task, guint interval_ms,
	pcmk_node_t *node)
	{
	char *key = pcmk__op_key(rsc->id, task, interval_ms);
	pcmk_action_t *required = custom_action(rsc, key, task, NULL, FALSE,
	rsc->cluster);

	pe_action_set_reason(required, "resource definition change", true);
	trigger_unfencing(rsc, node, "Device parameters changed", NULL,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Schedule a reload of a resource on a node
	*
	* \param[in,out] data Resource to reload
	* \param[in] user_data Where resource should be reloaded
	*/
	static void
	schedule_reload(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;
	pcmk_action_t *reload = NULL;

	// For collective resources, just call recursively for children
	if (rsc->variant > pcmk_rsc_variant_primitive) {
	g_list_foreach(rsc->children, schedule_reload, user_data);
	return;
	}

	// Skip the reload in certain situations
	if ((node == NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_managed)
	\|\| pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	pcmk__rsc_trace(rsc, "Skip reload of %s:%s%s %s",
	rsc->id,
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " unmanaged",
	pcmk_is_set(rsc->flags, pcmk_rsc_failed)? " failed" : "",
	(node == NULL)? "inactive" : node->details->uname);
	return;
	}

	/* If a resource's configuration changed while a start was pending,
	* force a full restart instead of a reload.
	*/
	if (pcmk_is_set(rsc->flags, pcmk_rsc_start_pending)) {
	pcmk__rsc_trace(rsc,
	"%s: preventing agent reload because start pending",
	rsc->id);
	custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE,
	rsc->cluster);
	return;
	}

	// Schedule the reload
	pcmk__set_rsc_flags(rsc, pcmk_rsc_reload);
	reload = custom_action(rsc, reload_key(rsc), PCMK_ACTION_RELOAD_AGENT, node,
	FALSE, rsc->cluster);
	pe_action_set_reason(reload, "resource definition change", FALSE);

	// Set orderings so that a required stop or demote cancels the reload
	pcmk__new_ordering(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
	pcmk__ar_ordered\|pcmk__ar_then_cancels_first,
	rsc->cluster);
	pcmk__new_ordering(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
	pcmk__ar_ordered\|pcmk__ar_then_cancels_first,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Handle any configuration change for an action
	*
	* Given an action from resource history, if the resource's configuration
	* changed since the action was done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, etc.).
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in,out] node Node that action was on
	* \param[in] xml_op Action XML from resource history
	*
	* \return true if action configuration changed, otherwise false
	*/
	bool
	pcmk__check_action_config(pcmk_resource_t rsc, pcmk_node_t node,
	const xmlNode *xml_op)
	{
	guint interval_ms = 0;
	const char *task = NULL;
	const pcmk__op_digest_t *digest_data = NULL;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (xml_op != NULL),
	return false);

	task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	CRM_CHECK(task != NULL, return false);

	crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &interval_ms);

	// If this is a recurring action, check whether it has been orphaned
	if (interval_ms > 0) {
	if (pcmk__find_action_config(rsc, task, interval_ms, false) != NULL) {
	pcmk__rsc_trace(rsc,
	"%s-interval %s for %s on %s is in configuration",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pcmk__node_name(node));
	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_cancel_removed_actions)) {
	pcmk__schedule_cancel(rsc,
	crm_element_value(xml_op, PCMK__XA_CALL_ID),
	task, interval_ms, node, "orphan");
	return true;
	} else {
	pcmk__rsc_debug(rsc, "%s-interval %s for %s on %s is orphaned",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pcmk__node_name(node));
	return true;
	}
	}

	crm_trace("Checking %s-interval %s for %s on %s for configuration changes",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pcmk__node_name(node));
	task = task_for_digest(task, interval_ms);
	digest_data = rsc_action_digest_cmp(rsc, xml_op, node, rsc->cluster);

	if (only_sanitized_changed(xml_op, digest_data, rsc->cluster)) {
	if (!pcmk__is_daemon && (rsc->cluster->priv != NULL)) {
	pcmk__output_t *out = rsc->cluster->priv;

	out->info(out,
	"Only 'private' parameters to %s-interval %s for %s "
	"on %s changed: %s",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pcmk__node_name(node),
	crm_element_value(xml_op, PCMK__XA_TRANSITION_MAGIC));
	}
	return false;
	}

	switch (digest_data->rc) {
	case pcmk__digest_restart:
	crm_log_xml_debug(digest_data->params_restart, "params:restart");
	force_restart(rsc, task, interval_ms, node);
	return true;

	case pcmk__digest_unknown:
	case pcmk__digest_mismatch:
	// Changes that can potentially be handled by an agent reload

	if (interval_ms > 0) {
	/* Recurring actions aren't reloaded per se, they are just
	* re-scheduled so the next run uses the new parameters.
	* The old instance will be cancelled automatically.
	*/
	crm_log_xml_debug(digest_data->params_all, "params:reschedule");
	pcmk__reschedule_recurring(rsc, task, interval_ms, node);

	} else if (crm_element_value(xml_op,
	PCMK__XA_OP_RESTART_DIGEST) != NULL) {
	// Agent supports reload, so use it
	trigger_unfencing(rsc, node,
	"Device parameters changed (reload)", NULL,
	rsc->cluster);
	crm_log_xml_debug(digest_data->params_all, "params:reload");
	schedule_reload((gpointer) rsc, (gpointer) node);

	} else {
	pcmk__rsc_trace(rsc,
	"Restarting %s "
	"because agent doesn't support reload",
	rsc->id);
	crm_log_xml_debug(digest_data->params_restart,
	"params:restart");
	force_restart(rsc, task, interval_ms, node);
	}
	return true;

	default:
	break;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Create a list of resource's action history entries, sorted by call ID
	*
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[out] start_index Where to store index of start-like action, if any
	* \param[out] stop_index Where to store index of stop action, if any
	*/
	static GList *
	rsc_history_as_list(const xmlNode rsc_entry, int start_index, int *stop_index)
	{
	GList *ops = NULL;

	for (xmlNode *rsc_op = first_named_child(rsc_entry, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {
	ops = g_list_prepend(ops, rsc_op);
	}
	ops = g_list_sort(ops, sort_op_by_callid);
	calculate_active_ops(ops, start_index, stop_index);
	return ops;
	}

	/*!
	* \internal
	* \brief Process a resource's action history from the CIB status
	*
	* Given a resource's action history, if the resource's configuration
	* changed since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[in,out] rsc Resource whose history is being processed
	* \param[in,out] node Node whose history is being processed
	*/
	static void
	process_rsc_history(const xmlNode rsc_entry, pcmk_resource_t rsc,
	pcmk_node_t *node)
	{
	int offset = -1;
	int stop_index = 0;
	int start_index = 0;
	GList *sorted_op_list = NULL;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pe_rsc_is_anon_clone(pe__const_top_resource(rsc, false))) {
	pcmk__rsc_trace(rsc,
	"Skipping configuration check "
	"for orphaned clone instance %s",
	rsc->id);
	} else {
	pcmk__rsc_trace(rsc,
	"Skipping configuration check and scheduling "
	"clean-up for orphaned resource %s", rsc->id);
	pcmk__schedule_cleanup(rsc, node, false);
	}
	return;
	}

	if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, false)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}
	pcmk__rsc_trace(rsc,
	"Skipping configuration check for %s "
	"because no longer active on %s",
	rsc->id, pcmk__node_name(node));
	return;
	}

	pcmk__rsc_trace(rsc, "Checking for configuration changes for %s on %s",
	rsc->id, pcmk__node_name(node));

	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, true)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}

	sorted_op_list = rsc_history_as_list(rsc_entry, &start_index, &stop_index);
	if (start_index < stop_index) {
	return; // Resource is stopped
	}

	for (GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	xmlNode rsc_op = (xmlNode ) iter->data;
	const char *task = NULL;
	guint interval_ms = 0;

	if (++offset < start_index) {
	// Skip actions that happened before a start
	continue;
	}

	task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	crm_element_value_ms(rsc_op, PCMK_META_INTERVAL, &interval_ms);

	if ((interval_ms > 0)
	&& (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)
	\|\| node->details->maintenance)) {
	// Maintenance mode cancels recurring operations
	pcmk__schedule_cancel(rsc,
	crm_element_value(rsc_op, PCMK__XA_CALL_ID),
	task, interval_ms, node, "maintenance mode");

	} else if ((interval_ms > 0)
	\|\| pcmk__strcase_any_of(task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* If a resource operation failed, and the operation's definition
	* has changed, clear any fail count so they can be retried fresh.
	*/

	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't assigned resources to nodes yet, so if the
	* REMOTE_CONTAINER_HACK is used, we may calculate the digest
	* based on the literal "#uname" value rather than the properly
	* substituted value. That would mistakenly make the action
	* definition appear to have been changed. Defer the check until
	* later in this case.
	*/
	pe__add_param_check(rsc_op, rsc, node, pcmk__check_active,
	rsc->cluster);

	} else if (pcmk__check_action_config(rsc, node, rsc_op)
	&& (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
	NULL) != 0)) {
	pe__clear_failcount(rsc, node, "action definition changed",
	rsc->cluster);
	}
	}
	}
	g_list_free(sorted_op_list);
	}

	/*!
	* \internal
	* \brief Process a node's action history from the CIB status
	*
	* Given a node's resource history, if the resource's configuration changed
	* since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in,out] node Node whose history is being processed
	* \param[in] lrm_rscs Node's <lrm_resources> from CIB status XML
	*/
	static void
	process_node_history(pcmk_node_t node, const xmlNode lrm_rscs)
	{
	crm_trace("Processing node history for %s", pcmk__node_name(node));
	for (const xmlNode *rsc_entry = first_named_child(lrm_rscs,
	XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	if (rsc_entry->children != NULL) {
	GList *result = pcmk__rscs_matching_id(ID(rsc_entry),
	node->details->data_set);

	for (GList *iter = result; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->variant == pcmk_rsc_variant_primitive) {
	process_rsc_history(rsc_entry, rsc, node);
	}
	}
	g_list_free(result);
	}
	}
	}

	// XPath to find a node's resource history
	#define XPATH_NODE_HISTORY "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='%s']" \
	"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES

	/*!
	* \internal
	* \brief Process any resource configuration changes in the CIB status
	*
	* Go through all nodes' resource history, and if a resource's configuration
	* changed since its actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__handle_rsc_config_changes(pcmk_scheduler_t *scheduler)
	{
	crm_trace("Check resource and action configuration for changes");

	/* Rather than iterate through the status section, iterate through the nodes
	* and search for the appropriate status subsection for each. This skips
	* orphaned nodes and lets us eliminate some cases before searching the XML.
	*/
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	/* Don't bother checking actions for a node that can't run actions ...
	* unless it's in maintenance mode, in which case we still need to
	* cancel any existing recurring monitors.
	*/
	if (node->details->maintenance
	\|\| pcmk__node_available(node, false, false)) {

	char *xpath = NULL;
	xmlNode *history = NULL;

	xpath = crm_strdup_printf(XPATH_NODE_HISTORY, node->details->uname);
	history = get_xpath_object(xpath, scheduler->input, LOG_NEVER);
	free(xpath);

	process_node_history(node, history);
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_bundle.c b/lib/pacemaker/pcmk_sched_bundle.c
	index bef7975d95..615004034d 100644
	--- a/lib/pacemaker/pcmk_sched_bundle.c
	+++ b/lib/pacemaker/pcmk_sched_bundle.c
	@@ -1,1060 +1,1060 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	struct assign_data {
	const pcmk_node_t *prefer;
	bool stop_if_fail;
	};

	/*!
	* \internal
	* \brief Assign a single bundle replica's resources (other than container)
	*
	* \param[in,out] replica Replica to assign
	* \param[in] user_data Preferred node, if any
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	assign_replica(pcmk__bundle_replica_t replica, void user_data)
	{
	pcmk_node_t *container_host = NULL;

	struct assign_data *assign_data = user_data;
	const pcmk_node_t *prefer = assign_data->prefer;
	bool stop_if_fail = assign_data->stop_if_fail;

	const pcmk_resource_t *bundle = pe__const_top_resource(replica->container,
	true);

	if (replica->ip != NULL) {
	pcmk__rsc_trace(bundle, "Assigning bundle %s IP %s",
	bundle->id, replica->ip->id);
	replica->ip->cmds->assign(replica->ip, prefer, stop_if_fail);
	}

	container_host = replica->container->allocated_to;
	if (replica->remote != NULL) {
	if (pe__is_guest_or_remote_node(container_host)) {
	/* REMOTE_CONTAINER_HACK: "Nested" connection resources must be on
	* the same host because Pacemaker Remote only supports a single
	* active connection.
	*/
	pcmk__new_colocation("#replica-remote-with-host-remote", NULL,
	INFINITY, replica->remote,
	container_host->details->remote_rsc, NULL,
	NULL, pcmk__coloc_influence);
	}
	pcmk__rsc_trace(bundle, "Assigning bundle %s connection %s",
	bundle->id, replica->remote->id);
	replica->remote->cmds->assign(replica->remote, prefer, stop_if_fail);
	}

	if (replica->child != NULL) {
	pcmk_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, replica->child->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	- if (!pe__same_node(node, replica->node)) {
	+ if (!pcmk__same_node(node, replica->node)) {
	node->weight = -INFINITY;
	} else if (!pcmk__threshold_reached(replica->child, node, NULL)) {
	node->weight = INFINITY;
	}
	}

	pcmk__set_rsc_flags(replica->child->parent, pcmk_rsc_assigning);
	pcmk__rsc_trace(bundle, "Assigning bundle %s replica child %s",
	bundle->id, replica->child->id);
	replica->child->cmds->assign(replica->child, replica->node,
	stop_if_fail);
	pcmk__clear_rsc_flags(replica->child->parent, pcmk_rsc_assigning);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Assign a bundle resource to a node
	*
	* \param[in,out] rsc Resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and a primitive descendant of \p rsc
	* can't be assigned to a node, set the
	* descendant's next role to stopped and update
	* existing actions
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pcmk_node_t *
	pcmk__bundle_assign(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *containers = NULL;
	pcmk_resource_t *bundled_resource = NULL;
	struct assign_data assign_data = { prefer, stop_if_fail };

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle));

	pcmk__rsc_trace(rsc, "Assigning bundle %s", rsc->id);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_assigning);

	pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->allowed_nodes, rsc->cluster);

	// Assign all containers first, so we know what nodes the bundle will be on
	containers = g_list_sort(pe__bundle_containers(rsc), pcmk__cmp_instance);
	pcmk__assign_instances(rsc, containers, pe__bundle_max(rsc),
	rsc->fns->max_per_node(rsc));
	g_list_free(containers);

	// Then assign remaining replica resources
	pe__foreach_bundle_replica(rsc, assign_replica, (void *) &assign_data);

	// Finally, assign the bundled resources to each bundle node
	bundled_resource = pe__bundled_resource(rsc);
	if (bundled_resource != NULL) {
	pcmk_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, bundled_resource->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
	if (pe__node_is_bundle_instance(rsc, node)) {
	node->weight = 0;
	} else {
	node->weight = -INFINITY;
	}
	}
	bundled_resource->cmds->assign(bundled_resource, prefer, stop_if_fail);
	}

	pcmk__clear_rsc_flags(rsc, pcmk_rsc_assigning\|pcmk_rsc_unassigned);
	return NULL;
	}

	/*!
	* \internal
	* \brief Create actions for a bundle replica's resources (other than child)
	*
	* \param[in,out] replica Replica to create actions for
	* \param[in] user_data Unused
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	create_replica_actions(pcmk__bundle_replica_t replica, void user_data)
	{
	if (replica->ip != NULL) {
	replica->ip->cmds->create_actions(replica->ip);
	}
	if (replica->container != NULL) {
	replica->container->cmds->create_actions(replica->container);
	}
	if (replica->remote != NULL) {
	replica->remote->cmds->create_actions(replica->remote);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given bundle resource
	*
	* \param[in,out] rsc Bundle resource to create actions for
	*/
	void
	pcmk__bundle_create_actions(pcmk_resource_t *rsc)
	{
	pcmk_action_t *action = NULL;
	GList *containers = NULL;
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle));

	pe__foreach_bundle_replica(rsc, create_replica_actions, NULL);

	containers = pe__bundle_containers(rsc);
	pcmk__create_instance_actions(rsc, containers);
	g_list_free(containers);

	bundled_resource = pe__bundled_resource(rsc);
	if (bundled_resource != NULL) {
	bundled_resource->cmds->create_actions(bundled_resource);

	if (pcmk_is_set(bundled_resource->flags, pcmk_rsc_promotable)) {
	pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_PROMOTE, true, true);
	action = pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_PROMOTED,
	true, true);
	action->priority = INFINITY;

	pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_DEMOTE, true, true);
	action = pe__new_rsc_pseudo_action(rsc, PCMK_ACTION_DEMOTED,
	true, true);
	action->priority = INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Create internal constraints for a bundle replica's resources
	*
	* \param[in,out] replica Replica to create internal constraints for
	* \param[in,out] user_data Replica's parent bundle
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	replica_internal_constraints(pcmk__bundle_replica_t replica, void user_data)
	{
	pcmk_resource_t *bundle = user_data;

	replica->container->cmds->internal_constraints(replica->container);

	// Start bundle -> start replica container
	pcmk__order_starts(bundle, replica->container,
	pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_then_implies_first_graphed);

	// Stop bundle -> stop replica child and container
	if (replica->child != NULL) {
	pcmk__order_stops(bundle, replica->child,
	pcmk__ar_then_implies_first_graphed);
	}
	pcmk__order_stops(bundle, replica->container,
	pcmk__ar_then_implies_first_graphed);

	// Start replica container -> bundle is started
	pcmk__order_resource_actions(replica->container, PCMK_ACTION_START, bundle,
	PCMK_ACTION_RUNNING,
	pcmk__ar_first_implies_then_graphed);

	// Stop replica container -> bundle is stopped
	pcmk__order_resource_actions(replica->container, PCMK_ACTION_STOP, bundle,
	PCMK_ACTION_STOPPED,
	pcmk__ar_first_implies_then_graphed);

	if (replica->ip != NULL) {
	replica->ip->cmds->internal_constraints(replica->ip);

	// Replica IP address -> replica container (symmetric)
	pcmk__order_starts(replica->ip, replica->container,
	pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_guest_allowed);
	pcmk__order_stops(replica->container, replica->ip,
	pcmk__ar_then_implies_first\|pcmk__ar_guest_allowed);

	pcmk__new_colocation("#ip-with-container", NULL, INFINITY, replica->ip,
	replica->container, NULL, NULL,
	pcmk__coloc_influence);
	}

	if (replica->remote != NULL) {
	/* This handles ordering and colocating remote relative to container
	* (via "#resource-with-container"). Since IP is also ordered and
	* colocated relative to the container, we don't need to do anything
	* explicit here with IP.
	*/
	replica->remote->cmds->internal_constraints(replica->remote);
	}

	if (replica->child != NULL) {
	CRM_ASSERT(replica->remote != NULL);
	// "Start remote then child" is implicit in scheduler's remote logic
	}
	return true;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a bundle resource
	*
	* \param[in,out] rsc Bundle resource to create implicit constraints for
	*/
	void
	pcmk__bundle_internal_constraints(pcmk_resource_t *rsc)
	{
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle));

	pe__foreach_bundle_replica(rsc, replica_internal_constraints, rsc);

	bundled_resource = pe__bundled_resource(rsc);
	if (bundled_resource == NULL) {
	return;
	}

	// Start bundle -> start bundled clone
	pcmk__order_resource_actions(rsc, PCMK_ACTION_START, bundled_resource,
	PCMK_ACTION_START,
	pcmk__ar_then_implies_first_graphed);

	// Bundled clone is started -> bundle is started
	pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_RUNNING,
	pcmk__ar_first_implies_then_graphed);

	// Stop bundle -> stop bundled clone
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP, bundled_resource,
	PCMK_ACTION_STOP,
	pcmk__ar_then_implies_first_graphed);

	// Bundled clone is stopped -> bundle is stopped
	pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_STOPPED,
	pcmk__ar_first_implies_then_graphed);

	bundled_resource->cmds->internal_constraints(bundled_resource);

	if (!pcmk_is_set(bundled_resource->flags, pcmk_rsc_promotable)) {
	return;
	}
	pcmk__promotable_restart_ordering(rsc);

	// Demote bundle -> demote bundled clone
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE, bundled_resource,
	PCMK_ACTION_DEMOTE,
	pcmk__ar_then_implies_first_graphed);

	// Bundled clone is demoted -> bundle is demoted
	pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_DEMOTED,
	pcmk__ar_first_implies_then_graphed);

	// Promote bundle -> promote bundled clone
	pcmk__order_resource_actions(rsc, PCMK_ACTION_PROMOTE,
	bundled_resource, PCMK_ACTION_PROMOTE,
	pcmk__ar_then_implies_first_graphed);

	// Bundled clone is promoted -> bundle is promoted
	pcmk__order_resource_actions(bundled_resource, PCMK_ACTION_PROMOTED,
	rsc, PCMK_ACTION_PROMOTED,
	pcmk__ar_first_implies_then_graphed);
	}

	struct match_data {
	const pcmk_node_t *node; // Node to compare against replica
	pcmk_resource_t *container; // Replica container corresponding to node
	};

	/*!
	* \internal
	* \brief Check whether a replica container is assigned to a given node
	*
	* \param[in] replica Replica to check
	* \param[in,out] user_data struct match_data with node to compare against
	*
	* \return true if the replica does not match (to indicate further replicas
	* should be processed), otherwise false
	*/
	static bool
	match_replica_container(const pcmk__bundle_replica_t replica, void user_data)
	{
	struct match_data *match_data = user_data;

	if (pcmk__instance_matches(replica->container, match_data->node,
	pcmk_role_unknown, false)) {
	match_data->container = replica->container;
	return false; // Match found, don't bother searching further replicas
	}
	return true; // No match, keep searching
	}

	/*!
	* \internal
	* \brief Get the host to which a bundle node is assigned
	*
	* \param[in] node Possible bundle node to check
	*
	* \return Node to which the container for \p node is assigned if \p node is a
	* bundle node, otherwise \p node itself
	*/
	static const pcmk_node_t *
	get_bundle_node_host(const pcmk_node_t *node)
	{
	if (pe__is_bundle_node(node)) {
	const pcmk_resource_t *container = node->details->remote_rsc->container;

	return container->fns->location(container, NULL, 0);
	}
	return node;
	}

	/*!
	* \internal
	* \brief Find a bundle container compatible with a dependent resource
	*
	* \param[in] dependent Dependent resource in colocation with bundle
	* \param[in] bundle Bundle that \p dependent is colocated with
	*
	* \return A container from \p bundle assigned to the same node as \p dependent
	* if assigned, otherwise assigned to any of dependent's allowed nodes,
	* otherwise NULL.
	*/
	static pcmk_resource_t *
	compatible_container(const pcmk_resource_t *dependent,
	const pcmk_resource_t *bundle)
	{
	GList *scratch = NULL;
	struct match_data match_data = { NULL, NULL };

	// If dependent is assigned, only check there
	match_data.node = dependent->fns->location(dependent, NULL, 0);
	match_data.node = get_bundle_node_host(match_data.node);
	if (match_data.node != NULL) {
	pe__foreach_const_bundle_replica(bundle, match_replica_container,
	&match_data);
	return match_data.container;
	}

	// Otherwise, check for any of the dependent's allowed nodes
	scratch = g_hash_table_get_values(dependent->allowed_nodes);
	scratch = pcmk__sort_nodes(scratch, NULL);
	for (const GList *iter = scratch; iter != NULL; iter = iter->next) {
	match_data.node = iter->data;
	match_data.node = get_bundle_node_host(match_data.node);
	if (match_data.node == NULL) {
	continue;
	}

	pe__foreach_const_bundle_replica(bundle, match_replica_container,
	&match_data);
	if (match_data.container != NULL) {
	break;
	}
	}
	g_list_free(scratch);
	return match_data.container;
	}

	struct coloc_data {
	const pcmk__colocation_t *colocation;
	pcmk_resource_t *dependent;
	GList *container_hosts;
	};

	/*!
	* \internal
	* \brief Apply a colocation score to replica node scores or resource priority
	*
	* \param[in] replica Replica of primary bundle resource in colocation
	* \param[in,out] user_data struct coloc_data for colocation being applied
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	replica_apply_coloc_score(const pcmk__bundle_replica_t *replica,
	void *user_data)
	{
	struct coloc_data *coloc_data = user_data;
	pcmk_node_t *chosen = NULL;

	if (coloc_data->colocation->score < INFINITY) {
	replica->container->cmds->apply_coloc_score(coloc_data->dependent,
	replica->container,
	coloc_data->colocation,
	false);
	return true;
	}

	chosen = replica->container->fns->location(replica->container, NULL, 0);
	if ((chosen == NULL)
	\|\| is_set_recursive(replica->container, pcmk_rsc_blocked, true)) {
	return true;
	}

	if ((coloc_data->colocation->primary_role >= pcmk_role_promoted)
	&& ((replica->child == NULL)
	\|\| (replica->child->next_role < pcmk_role_promoted))) {
	return true;
	}

	pcmk__rsc_trace(pe__const_top_resource(replica->container, true),
	"Allowing mandatory colocation %s using %s @%d",
	coloc_data->colocation->id, pcmk__node_name(chosen),
	chosen->weight);
	coloc_data->container_hosts = g_list_prepend(coloc_data->container_hosts,
	chosen);
	return true;
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__bundle_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	struct coloc_data coloc_data = { colocation, dependent, NULL };

	/* This should never be called for the bundle itself as a dependent.
	* Instead, we add its colocation constraints to its containers and bundled
	* primitive and call the apply_coloc_score() method for them as dependents.
	*/
	CRM_ASSERT((primary != NULL)
	&& (primary->variant == pcmk_rsc_variant_bundle)
	&& (dependent != NULL)
	&& (dependent->variant == pcmk_rsc_variant_primitive)
	&& (colocation != NULL) && !for_dependent);

	if (pcmk_is_set(primary->flags, pcmk_rsc_unassigned)) {
	pcmk__rsc_trace(primary,
	"Skipping applying colocation %s "
	"because %s is still provisional",
	colocation->id, primary->id);
	return;
	}
	pcmk__rsc_trace(primary, "Applying colocation %s (%s with %s at %s)",
	colocation->id, dependent->id, primary->id,
	pcmk_readable_score(colocation->score));

	/* If the constraint dependent is a clone or bundle, "dependent" here is one
	* of its instances. Look for a compatible instance of this bundle.
	*/
	if (colocation->dependent->variant > pcmk_rsc_variant_group) {
	const pcmk_resource_t *primary_container = NULL;

	primary_container = compatible_container(dependent, primary);
	if (primary_container != NULL) { // Success, we found one
	pcmk__rsc_debug(primary, "Pairing %s with %s",
	dependent->id, primary_container->id);
	dependent->cmds->apply_coloc_score(dependent, primary_container,
	colocation, true);

	} else if (colocation->score >= INFINITY) { // Failure, and it's fatal
	crm_notice("%s cannot run because there is no compatible "
	"instance of %s to colocate with",
	dependent->id, primary->id);
	pcmk__assign_resource(dependent, NULL, true, true);

	} else { // Failure, but we can ignore it
	pcmk__rsc_debug(primary,
	"%s cannot be colocated with any instance of %s",
	dependent->id, primary->id);
	}
	return;
	}

	pe__foreach_const_bundle_replica(primary, replica_apply_coloc_score,
	&coloc_data);

	if (colocation->score >= INFINITY) {
	pcmk__colocation_intersect_nodes(dependent, primary, colocation,
	coloc_data.container_hosts, false);
	}
	g_list_free(coloc_data.container_hosts);
	}

	// Bundle implementation of pcmk_assignment_methods_t:with_this_colocations()
	void
	pcmk__with_bundle_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	const pcmk_resource_t *bundled_rsc = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)
	&& (orig_rsc != NULL) && (list != NULL));

	// The bundle itself and its containers always get its colocations
	if ((orig_rsc == rsc)
	\|\| pcmk_is_set(orig_rsc->flags, pcmk_rsc_replica_container)) {

	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);
	return;
	}

	/* The bundled resource gets the colocations if it's promotable and we've
	* begun choosing roles
	*/
	bundled_rsc = pe__bundled_resource(rsc);
	if ((bundled_rsc == NULL)
	\|\| !pcmk_is_set(bundled_rsc->flags, pcmk_rsc_promotable)
	\|\| (pe__const_top_resource(orig_rsc, false) != bundled_rsc)) {
	return;
	}

	if (orig_rsc == bundled_rsc) {
	if (pe__clone_flag_is_set(orig_rsc,
	pcmk__clone_promotion_constrained)) {
	/* orig_rsc is the clone and we're setting roles (or have already
	* done so)
	*/
	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);
	}

	} else if (!pcmk_is_set(orig_rsc->flags, pcmk_rsc_unassigned)) {
	/* orig_rsc is an instance and is already assigned. If something
	* requests colocations for orig_rsc now, it's for setting roles.
	*/
	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);
	}
	}

	// Bundle implementation of pcmk_assignment_methods_t:this_with_colocations()
	void
	pcmk__bundle_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	const pcmk_resource_t *bundled_rsc = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)
	&& (orig_rsc != NULL) && (list != NULL));

	// The bundle itself and its containers always get its colocations
	if ((orig_rsc == rsc)
	\|\| pcmk_is_set(orig_rsc->flags, pcmk_rsc_replica_container)) {

	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);
	return;
	}

	/* The bundled resource gets the colocations if it's promotable and we've
	* begun choosing roles
	*/
	bundled_rsc = pe__bundled_resource(rsc);
	if ((bundled_rsc == NULL)
	\|\| !pcmk_is_set(bundled_rsc->flags, pcmk_rsc_promotable)
	\|\| (pe__const_top_resource(orig_rsc, false) != bundled_rsc)) {
	return;
	}

	if (orig_rsc == bundled_rsc) {
	if (pe__clone_flag_is_set(orig_rsc,
	pcmk__clone_promotion_constrained)) {
	/* orig_rsc is the clone and we're setting roles (or have already
	* done so)
	*/
	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);
	}

	} else if (!pcmk_is_set(orig_rsc->flags, pcmk_rsc_unassigned)) {
	/* orig_rsc is an instance and is already assigned. If something
	* requests colocations for orig_rsc now, it's for setting roles.
	*/
	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given bundle resource action
	*
	* \param[in,out] action Bundle resource action to get flags for
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__bundle_action_flags(pcmk_action_t action, const pcmk_node_t node)
	{
	GList *containers = NULL;
	uint32_t flags = 0;
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT((action != NULL) && (action->rsc != NULL)
	&& (action->rsc->variant == pcmk_rsc_variant_bundle));

	bundled_resource = pe__bundled_resource(action->rsc);
	if (bundled_resource != NULL) {
	// Clone actions are done on the bundled clone resource, not container
	switch (get_complex_task(bundled_resource, action->task)) {
	case pcmk_action_unspecified:
	case pcmk_action_notify:
	case pcmk_action_notified:
	case pcmk_action_promote:
	case pcmk_action_promoted:
	case pcmk_action_demote:
	case pcmk_action_demoted:
	return pcmk__collective_action_flags(action,
	bundled_resource->children,
	node);
	default:
	break;
	}
	}

	containers = pe__bundle_containers(action->rsc);
	flags = pcmk__collective_action_flags(action, containers, node);
	g_list_free(containers);
	return flags;
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a bundle replica
	*
	* \param[in,out] replica Replica to apply constraint to
	* \param[in,out] user_data Location constraint to apply
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	apply_location_to_replica(pcmk__bundle_replica_t replica, void user_data)
	{
	pcmk__location_t *location = user_data;

	if (replica->container != NULL) {
	replica->container->cmds->apply_location(replica->container, location);
	}
	if (replica->ip != NULL) {
	replica->ip->cmds->apply_location(replica->ip, location);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a bundle resource's allowed node scores
	*
	* \param[in,out] rsc Bundle resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*/
	void
	pcmk__bundle_apply_location(pcmk_resource_t rsc, pcmk__location_t location)
	{
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle)
	&& (location != NULL));

	pcmk__apply_location(rsc, location);
	pe__foreach_bundle_replica(rsc, apply_location_to_replica, location);

	bundled_resource = pe__bundled_resource(rsc);
	if ((bundled_resource != NULL)
	&& ((location->role_filter == pcmk_role_unpromoted)
	\|\| (location->role_filter == pcmk_role_promoted))) {
	bundled_resource->cmds->apply_location(bundled_resource, location);
	bundled_resource->rsc_location = g_list_prepend(
	bundled_resource->rsc_location, location);
	}
	}

	#define XPATH_REMOTE "//nvpair[@name='" PCMK_REMOTE_RA_ADDR "']"

	/*!
	* \internal
	* \brief Add a bundle replica's actions to transition graph
	*
	* \param[in,out] replica Replica to add to graph
	* \param[in] user_data Bundle that replica belongs to (for logging only)
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	add_replica_actions_to_graph(pcmk__bundle_replica_t replica, void user_data)
	{
	if ((replica->remote != NULL) && (replica->container != NULL)
	&& pe__bundle_needs_remote_name(replica->remote)) {

	/* REMOTE_CONTAINER_HACK: Allow remote nodes to run containers that
	* run pacemaker-remoted inside, without needing a separate IP for
	* the container. This is done by configuring the inner remote's
	* connection host as the magic string "#uname", then
	* replacing it with the underlying host when needed.
	*/
	xmlNode *nvpair = get_xpath_object(XPATH_REMOTE, replica->remote->xml,
	LOG_ERR);
	const char *calculated_addr = NULL;

	// Replace the value in replica->remote->xml (if appropriate)
	calculated_addr = pe__add_bundle_remote_name(replica->remote,
	replica->remote->cluster,
	nvpair, PCMK_XA_VALUE);
	if (calculated_addr != NULL) {
	/* Since this is for the bundle as a resource, and not any
	* particular action, replace the value in the default
	* parameters (not evaluated for node). create_graph_action()
	* will grab it from there to replace it in node-evaluated
	* parameters.
	*/
	GHashTable *params = pe_rsc_params(replica->remote,
	NULL, replica->remote->cluster);

	g_hash_table_replace(params,
	strdup(PCMK_REMOTE_RA_ADDR),
	strdup(calculated_addr));
	} else {
	pcmk_resource_t *bundle = user_data;

	/* The only way to get here is if the remote connection is
	* neither currently running nor scheduled to run. That means we
	* won't be doing any operations that require addr (only start
	* requires it; we additionally use it to compare digests when
	* unpacking status, promote, and migrate_from history, but
	* that's already happened by this point).
	*/
	pcmk__rsc_info(bundle,
	"Unable to determine address for bundle %s "
	"remote connection", bundle->id);
	}
	}
	if (replica->ip != NULL) {
	replica->ip->cmds->add_actions_to_graph(replica->ip);
	}
	if (replica->container != NULL) {
	replica->container->cmds->add_actions_to_graph(replica->container);
	}
	if (replica->remote != NULL) {
	replica->remote->cmds->add_actions_to_graph(replica->remote);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Add a bundle resource's actions to the transition graph
	*
	* \param[in,out] rsc Bundle resource whose actions should be added
	*/
	void
	pcmk__bundle_add_actions_to_graph(pcmk_resource_t *rsc)
	{
	pcmk_resource_t *bundled_resource = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle));

	bundled_resource = pe__bundled_resource(rsc);
	if (bundled_resource != NULL) {
	bundled_resource->cmds->add_actions_to_graph(bundled_resource);
	}
	pe__foreach_bundle_replica(rsc, add_replica_actions_to_graph, rsc);
	}

	struct probe_data {
	pcmk_resource_t *bundle; // Bundle being probed
	pcmk_node_t *node; // Node to create probes on
	bool any_created; // Whether any probes have been created
	};

	/*!
	* \internal
	* \brief Order a bundle replica's start after another replica's probe
	*
	* \param[in,out] replica Replica to order start for
	* \param[in,out] user_data Replica with probe to order after
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	order_replica_start_after(pcmk__bundle_replica_t replica, void user_data)
	{
	pcmk__bundle_replica_t *probed_replica = user_data;

	if ((replica == probed_replica) \|\| (replica->container == NULL)) {
	return true;
	}
	pcmk__new_ordering(probed_replica->container,
	pcmk__op_key(probed_replica->container->id,
	PCMK_ACTION_MONITOR, 0),
	NULL, replica->container,
	pcmk__op_key(replica->container->id, PCMK_ACTION_START,
	0),
	NULL, pcmk__ar_ordered\|pcmk__ar_if_on_same_node,
	replica->container->cluster);
	return true;
	}

	/*!
	* \internal
	* \brief Create probes for a bundle replica's resources
	*
	* \param[in,out] replica Replica to create probes for
	* \param[in,out] user_data struct probe_data
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	create_replica_probes(pcmk__bundle_replica_t replica, void user_data)
	{
	struct probe_data *probe_data = user_data;

	if ((replica->ip != NULL)
	&& replica->ip->cmds->create_probe(replica->ip, probe_data->node)) {
	probe_data->any_created = true;
	}
	if ((replica->child != NULL)
	- && pe__same_node(probe_data->node, replica->node)
	+ && pcmk__same_node(probe_data->node, replica->node)
	&& replica->child->cmds->create_probe(replica->child,
	probe_data->node)) {
	probe_data->any_created = true;
	}
	if ((replica->container != NULL)
	&& replica->container->cmds->create_probe(replica->container,
	probe_data->node)) {
	probe_data->any_created = true;

	/* If we're limited to one replica per host (due to
	* the lack of an IP range probably), then we don't
	* want any of our peer containers starting until
	* we've established that no other copies are already
	* running.
	*
	* Partly this is to ensure that the maximum replicas per host is
	* observed, but also to ensure that the containers
	* don't fail to start because the necessary port
	* mappings (which won't include an IP for uniqueness)
	* are already taken
	*/
	if (probe_data->bundle->fns->max_per_node(probe_data->bundle) == 1) {
	pe__foreach_bundle_replica(probe_data->bundle,
	order_replica_start_after, replica);
	}
	}
	if ((replica->container != NULL) && (replica->remote != NULL)
	&& replica->remote->cmds->create_probe(replica->remote,
	probe_data->node)) {
	/* Do not probe the remote resource until we know where the container is
	* running. This is required for REMOTE_CONTAINER_HACK to correctly
	* probe remote resources.
	*/
	char *probe_uuid = pcmk__op_key(replica->remote->id,
	PCMK_ACTION_MONITOR, 0);
	pcmk_action_t *probe = find_first_action(replica->remote->actions,
	probe_uuid, NULL,
	probe_data->node);

	free(probe_uuid);
	if (probe != NULL) {
	probe_data->any_created = true;
	pcmk__rsc_trace(probe_data->bundle, "Ordering %s probe on %s",
	replica->remote->id,
	pcmk__node_name(probe_data->node));
	pcmk__new_ordering(replica->container,
	pcmk__op_key(replica->container->id,
	PCMK_ACTION_START, 0),
	NULL, replica->remote, NULL, probe,
	pcmk__ar_nested_remote_probe,
	probe_data->bundle->cluster);
	}
	}
	return true;
	}

	/*!
	* \internal
	*
	* \brief Schedule any probes needed for a bundle resource on a node
	*
	* \param[in,out] rsc Bundle resource to create probes for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__bundle_create_probe(pcmk_resource_t rsc, pcmk_node_t node)
	{
	struct probe_data probe_data = { rsc, node, false };

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle));
	pe__foreach_bundle_replica(rsc, create_replica_probes, &probe_data);
	return probe_data.any_created;
	}

	/*!
	* \internal
	* \brief Output actions for one bundle replica
	*
	* \param[in,out] replica Replica to output actions for
	* \param[in] user_data Unused
	*
	* \return true (to indicate that any further replicas should be processed)
	*/
	static bool
	output_replica_actions(pcmk__bundle_replica_t replica, void user_data)
	{
	if (replica->ip != NULL) {
	replica->ip->cmds->output_actions(replica->ip);
	}
	if (replica->container != NULL) {
	replica->container->cmds->output_actions(replica->container);
	}
	if (replica->remote != NULL) {
	replica->remote->cmds->output_actions(replica->remote);
	}
	if (replica->child != NULL) {
	replica->child->cmds->output_actions(replica->child);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Output a summary of scheduled actions for a bundle resource
	*
	* \param[in,out] rsc Bundle resource to output actions for
	*/
	void
	pcmk__output_bundle_actions(pcmk_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle));
	pe__foreach_bundle_replica(rsc, output_replica_actions, NULL);
	}

	// Bundle implementation of pcmk_assignment_methods_t:add_utilization()
	void
	pcmk__bundle_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization)
	{
	pcmk_resource_t *container = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return;
	}

	/* All bundle replicas are identical, so using the utilization of the first
	* is sufficient for any. Only the implicit container resource can have
	* utilization values.
	*/
	container = pe__first_container(rsc);
	if (container != NULL) {
	container->cmds->add_utilization(container, orig_rsc, all_rscs,
	utilization);
	}
	}

	// Bundle implementation of pcmk_assignment_methods_t:shutdown_lock()
	void
	pcmk__bundle_shutdown_lock(pcmk_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_bundle));
	// Bundles currently don't support shutdown locks
	}
	diff --git a/lib/pacemaker/pcmk_sched_clone.c b/lib/pacemaker/pcmk_sched_clone.c
	index 19925ca29d..6039626006 100644
	--- a/lib/pacemaker/pcmk_sched_clone.c
	+++ b/lib/pacemaker/pcmk_sched_clone.c
	@@ -1,709 +1,709 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Assign a clone resource's instances to nodes
	*
	* \param[in,out] rsc Clone resource to assign
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and a primitive descendant of \p rsc
	* can't be assigned to a node, set the
	* descendant's next role to stopped and update
	* existing actions
	*
	* \return NULL (clones are not assigned to a single node)
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pcmk_node_t *
	pcmk__clone_assign(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *colocations = NULL;

	CRM_ASSERT(pe_rsc_is_clone(rsc));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return NULL; // Assignment has already been done
	}

	// Detect assignment loops
	if (pcmk_is_set(rsc->flags, pcmk_rsc_assigning)) {
	pcmk__rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
	return NULL;
	}
	pcmk__set_rsc_flags(rsc, pcmk_rsc_assigning);

	// If this clone is promotable, consider nodes' promotion scores
	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	pcmk__add_promotion_scores(rsc);
	}

	// If this clone is colocated with any other resources, assign those first
	colocations = pcmk__this_with_colocations(rsc);
	for (GList *iter = colocations; iter != NULL; iter = iter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) iter->data;

	pcmk__rsc_trace(rsc, "%s: Assigning colocation %s primary %s first",
	rsc->id, constraint->id, constraint->primary->id);
	constraint->primary->cmds->assign(constraint->primary, prefer,
	stop_if_fail);
	}
	g_list_free(colocations);

	// If any resources are colocated with this one, consider their preferences
	colocations = pcmk__with_this_colocations(rsc);
	g_list_foreach(colocations, pcmk__add_dependent_scores, rsc);
	g_list_free(colocations);

	pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->allowed_nodes, rsc->cluster);

	rsc->children = g_list_sort(rsc->children, pcmk__cmp_instance);
	pcmk__assign_instances(rsc, rsc->children, pe__clone_max(rsc),
	pe__clone_node_max(rsc));

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	pcmk__set_instance_roles(rsc);
	}

	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unassigned\|pcmk_rsc_assigning);
	pcmk__rsc_trace(rsc, "Assigned clone %s", rsc->id);
	return NULL;
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given clone resource
	*
	* \param[in,out] rsc Clone resource to create actions for
	*/
	void
	pcmk__clone_create_actions(pcmk_resource_t *rsc)
	{
	CRM_ASSERT(pe_rsc_is_clone(rsc));

	pcmk__rsc_trace(rsc, "Creating actions for clone %s", rsc->id);
	pcmk__create_instance_actions(rsc, rsc->children);
	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	pcmk__create_promotable_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a clone resource
	*
	* \param[in,out] rsc Clone resource to create implicit constraints for
	*/
	void
	pcmk__clone_internal_constraints(pcmk_resource_t *rsc)
	{
	bool ordered = false;

	CRM_ASSERT(pe_rsc_is_clone(rsc));

	pcmk__rsc_trace(rsc, "Creating internal constraints for clone %s", rsc->id);

	// Restart ordering: Stop -> stopped -> start -> started
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_START,
	rsc, PCMK_ACTION_RUNNING,
	pcmk__ar_unrunnable_first_blocks);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_STOPPED,
	pcmk__ar_unrunnable_first_blocks);

	// Demoted -> stop and started -> promote
	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_unrunnable_first_blocks);
	}

	ordered = pe__clone_is_ordered(rsc);
	if (ordered) {
	/* Ordered clone instances must start and stop by instance number. The
	* instances might have been previously shuffled for assignment or
	* promotion purposes, so re-sort them.
	*/
	rsc->children = g_list_sort(rsc->children, pcmk__cmp_instance_number);
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->cmds->internal_constraints(instance);

	// Start clone -> start instance -> clone started
	pcmk__order_starts(rsc, instance, pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_then_implies_first_graphed);
	pcmk__order_resource_actions(instance, PCMK_ACTION_START,
	rsc, PCMK_ACTION_RUNNING,
	pcmk__ar_first_implies_then_graphed);

	// Stop clone -> stop instance -> clone stopped
	pcmk__order_stops(rsc, instance, pcmk__ar_then_implies_first_graphed);
	pcmk__order_resource_actions(instance, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_STOPPED,
	pcmk__ar_first_implies_then_graphed);

	/* Instances of ordered clones must be started and stopped by instance
	* number. Since only some instances may be starting or stopping, order
	* each instance relative to every later instance.
	*/
	if (ordered) {
	for (GList *later = iter->next;
	later != NULL; later = later->next) {
	pcmk__order_starts(instance, (pcmk_resource_t *) later->data,
	pcmk__ar_ordered);
	pcmk__order_stops((pcmk_resource_t *) later->data, instance,
	pcmk__ar_ordered);
	}
	}
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	pcmk__order_promotable_instances(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether colocated resources can be interleaved
	*
	* \param[in] colocation Colocation constraint with clone as primary
	*
	* \return true if colocated resources can be interleaved, otherwise false
	*/
	static bool
	can_interleave(const pcmk__colocation_t *colocation)
	{
	const pcmk_resource_t *dependent = colocation->dependent;

	// Only colocations between clone or bundle resources use interleaving
	if (dependent->variant <= pcmk_rsc_variant_group) {
	return false;
	}

	// Only the dependent needs to be marked for interleaving
	if (!crm_is_true(g_hash_table_lookup(dependent->meta,
	PCMK_META_INTERLEAVE))) {
	return false;
	}

	/* @TODO Do we actually care about multiple primary instances sharing a
	* dependent instance?
	*/
	if (dependent->fns->max_per_node(dependent)
	!= colocation->primary->fns->max_per_node(colocation->primary)) {
	pcmk__config_err("Cannot interleave %s and %s because they do not "
	"support the same number of instances per node",
	dependent->id, colocation->primary->id);
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__clone_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	const GList *iter = NULL;

	/* This should never be called for the clone itself as a dependent. Instead,
	* we add its colocation constraints to its instances and call the
	* apply_coloc_score() method for the instances as dependents.
	*/
	CRM_ASSERT(!for_dependent);

	CRM_ASSERT((colocation != NULL) && pe_rsc_is_clone(primary)
	&& (dependent != NULL)
	&& (dependent->variant == pcmk_rsc_variant_primitive));

	if (pcmk_is_set(primary->flags, pcmk_rsc_unassigned)) {
	pcmk__rsc_trace(primary,
	"Delaying processing colocation %s "
	"because cloned primary %s is still provisional",
	colocation->id, primary->id);
	return;
	}

	pcmk__rsc_trace(primary, "Processing colocation %s (%s with clone %s @%s)",
	colocation->id, dependent->id, primary->id,
	pcmk_readable_score(colocation->score));

	// Apply role-specific colocations
	if (pcmk_is_set(primary->flags, pcmk_rsc_promotable)
	&& (colocation->primary_role != pcmk_role_unknown)) {

	if (pcmk_is_set(dependent->flags, pcmk_rsc_unassigned)) {
	// We're assigning the dependent to a node
	pcmk__update_dependent_with_promotable(primary, dependent,
	colocation);
	return;
	}

	if (colocation->dependent_role == pcmk_role_promoted) {
	// We're choosing a role for the dependent
	pcmk__update_promotable_dependent_priority(primary, dependent,
	colocation);
	return;
	}
	}

	// Apply interleaved colocations
	if (can_interleave(colocation)) {
	const pcmk_resource_t *primary_instance = NULL;

	primary_instance = pcmk__find_compatible_instance(dependent, primary,
	pcmk_role_unknown,
	false);
	if (primary_instance != NULL) {
	pcmk__rsc_debug(primary, "Interleaving %s with %s",
	dependent->id, primary_instance->id);
	dependent->cmds->apply_coloc_score(dependent, primary_instance,
	colocation, true);

	} else if (colocation->score >= INFINITY) {
	crm_notice("%s cannot run because it cannot interleave with "
	"any instance of %s", dependent->id, primary->id);
	pcmk__assign_resource(dependent, NULL, true, true);

	} else {
	pcmk__rsc_debug(primary,
	"%s will not colocate with %s "
	"because no instance can interleave with it",
	dependent->id, primary->id);
	}

	return;
	}

	// Apply mandatory colocations
	if (colocation->score >= INFINITY) {
	GList *primary_nodes = NULL;

	// Dependent can run only where primary will have unblocked instances
	for (iter = primary->children; iter != NULL; iter = iter->next) {
	const pcmk_resource_t *instance = iter->data;
	pcmk_node_t *chosen = instance->fns->location(instance, NULL, 0);

	if ((chosen != NULL)
	&& !is_set_recursive(instance, pcmk_rsc_blocked, TRUE)) {
	pcmk__rsc_trace(primary, "Allowing %s: %s %d",
	colocation->id, pcmk__node_name(chosen),
	chosen->weight);
	primary_nodes = g_list_prepend(primary_nodes, chosen);
	}
	}
	pcmk__colocation_intersect_nodes(dependent, primary, colocation,
	primary_nodes, false);
	g_list_free(primary_nodes);
	return;
	}

	// Apply optional colocations
	for (iter = primary->children; iter != NULL; iter = iter->next) {
	const pcmk_resource_t *instance = iter->data;

	instance->cmds->apply_coloc_score(dependent, instance, colocation,
	false);
	}
	}

	// Clone implementation of pcmk_assignment_methods_t:with_this_colocations()
	void
	pcmk__with_clone_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	CRM_CHECK((rsc != NULL) && (orig_rsc != NULL) && (list != NULL), return);

	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);

	if (rsc->parent != NULL) {
	rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc, list);
	}
	}

	// Clone implementation of pcmk_assignment_methods_t:this_with_colocations()
	void
	pcmk__clone_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	CRM_CHECK((rsc != NULL) && (orig_rsc != NULL) && (list != NULL), return);

	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);

	if (rsc->parent != NULL) {
	rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc, list);
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given clone resource action
	*
	* \param[in,out] action Action to get flags for
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__clone_action_flags(pcmk_action_t action, const pcmk_node_t node)
	{
	CRM_ASSERT((action != NULL) && pe_rsc_is_clone(action->rsc));

	return pcmk__collective_action_flags(action, action->rsc->children, node);
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a clone resource's allowed node scores
	*
	* \param[in,out] rsc Clone resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*/
	void
	pcmk__clone_apply_location(pcmk_resource_t rsc, pcmk__location_t location)
	{
	CRM_CHECK((location != NULL) && pe_rsc_is_clone(rsc), return);

	pcmk__apply_location(rsc, location);

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->cmds->apply_location(instance, location);
	}
	}

	// GFunc wrapper for calling the action_flags() resource method
	static void
	call_action_flags(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = user_data;

	rsc->cmds->action_flags((pcmk_action_t *) data, NULL);
	}

	/*!
	* \internal
	* \brief Add a clone resource's actions to the transition graph
	*
	* \param[in,out] rsc Resource whose actions should be added
	*/
	void
	pcmk__clone_add_actions_to_graph(pcmk_resource_t *rsc)
	{
	CRM_ASSERT(pe_rsc_is_clone(rsc));

	g_list_foreach(rsc->actions, call_action_flags, rsc);
	pe__create_clone_notifications(rsc);

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) iter->data;

	child_rsc->cmds->add_actions_to_graph(child_rsc);
	}

	pcmk__add_rsc_actions_to_graph(rsc);
	pe__free_clone_notification_data(rsc);
	}

	/*!
	* \internal
	* \brief Check whether a resource or any children have been probed on a node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is in the known_on table of \p rsc or any of its
	* children, otherwise false
	*/
	static bool
	rsc_probed_on(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	if (rsc->children != NULL) {
	for (GList *child_iter = rsc->children; child_iter != NULL;
	child_iter = child_iter->next) {

	pcmk_resource_t child = (pcmk_resource_t ) child_iter->data;

	if (rsc_probed_on(child, node)) {
	return true;
	}
	}
	return false;
	}

	if (rsc->known_on != NULL) {
	GHashTableIter iter;
	pcmk_node_t *known_node = NULL;

	g_hash_table_iter_init(&iter, rsc->known_on);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &known_node)) {
	- if (pe__same_node(node, known_node)) {
	+ if (pcmk__same_node(node, known_node)) {
	return true;
	}
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Find clone instance that has been probed on given node
	*
	* \param[in] clone Clone resource to check
	* \param[in] node Node to check
	*
	* \return Instance of \p clone that has been probed on \p node if any,
	* otherwise NULL
	*/
	static pcmk_resource_t *
	find_probed_instance_on(const pcmk_resource_t clone, const pcmk_node_t node)
	{
	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	if (rsc_probed_on(instance, node)) {
	return instance;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Probe an anonymous clone on a node
	*
	* \param[in,out] clone Anonymous clone to probe
	* \param[in,out] node Node to probe \p clone on
	*/
	static bool
	probe_anonymous_clone(pcmk_resource_t clone, pcmk_node_t node)
	{
	// Check whether we already probed an instance on this node
	pcmk_resource_t *child = find_probed_instance_on(clone, node);

	// Otherwise, check if we plan to start an instance on this node
	for (GList *iter = clone->children; (iter != NULL) && (child == NULL);
	iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;
	const pcmk_node_t *instance_node = NULL;

	instance_node = instance->fns->location(instance, NULL, 0);
	- if (pe__same_node(instance_node, node)) {
	+ if (pcmk__same_node(instance_node, node)) {
	child = instance;
	}
	}

	// Otherwise, use the first clone instance
	if (child == NULL) {
	child = clone->children->data;
	}

	// Anonymous clones only need to probe a single instance
	return child->cmds->create_probe(child, node);
	}

	/*!
	* \internal
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__clone_create_probe(pcmk_resource_t rsc, pcmk_node_t node)
	{
	CRM_ASSERT((node != NULL) && pe_rsc_is_clone(rsc));

	if (rsc->exclusive_discover) {
	/* The clone is configured to be probed only where a location constraint
	* exists with PCMK_XA_RESOURCE_DISCOVERY set to exclusive.
	*
	* This check is not strictly necessary here since the instance's
	* create_probe() method would also check, but doing it here is more
	* efficient (especially for unique clones with a large number of
	* instances), and affects the CRM_meta_notify_available_uname variable
	* passed with notify actions.
	*/
	pcmk_node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes,
	node->details->id);

	if ((allowed == NULL)
	\|\| (allowed->rsc_discover_mode != pcmk_probe_exclusive)) {
	/* This node is not marked for resource discovery. Remove it from
	* allowed_nodes so that notifications contain only nodes that the
	* clone can possibly run on.
	*/
	pcmk__rsc_trace(rsc,
	"Skipping probe for %s on %s because resource has "
	"exclusive discovery but is not allowed on node",
	rsc->id, pcmk__node_name(node));
	g_hash_table_remove(rsc->allowed_nodes, node->details->id);
	return false;
	}
	}

	rsc->children = g_list_sort(rsc->children, pcmk__cmp_instance_number);
	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	return pcmk__probe_resource_list(rsc->children, node);
	} else {
	return probe_anonymous_clone(rsc, node);
	}
	}

	/*!
	* \internal
	* \brief Add meta-attributes relevant to transition graph actions to XML
	*
	* Add clone-specific meta-attributes needed for transition graph actions.
	*
	* \param[in] rsc Clone resource whose meta-attributes should be added
	* \param[in,out] xml Transition graph action attributes XML to add to
	*/
	void
	pcmk__clone_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml)
	{
	char *name = NULL;

	CRM_ASSERT(pe_rsc_is_clone(rsc) && (xml != NULL));

	name = crm_meta_name(PCMK_META_GLOBALLY_UNIQUE);
	crm_xml_add(xml, name, pcmk__flag_text(rsc->flags, pcmk_rsc_unique));
	free(name);

	name = crm_meta_name(PCMK_META_NOTIFY);
	crm_xml_add(xml, name, pcmk__flag_text(rsc->flags, pcmk_rsc_notify));
	free(name);

	name = crm_meta_name(PCMK_META_CLONE_MAX);
	crm_xml_add_int(xml, name, pe__clone_max(rsc));
	free(name);

	name = crm_meta_name(PCMK_META_CLONE_NODE_MAX);
	crm_xml_add_int(xml, name, pe__clone_node_max(rsc));
	free(name);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	int promoted_max = pe__clone_promoted_max(rsc);
	int promoted_node_max = pe__clone_promoted_node_max(rsc);

	name = crm_meta_name(PCMK_META_PROMOTED_MAX);
	crm_xml_add_int(xml, name, promoted_max);
	free(name);

	name = crm_meta_name(PCMK_META_PROMOTED_NODE_MAX);
	crm_xml_add_int(xml, name, promoted_node_max);
	free(name);

	/* @COMPAT Maintain backward compatibility with resource agents that
	* expect the old names (deprecated since 2.0.0).
	*/
	name = crm_meta_name(PCMK__META_PROMOTED_MAX_LEGACY);
	crm_xml_add_int(xml, name, promoted_max);
	free(name);

	name = crm_meta_name(PCMK__META_PROMOTED_NODE_MAX_LEGACY);
	crm_xml_add_int(xml, name, promoted_node_max);
	free(name);
	}
	}

	// Clone implementation of pcmk_assignment_methods_t:add_utilization()
	void
	pcmk__clone_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization)
	{
	bool existing = false;
	pcmk_resource_t *child = NULL;

	CRM_ASSERT(pe_rsc_is_clone(rsc) && (orig_rsc != NULL)
	&& (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return;
	}

	// Look for any child already existing in the list
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	child = (pcmk_resource_t *) iter->data;
	if (g_list_find(all_rscs, child)) {
	existing = true; // Keep checking remaining children
	} else {
	// If this is a clone of a group, look for group's members
	for (GList *member_iter = child->children; member_iter != NULL;
	member_iter = member_iter->next) {

	pcmk_resource_t member = (pcmk_resource_t ) member_iter->data;

	if (g_list_find(all_rscs, member) != NULL) {
	// Add child's utilization, not group member's
	child->cmds->add_utilization(child, orig_rsc, all_rscs,
	utilization);
	existing = true;
	break;
	}
	}
	}
	}

	if (!existing && (rsc->children != NULL)) {
	// If nothing was found, still add first child's utilization
	child = (pcmk_resource_t *) rsc->children->data;

	child->cmds->add_utilization(child, orig_rsc, all_rscs, utilization);
	}
	}

	// Clone implementation of pcmk_assignment_methods_t:shutdown_lock()
	void
	pcmk__clone_shutdown_lock(pcmk_resource_t *rsc)
	{
	CRM_ASSERT(pe_rsc_is_clone(rsc));
	return; // Clones currently don't support shutdown locks
	}
	diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c
	index 0866588845..ba4b14e730 100644
	--- a/lib/pacemaker/pcmk_sched_colocation.c
	+++ b/lib/pacemaker/pcmk_sched_colocation.c
	@@ -1,1908 +1,1908 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include "crm/common/util.h"
	#include "crm/common/xml_internal.h"
	#include "crm/msg_xml.h"
	#include "libpacemaker_private.h"

	// Used to temporarily mark a node as unusable
	#define INFINITY_HACK (INFINITY * -100)

	/*!
	* \internal
	* \brief Compare two colocations according to priority
	*
	* Compare two colocations according to the order in which they should be
	* considered, based on either their dependent resources or their primary
	* resources -- preferring (in order):
	* * Colocation that is not \c NULL
	* * Colocation whose resource has higher priority
	* * Colocation whose resource is of a higher-level variant
	* (bundle > clone > group > primitive)
	* * Colocation whose resource is promotable, if both are clones
	* * Colocation whose resource has lower ID in lexicographic order
	*
	* \param[in] colocation1 First colocation to compare
	* \param[in] colocation2 Second colocation to compare
	* \param[in] dependent If \c true, compare colocations by dependent
	* priority; otherwise compare them by primary priority
	*
	* \return A negative number if \p colocation1 should be considered first,
	* a positive number if \p colocation2 should be considered first,
	* or 0 if order doesn't matter
	*/
	static gint
	cmp_colocation_priority(const pcmk__colocation_t *colocation1,
	const pcmk__colocation_t *colocation2, bool dependent)
	{
	const pcmk_resource_t *rsc1 = NULL;
	const pcmk_resource_t *rsc2 = NULL;

	if (colocation1 == NULL) {
	return 1;
	}
	if (colocation2 == NULL) {
	return -1;
	}

	if (dependent) {
	rsc1 = colocation1->dependent;
	rsc2 = colocation2->dependent;
	CRM_ASSERT(colocation1->primary != NULL);
	} else {
	rsc1 = colocation1->primary;
	rsc2 = colocation2->primary;
	CRM_ASSERT(colocation1->dependent != NULL);
	}
	CRM_ASSERT((rsc1 != NULL) && (rsc2 != NULL));

	if (rsc1->priority > rsc2->priority) {
	return -1;
	}
	if (rsc1->priority < rsc2->priority) {
	return 1;
	}

	// Process clones before primitives and groups
	if (rsc1->variant > rsc2->variant) {
	return -1;
	}
	if (rsc1->variant < rsc2->variant) {
	return 1;
	}

	/* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
	* clones (probably unnecessary, but avoids having to update regression
	* tests)
	*/
	if (rsc1->variant == pcmk_rsc_variant_clone) {
	if (pcmk_is_set(rsc1->flags, pcmk_rsc_promotable)
	&& !pcmk_is_set(rsc2->flags, pcmk_rsc_promotable)) {
	return -1;
	}
	if (!pcmk_is_set(rsc1->flags, pcmk_rsc_promotable)
	&& pcmk_is_set(rsc2->flags, pcmk_rsc_promotable)) {
	return 1;
	}
	}

	return strcmp(rsc1->id, rsc2->id);
	}

	/*!
	* \internal
	* \brief Compare two colocations according to priority based on dependents
	*
	* Compare two colocations according to the order in which they should be
	* considered, based on their dependent resources -- preferring (in order):
	* * Colocation that is not \c NULL
	* * Colocation whose resource has higher priority
	* * Colocation whose resource is of a higher-level variant
	* (bundle > clone > group > primitive)
	* * Colocation whose resource is promotable, if both are clones
	* * Colocation whose resource has lower ID in lexicographic order
	*
	* \param[in] a First colocation to compare
	* \param[in] b Second colocation to compare
	*
	* \return A negative number if \p a should be considered first,
	* a positive number if \p b should be considered first,
	* or 0 if order doesn't matter
	*/
	static gint
	cmp_dependent_priority(gconstpointer a, gconstpointer b)
	{
	return cmp_colocation_priority(a, b, true);
	}

	/*!
	* \internal
	* \brief Compare two colocations according to priority based on primaries
	*
	* Compare two colocations according to the order in which they should be
	* considered, based on their primary resources -- preferring (in order):
	* * Colocation that is not \c NULL
	* * Colocation whose primary has higher priority
	* * Colocation whose primary is of a higher-level variant
	* (bundle > clone > group > primitive)
	* * Colocation whose primary is promotable, if both are clones
	* * Colocation whose primary has lower ID in lexicographic order
	*
	* \param[in] a First colocation to compare
	* \param[in] b Second colocation to compare
	*
	* \return A negative number if \p a should be considered first,
	* a positive number if \p b should be considered first,
	* or 0 if order doesn't matter
	*/
	static gint
	cmp_primary_priority(gconstpointer a, gconstpointer b)
	{
	return cmp_colocation_priority(a, b, false);
	}

	/*!
	* \internal
	* \brief Add a "this with" colocation constraint to a sorted list
	*
	* \param[in,out] list List of constraints to add \p colocation to
	* \param[in] colocation Colocation constraint to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The list will be sorted using cmp_primary_priority().
	*/
	void
	pcmk__add_this_with(GList *list, const pcmk__colocation_t colocation,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (colocation != NULL) && (rsc != NULL));

	pcmk__rsc_trace(rsc,
	"Adding colocation %s (%s with %s using %s @%s) to "
	"'this with' list for %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, colocation->node_attribute,
	pcmk_readable_score(colocation->score), rsc->id);
	list = g_list_insert_sorted(list, (gpointer) colocation,
	cmp_primary_priority);
	}

	/*!
	* \internal
	* \brief Add a list of "this with" colocation constraints to a list
	*
	* \param[in,out] list List of constraints to add \p addition to
	* \param[in] addition List of colocation constraints to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The lists must be pre-sorted by cmp_primary_priority().
	*/
	void
	pcmk__add_this_with_list(GList *list, GList addition,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (rsc != NULL));

	pcmk__if_tracing(
	{}, // Always add each colocation individually if tracing
	{
	if (*list == NULL) {
	// Trivial case for efficiency if not tracing
	*list = g_list_copy(addition);
	return;
	}
	}
	);

	for (const GList *iter = addition; iter != NULL; iter = iter->next) {
	pcmk__add_this_with(list, addition->data, rsc);
	}
	}

	/*!
	* \internal
	* \brief Add a "with this" colocation constraint to a sorted list
	*
	* \param[in,out] list List of constraints to add \p colocation to
	* \param[in] colocation Colocation constraint to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The list will be sorted using cmp_dependent_priority().
	*/
	void
	pcmk__add_with_this(GList *list, const pcmk__colocation_t colocation,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (colocation != NULL) && (rsc != NULL));

	pcmk__rsc_trace(rsc,
	"Adding colocation %s (%s with %s using %s @%s) to "
	"'with this' list for %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, colocation->node_attribute,
	pcmk_readable_score(colocation->score), rsc->id);
	list = g_list_insert_sorted(list, (gpointer) colocation,
	cmp_dependent_priority);
	}

	/*!
	* \internal
	* \brief Add a list of "with this" colocation constraints to a list
	*
	* \param[in,out] list List of constraints to add \p addition to
	* \param[in] addition List of colocation constraints to add to \p list
	* \param[in] rsc Resource whose colocations we're getting (for
	* logging only)
	*
	* \note The lists must be pre-sorted by cmp_dependent_priority().
	*/
	void
	pcmk__add_with_this_list(GList *list, GList addition,
	const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((list != NULL) && (rsc != NULL));

	pcmk__if_tracing(
	{}, // Always add each colocation individually if tracing
	{
	if (*list == NULL) {
	// Trivial case for efficiency if not tracing
	*list = g_list_copy(addition);
	return;
	}
	}
	);

	for (const GList *iter = addition; iter != NULL; iter = iter->next) {
	pcmk__add_with_this(list, addition->data, rsc);
	}
	}

	/*!
	* \internal
	* \brief Add orderings necessary for an anti-colocation constraint
	*
	* \param[in,out] first_rsc One resource in an anti-colocation
	* \param[in] first_role Anti-colocation role of \p first_rsc
	* \param[in] then_rsc Other resource in the anti-colocation
	* \param[in] then_role Anti-colocation role of \p then_rsc
	*/
	static void
	anti_colocation_order(pcmk_resource_t *first_rsc, int first_role,
	pcmk_resource_t *then_rsc, int then_role)
	{
	const char *first_tasks[] = { NULL, NULL };
	const char *then_tasks[] = { NULL, NULL };

	/* Actions to make first_rsc lose first_role */
	if (first_role == pcmk_role_promoted) {
	first_tasks[0] = PCMK_ACTION_DEMOTE;

	} else {
	first_tasks[0] = PCMK_ACTION_STOP;

	if (first_role == pcmk_role_unpromoted) {
	first_tasks[1] = PCMK_ACTION_PROMOTE;
	}
	}

	/* Actions to make then_rsc gain then_role */
	if (then_role == pcmk_role_promoted) {
	then_tasks[0] = PCMK_ACTION_PROMOTE;

	} else {
	then_tasks[0] = PCMK_ACTION_START;

	if (then_role == pcmk_role_unpromoted) {
	then_tasks[1] = PCMK_ACTION_DEMOTE;
	}
	}

	for (int first_lpc = 0;
	(first_lpc <= 1) && (first_tasks[first_lpc] != NULL); first_lpc++) {

	for (int then_lpc = 0;
	(then_lpc <= 1) && (then_tasks[then_lpc] != NULL); then_lpc++) {

	pcmk__order_resource_actions(first_rsc, first_tasks[first_lpc],
	then_rsc, then_tasks[then_lpc],
	pcmk__ar_if_required_on_same_node);
	}
	}
	}

	/*!
	* \internal
	* \brief Add a new colocation constraint to scheduler data
	*
	* \param[in] id XML ID for this constraint
	* \param[in] node_attr Colocate by this attribute (NULL for #uname)
	* \param[in] score Constraint score
	* \param[in,out] dependent Resource to be colocated
	* \param[in,out] primary Resource to colocate \p dependent with
	* \param[in] dependent_role Current role of \p dependent
	* \param[in] primary_role Current role of \p primary
	* \param[in] flags Group of enum pcmk__coloc_flags
	*/
	void
	pcmk__new_colocation(const char id, const char node_attr, int score,
	pcmk_resource_t dependent, pcmk_resource_t primary,
	const char dependent_role, const char primary_role,
	uint32_t flags)
	{
	pcmk__colocation_t *new_con = NULL;

	CRM_CHECK(id != NULL, return);

	if ((dependent == NULL) \|\| (primary == NULL)) {
	pcmk__config_err("Ignoring colocation '%s' because resource "
	"does not exist", id);
	return;
	}

	if (score == 0) {
	pcmk__rsc_trace(dependent,
	"Ignoring colocation '%s' (%s with %s) because score is 0",
	id, dependent->id, primary->id);
	return;
	}

	new_con = calloc(1, sizeof(pcmk__colocation_t));
	CRM_ASSERT(new_con != NULL);

	if (pcmk__str_eq(dependent_role, PCMK__ROLE_STARTED,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	dependent_role = PCMK__ROLE_UNKNOWN;
	}

	if (pcmk__str_eq(primary_role, PCMK__ROLE_STARTED,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	primary_role = PCMK__ROLE_UNKNOWN;
	}

	new_con->id = id;
	new_con->dependent = dependent;
	new_con->primary = primary;
	new_con->score = score;
	new_con->dependent_role = text2role(dependent_role);
	new_con->primary_role = text2role(primary_role);
	new_con->node_attribute = pcmk__s(node_attr, CRM_ATTR_UNAME);
	new_con->flags = flags;

	pcmk__add_this_with(&(dependent->rsc_cons), new_con, dependent);
	pcmk__add_with_this(&(primary->rsc_cons_lhs), new_con, primary);

	dependent->cluster->colocation_constraints = g_list_prepend(
	dependent->cluster->colocation_constraints, new_con);

	if (score <= -INFINITY) {
	anti_colocation_order(dependent, new_con->dependent_role, primary,
	new_con->primary_role);
	anti_colocation_order(primary, new_con->primary_role, dependent,
	new_con->dependent_role);
	}
	}

	/*!
	* \internal
	* \brief Return the boolean influence corresponding to configuration
	*
	* \param[in] coloc_id Colocation XML ID (for error logging)
	* \param[in] rsc Resource involved in constraint (for default)
	* \param[in] influence_s String value of influence option
	*
	* \return \c pcmk__coloc_influence if string evaluates true, or string is
	* \c NULL or invalid and resource's \c PCMK_META_CRITICAL option
	* evaluates true, otherwise \c pcmk__coloc_none
	*/
	static uint32_t
	unpack_influence(const char coloc_id, const pcmk_resource_t rsc,
	const char *influence_s)
	{
	if (influence_s != NULL) {
	int influence_i = 0;

	if (crm_str_to_boolean(influence_s, &influence_i) < 0) {
	pcmk__config_err("Constraint '%s' has invalid value for "
	XML_COLOC_ATTR_INFLUENCE " (using default)",
	coloc_id);
	} else {
	return (influence_i == 0)? pcmk__coloc_none : pcmk__coloc_influence;
	}
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_critical)) {
	return pcmk__coloc_influence;
	}
	return pcmk__coloc_none;
	}

	static void
	unpack_colocation_set(xmlNode set, int score, const char coloc_id,
	const char influence_s, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_rsc = NULL;
	pcmk_resource_t *other = NULL;
	pcmk_resource_t *resource = NULL;
	const char *set_id = ID(set);
	const char *role = crm_element_value(set, PCMK_XA_ROLE);
	bool with_previous = false;
	int local_score = score;
	bool sequential = false;
	uint32_t flags = pcmk__coloc_none;
	const char *xml_rsc_id = NULL;
	const char *score_s = crm_element_value(set, PCMK_XA_SCORE);

	if (score_s) {
	local_score = char2score(score_s);
	}
	if (local_score == 0) {
	crm_trace("Ignoring colocation '%s' for set '%s' because score is 0",
	coloc_id, set_id);
	return;
	}

	/* @COMPAT The deprecated "ordering" attribute specifies whether resources
	* in a positive-score set are colocated with the previous or next resource.
	*/
	if (pcmk__str_eq(crm_element_value(set, "ordering"), "group",
	pcmk__str_null_matches\|pcmk__str_casei)) {
	with_previous = true;
	} else {
	pcmk__warn_once(pcmk__wo_set_ordering,
	"Support for 'ordering' other than 'group' in "
	XML_CONS_TAG_RSC_SET " (such as %s) is deprecated and "
	"will be removed in a future release", set_id);
	}

	if ((pcmk__xe_get_bool_attr(set, "sequential", &sequential) == pcmk_rc_ok)
	&& !sequential) {
	return;
	}

	if (local_score > 0) {
	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xml_rsc_id = ID(xml_rsc);
	resource = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (resource == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring %s and later resources in set %s: "
	"No such resource", xml_rsc_id, set_id);
	return;
	}
	if (other != NULL) {
	flags = pcmk__coloc_explicit
	\| unpack_influence(coloc_id, resource, influence_s);
	if (with_previous) {
	pcmk__rsc_trace(resource, "Colocating %s with %s in set %s",
	resource->id, other->id, set_id);
	pcmk__new_colocation(set_id, NULL, local_score, resource,
	other, role, role, flags);
	} else {
	pcmk__rsc_trace(resource, "Colocating %s with %s in set %s",
	other->id, resource->id, set_id);
	pcmk__new_colocation(set_id, NULL, local_score, other,
	resource, role, role, flags);
	}
	}
	other = resource;
	}

	} else {
	/* Anti-colocating with every prior resource is
	* the only way to ensure the intuitive result
	* (i.e. that no one in the set can run with anyone else in the set)
	*/

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xmlNode *xml_rsc_with = NULL;

	xml_rsc_id = ID(xml_rsc);
	resource = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (resource == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring %s and later resources in set %s: "
	"No such resource", xml_rsc_id, set_id);
	return;
	}
	flags = pcmk__coloc_explicit
	\| unpack_influence(coloc_id, resource, influence_s);
	for (xml_rsc_with = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc_with != NULL;
	xml_rsc_with = crm_next_same_xml(xml_rsc_with)) {

	xml_rsc_id = ID(xml_rsc_with);
	if (pcmk__str_eq(resource->id, xml_rsc_id, pcmk__str_none)) {
	break;
	}
	other = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	CRM_ASSERT(other != NULL); // We already processed it
	pcmk__new_colocation(set_id, NULL, local_score,
	resource, other, role, role, flags);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Colocate two resource sets relative to each other
	*
	* \param[in] id Colocation XML ID
	* \param[in] set1 Dependent set
	* \param[in] set2 Primary set
	* \param[in] score Colocation score
	* \param[in] influence_s Value of colocation's "influence" attribute
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	colocate_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	int score, const char *influence_s,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *xml_rsc = NULL;
	pcmk_resource_t *rsc_1 = NULL;
	pcmk_resource_t *rsc_2 = NULL;

	const char *xml_rsc_id = NULL;
	const char *role_1 = crm_element_value(set1, PCMK_XA_ROLE);
	const char *role_2 = crm_element_value(set2, PCMK_XA_ROLE);

	int rc = pcmk_rc_ok;
	bool sequential = false;
	uint32_t flags = pcmk__coloc_none;

	if (score == 0) {
	crm_trace("Ignoring colocation '%s' between sets %s and %s "
	"because score is 0", id, ID(set1), ID(set2));
	return;
	}

	rc = pcmk__xe_get_bool_attr(set1, "sequential", &sequential);
	if ((rc != pcmk_rc_ok) \|\| sequential) {
	// Get the first one
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	xml_rsc_id = ID(xml_rsc);
	rsc_1 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_1 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s with set %s "
	"because first resource %s not found",
	ID(set1), ID(set2), xml_rsc_id);
	return;
	}
	}
	}

	rc = pcmk__xe_get_bool_attr(set2, "sequential", &sequential);
	if ((rc != pcmk_rc_ok) \|\| sequential) {
	// Get the last one
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xml_rsc_id = ID(xml_rsc);
	}
	rsc_2 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_2 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s with set %s "
	"because last resource %s not found",
	ID(set1), ID(set2), xml_rsc_id);
	return;
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) { // Both sets are sequential
	flags = pcmk__coloc_explicit \| unpack_influence(id, rsc_1, influence_s);
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2,
	flags);

	} else if (rsc_1 != NULL) { // Only set1 is sequential
	flags = pcmk__coloc_explicit \| unpack_influence(id, rsc_1, influence_s);
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xml_rsc_id = ID(xml_rsc);
	rsc_2 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_2 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring set %s colocation with resource %s "
	"in set %s: No such resource",
	ID(set1), xml_rsc_id, ID(set2));
	continue;
	}
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	role_2, flags);
	}

	} else if (rsc_2 != NULL) { // Only set2 is sequential
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xml_rsc_id = ID(xml_rsc);
	rsc_1 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_1 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s resource %s "
	"with set %s: No such resource",
	ID(set1), xml_rsc_id, ID(set2));
	continue;
	}
	flags = pcmk__coloc_explicit
	\| unpack_influence(id, rsc_1, influence_s);
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	role_2, flags);
	}

	} else { // Neither set is sequential
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xmlNode *xml_rsc_2 = NULL;

	xml_rsc_id = ID(xml_rsc);
	rsc_1 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_1 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s resource %s "
	"with set %s: No such resource",
	ID(set1), xml_rsc_id, ID(set2));
	continue;
	}

	flags = pcmk__coloc_explicit
	\| unpack_influence(id, rsc_1, influence_s);
	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL;
	xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	xml_rsc_id = ID(xml_rsc_2);
	rsc_2 = pcmk__find_constraint_resource(scheduler->resources,
	xml_rsc_id);
	if (rsc_2 == NULL) {
	// Should be possible only with validation disabled
	pcmk__config_err("Ignoring colocation of set %s resource "
	"%s with set %s resource %s: No such "
	"resource", ID(set1), ID(xml_rsc),
	ID(set2), xml_rsc_id);
	continue;
	}
	pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2,
	role_1, role_2, flags);
	}
	}
	}
	}

	static void
	unpack_simple_colocation(xmlNode xml_obj, const char id,
	const char influence_s, pcmk_scheduler_t scheduler)
	{
	int score_i = 0;
	uint32_t flags = pcmk__coloc_none;

	const char *score = crm_element_value(xml_obj, PCMK_XA_SCORE);
	const char *dependent_id = crm_element_value(xml_obj,
	XML_COLOC_ATTR_SOURCE);
	const char *primary_id = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	const char *dependent_role = crm_element_value(xml_obj,
	XML_COLOC_ATTR_SOURCE_ROLE);
	const char *primary_role = crm_element_value(xml_obj,
	XML_COLOC_ATTR_TARGET_ROLE);
	const char *attr = crm_element_value(xml_obj, XML_COLOC_ATTR_NODE_ATTR);

	const char *primary_instance = NULL;
	const char *dependent_instance = NULL;
	pcmk_resource_t *primary = NULL;
	pcmk_resource_t *dependent = NULL;

	primary = pcmk__find_constraint_resource(scheduler->resources, primary_id);
	dependent = pcmk__find_constraint_resource(scheduler->resources,
	dependent_id);

	// @COMPAT: Deprecated since 2.1.5
	primary_instance = crm_element_value(xml_obj, PCMK__XA_WITH_RSC_INSTANCE);
	dependent_instance = crm_element_value(xml_obj, PCMK__XA_RSC_INSTANCE);
	if (dependent_instance != NULL) {
	pcmk__warn_once(pcmk__wo_coloc_inst,
	"Support for " PCMK__XA_RSC_INSTANCE " is deprecated "
	"and will be removed in a future release");
	}
	if (primary_instance != NULL) {
	pcmk__warn_once(pcmk__wo_coloc_inst,
	"Support for " PCMK__XA_WITH_RSC_INSTANCE " is "
	"deprecated and will be removed in a future release");
	}

	if (dependent == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, dependent_id);
	return;

	} else if (primary == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, primary_id);
	return;

	} else if ((dependent_instance != NULL) && !pe_rsc_is_clone(dependent)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	id, dependent_id, dependent_instance);
	return;

	} else if ((primary_instance != NULL) && !pe_rsc_is_clone(primary)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	id, primary_id, primary_instance);
	return;
	}

	if (dependent_instance != NULL) {
	dependent = find_clone_instance(dependent, dependent_instance);
	if (dependent == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	id, dependent_id, dependent_instance);
	return;
	}
	}

	if (primary_instance != NULL) {
	primary = find_clone_instance(primary, primary_instance);
	if (primary == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", id, primary_id, primary_instance);
	return;
	}
	}

	if (pcmk__xe_attr_is_true(xml_obj, PCMK_XA_SYMMETRICAL)) {
	pcmk__config_warn("The colocation constraint "
	"'" PCMK_XA_SYMMETRICAL "' attribute has been "
	"removed");
	}

	if (score) {
	score_i = char2score(score);
	}

	flags = pcmk__coloc_explicit \| unpack_influence(id, dependent, influence_s);
	pcmk__new_colocation(id, attr, score_i, dependent, primary,
	dependent_role, primary_role, flags);
	}

	// \return Standard Pacemaker return code
	static int
	unpack_colocation_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	const char *id = NULL;
	const char *dependent_id = NULL;
	const char *primary_id = NULL;
	const char *dependent_role = NULL;
	const char *primary_role = NULL;

	pcmk_resource_t *dependent = NULL;
	pcmk_resource_t *primary = NULL;

	pcmk_tag_t *dependent_tag = NULL;
	pcmk_tag_t *primary_tag = NULL;

	xmlNode *dependent_set = NULL;
	xmlNode *primary_set = NULL;
	bool any_sets = false;

	*expanded_xml = NULL;

	CRM_CHECK(xml_obj != NULL, return EINVAL);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	return pcmk_rc_ok;
	}

	dependent_id = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	primary_id = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	if ((dependent_id == NULL) \|\| (primary_id == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, dependent_id, &dependent,
	&dependent_tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, dependent_id);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, primary_id, &primary,
	&primary_tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, primary_id);
	return pcmk_rc_unpack_error;
	}

	if ((dependent != NULL) && (primary != NULL)) {
	/* Neither side references any template/tag. */
	return pcmk_rc_ok;
	}

	if ((dependent_tag != NULL) && (primary_tag != NULL)) {
	// A colocation constraint between two templates/tags makes no sense
	pcmk__config_err("Ignoring constraint '%s' because two templates or "
	"tags cannot be colocated", id);
	return pcmk_rc_unpack_error;
	}

	dependent_role = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
	primary_role = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);

	*expanded_xml = copy_xml(xml_obj);

	// Convert dependent's template/tag reference into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &dependent_set, XML_COLOC_ATTR_SOURCE,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (dependent_set != NULL) {
	if (dependent_role != NULL) {
	// Move "rsc-role" into converted resource_set as PCMK_XA_ROLE
	crm_xml_add(dependent_set, PCMK_XA_ROLE, dependent_role);
	xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_SOURCE_ROLE);
	}
	any_sets = true;
	}

	// Convert primary's template/tag reference into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &primary_set, XML_COLOC_ATTR_TARGET,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (primary_set != NULL) {
	if (primary_role != NULL) {
	// Move "with-rsc-role" into converted resource_set as PCMK_XA_ROLE
	crm_xml_add(primary_set, PCMK_XA_ROLE, primary_role);
	xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_TARGET_ROLE);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Parse a colocation constraint from XML into scheduler data
	*
	* \param[in,out] xml_obj Colocation constraint XML to unpack
	* \param[in,out] scheduler Scheduler data to add constraint to
	*/
	void
	pcmk__unpack_colocation(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	int score_i = 0;
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
	const char *score = NULL;
	const char *influence_s = NULL;

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_DEPEND
	" without " CRM_ATTR_ID);
	return;
	}

	if (unpack_colocation_tags(xml_obj, &expanded_xml,
	scheduler) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	score = crm_element_value(xml_obj, PCMK_XA_SCORE);
	if (score != NULL) {
	score_i = char2score(score);
	}
	influence_s = crm_element_value(xml_obj, XML_COLOC_ATTR_INFLUENCE);

	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	set = crm_next_same_xml(set)) {

	set = expand_idref(set, scheduler->input);
	if (set == NULL) { // Configuration error, message already logged
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if (pcmk__str_empty(ID(set))) {
	pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_SET
	" without " CRM_ATTR_ID);
	continue;
	}
	unpack_colocation_set(set, score_i, id, influence_s, scheduler);

	if (last != NULL) {
	colocate_rsc_sets(id, last, set, score_i, influence_s, scheduler);
	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	if (last == NULL) {
	unpack_simple_colocation(xml_obj, id, influence_s, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Make actions of a given type unrunnable for a given resource
	*
	* \param[in,out] rsc Resource whose actions should be blocked
	* \param[in] task Name of action to block
	* \param[in] reason Unrunnable start action causing the block
	*/
	static void
	mark_action_blocked(pcmk_resource_t rsc, const char task,
	const pcmk_resource_t *reason)
	{
	GList *iter = NULL;
	char *reason_text = crm_strdup_printf("colocation with %s", reason->id);

	for (iter = rsc->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t *action = iter->data;

	if (pcmk_is_set(action->flags, pcmk_action_runnable)
	&& pcmk__str_eq(action->task, task, pcmk__str_none)) {

	pcmk__clear_action_flags(action, pcmk_action_runnable);
	pe_action_set_reason(action, reason_text, false);
	pcmk__block_colocation_dependents(action);
	pcmk__update_action_for_orderings(action, rsc->cluster);
	}
	}

	// If parent resource can't perform an action, neither can any children
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	mark_action_blocked((pcmk_resource_t *) (iter->data), task, reason);
	}
	free(reason_text);
	}

	/*!
	* \internal
	* \brief If an action is unrunnable, block any relevant dependent actions
	*
	* If a given action is an unrunnable start or promote, block the start or
	* promote actions of resources colocated with it, as appropriate to the
	* colocations' configured roles.
	*
	* \param[in,out] action Action to check
	*/
	void
	pcmk__block_colocation_dependents(pcmk_action_t *action)
	{
	GList *iter = NULL;
	GList *colocations = NULL;
	pcmk_resource_t *rsc = NULL;
	bool is_start = false;

	if (pcmk_is_set(action->flags, pcmk_action_runnable)) {
	return; // Only unrunnable actions block dependents
	}

	is_start = pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_none);
	if (!is_start
	&& !pcmk__str_eq(action->task, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return; // Only unrunnable starts and promotes block dependents
	}

	CRM_ASSERT(action->rsc != NULL); // Start and promote are resource actions

	/* If this resource is part of a collective resource, dependents are blocked
	* only if all instances of the collective are unrunnable, so check the
	* collective resource.
	*/
	rsc = uber_parent(action->rsc);
	if (rsc->parent != NULL) {
	rsc = rsc->parent; // Bundle
	}

	// Colocation fails only if entire primary can't reach desired role
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t *child = iter->data;
	pcmk_action_t *child_action = find_first_action(child->actions, NULL,
	action->task, NULL);

	if ((child_action == NULL)
	\|\| pcmk_is_set(child_action->flags, pcmk_action_runnable)) {
	crm_trace("Not blocking %s colocation dependents because "
	"at least %s has runnable %s",
	rsc->id, child->id, action->task);
	return; // At least one child can reach desired role
	}
	}

	crm_trace("Blocking %s colocation dependents due to unrunnable %s %s",
	rsc->id, action->rsc->id, action->task);

	// Check each colocation where this resource is primary
	colocations = pcmk__with_this_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (colocation->score < INFINITY) {
	continue; // Only mandatory colocations block dependent
	}

	/* If the primary can't start, the dependent can't reach its colocated
	* role, regardless of what the primary or dependent colocation role is.
	*
	* If the primary can't be promoted, the dependent can't reach its
	* colocated role if the primary's colocation role is promoted.
	*/
	if (!is_start && (colocation->primary_role != pcmk_role_promoted)) {
	continue;
	}

	// Block the dependent from reaching its colocated role
	if (colocation->dependent_role == pcmk_role_promoted) {
	mark_action_blocked(colocation->dependent, PCMK_ACTION_PROMOTE,
	action->rsc);
	} else {
	mark_action_blocked(colocation->dependent, PCMK_ACTION_START,
	action->rsc);
	}
	}
	g_list_free(colocations);
	}

	/*!
	* \internal
	* \brief Get the resource to use for role comparisons
	*
	* A bundle replica includes a container and possibly an instance of the bundled
	* resource. The dependent in a "with bundle" colocation is colocated with a
	* particular bundle container. However, if the colocation includes a role, then
	* the role must be checked on the bundled resource instance inside the
	* container. The container itself will never be promoted; the bundled resource
	* may be.
	*
	* If the given resource is a bundle replica container, return the resource
	* inside it, if any. Otherwise, return the resource itself.
	*
	* \param[in] rsc Resource to check
	*
	* \return Resource to use for role comparisons
	*/
	static const pcmk_resource_t *
	get_resource_for_role(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_replica_container)) {
	const pcmk_resource_t *child = pe__get_rsc_in_container(rsc);

	if (child != NULL) {
	return child;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine how a colocation constraint should affect a resource
	*
	* Colocation constraints have different effects at different points in the
	* scheduler sequence. Initially, they affect a resource's location; once that
	* is determined, then for promotable clones they can affect a resource
	* instance's role; after both are determined, the constraints no longer matter.
	* Given a specific colocation constraint, check what has been done so far to
	* determine what should be affected at the current point in the scheduler.
	*
	* \param[in] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint
	* \param[in] preview If true, pretend resources have already been assigned
	*
	* \return How colocation constraint should be applied at this point
	*/
	enum pcmk__coloc_affects
	pcmk__colocation_affects(const pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation, bool preview)
	{
	const pcmk_resource_t *dependent_role_rsc = NULL;
	const pcmk_resource_t *primary_role_rsc = NULL;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (!preview && pcmk_is_set(primary->flags, pcmk_rsc_unassigned)) {
	// Primary resource has not been assigned yet, so we can't do anything
	return pcmk__coloc_affects_nothing;
	}

	dependent_role_rsc = get_resource_for_role(dependent);
	primary_role_rsc = get_resource_for_role(primary);

	if ((colocation->dependent_role >= pcmk_role_unpromoted)
	&& (dependent_role_rsc->parent != NULL)
	&& pcmk_is_set(dependent_role_rsc->parent->flags, pcmk_rsc_promotable)
	&& !pcmk_is_set(dependent_role_rsc->flags, pcmk_rsc_unassigned)) {

	/* This is a colocation by role, and the dependent is a promotable clone
	* that has already been assigned, so the colocation should now affect
	* the role.
	*/
	return pcmk__coloc_affects_role;
	}

	if (!preview && !pcmk_is_set(dependent->flags, pcmk_rsc_unassigned)) {
	/* The dependent resource has already been through assignment, so the
	* constraint no longer has any effect. Log an error if a mandatory
	* colocation constraint has been violated.
	*/

	const pcmk_node_t *primary_node = primary->allocated_to;

	if (dependent->allocated_to == NULL) {
	crm_trace("Skipping colocation '%s': %s will not run anywhere",
	colocation->id, dependent->id);

	} else if (colocation->score >= INFINITY) {
	// Dependent resource must colocate with primary resource

	- if (!pe__same_node(primary_node, dependent->allocated_to)) {
	+ if (!pcmk__same_node(primary_node, dependent->allocated_to)) {
	pcmk__sched_err("%s must be colocated with %s but is not "
	"(%s vs. %s)",
	dependent->id, primary->id,
	pcmk__node_name(dependent->allocated_to),
	pcmk__node_name(primary_node));
	}

	} else if (colocation->score <= -CRM_SCORE_INFINITY) {
	// Dependent resource must anti-colocate with primary resource

	- if (pe__same_node(dependent->allocated_to, primary_node)) {
	+ if (pcmk__same_node(dependent->allocated_to, primary_node)) {
	pcmk__sched_err("%s and %s must be anti-colocated but are "
	"assigned to the same node (%s)",
	dependent->id, primary->id,
	pcmk__node_name(primary_node));
	}
	}
	return pcmk__coloc_affects_nothing;
	}

	if ((colocation->dependent_role != pcmk_role_unknown)
	&& (colocation->dependent_role != dependent_role_rsc->next_role)) {
	crm_trace("Skipping %scolocation '%s': dependent limited to %s role "

	"but %s next role is %s",
	((colocation->score < 0)? "anti-" : ""),
	colocation->id, role2text(colocation->dependent_role),
	dependent_role_rsc->id,
	role2text(dependent_role_rsc->next_role));
	return pcmk__coloc_affects_nothing;
	}

	if ((colocation->primary_role != pcmk_role_unknown)
	&& (colocation->primary_role != primary_role_rsc->next_role)) {
	crm_trace("Skipping %scolocation '%s': primary limited to %s role "
	"but %s next role is %s",
	((colocation->score < 0)? "anti-" : ""),
	colocation->id, role2text(colocation->primary_role),
	primary_role_rsc->id, role2text(primary_role_rsc->next_role));
	return pcmk__coloc_affects_nothing;
	}

	return pcmk__coloc_affects_location;
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent for assignment purposes
	*
	* Update the allowed node scores of the dependent resource in a colocation,
	* for the purposes of assigning it to a node.
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint
	*/
	void
	pcmk__apply_coloc_to_scores(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation)
	{
	const char *attr = colocation->node_attribute;
	const char *value = NULL;
	GHashTable *work = NULL;
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	if (primary->allocated_to != NULL) {
	value = pcmk__colocation_node_attr(primary->allocated_to, attr,
	primary);

	} else if (colocation->score < 0) {
	// Nothing to do (anti-colocation with something that is not running)
	return;
	}

	work = pcmk__copy_node_table(dependent->allowed_nodes);

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (primary->allocated_to == NULL) {
	node->weight = pcmk__add_scores(-colocation->score, node->weight);
	pcmk__rsc_trace(dependent,
	"Applied %s to %s score on %s (now %s after "
	"subtracting %s because primary %s inactive)",
	colocation->id, dependent->id,
	pcmk__node_name(node),
	pcmk_readable_score(node->weight),
	pcmk_readable_score(colocation->score), primary->id);
	continue;
	}

	if (pcmk__str_eq(pcmk__colocation_node_attr(node, attr, dependent),
	value, pcmk__str_casei)) {

	/* Add colocation score only if optional (or minus infinity). A
	* mandatory colocation is a requirement rather than a preference,
	* so we don't need to consider it for relative assignment purposes.
	* The resource will simply be forbidden from running on the node if
	* the primary isn't active there (via the condition above).
	*/
	if (colocation->score < CRM_SCORE_INFINITY) {
	node->weight = pcmk__add_scores(colocation->score,
	node->weight);
	pcmk__rsc_trace(dependent,
	"Applied %s to %s score on %s (now %s after "
	"adding %s)",
	colocation->id, dependent->id,
	pcmk__node_name(node),
	pcmk_readable_score(node->weight),
	pcmk_readable_score(colocation->score));
	}
	continue;
	}

	if (colocation->score >= CRM_SCORE_INFINITY) {
	/* Only mandatory colocations are relevant when the colocation
	* attribute doesn't match, because an attribute not matching is not
	* a negative preference -- the colocation is simply relevant only
	* where it matches.
	*/
	node->weight = -CRM_SCORE_INFINITY;
	pcmk__rsc_trace(dependent,
	"Banned %s from %s because colocation %s attribute %s "
	"does not match",
	dependent->id, pcmk__node_name(node),
	colocation->id, attr);
	}
	}

	if ((colocation->score <= -INFINITY) \|\| (colocation->score >= INFINITY)
	\|\| pcmk__any_node_available(work)) {

	g_hash_table_destroy(dependent->allowed_nodes);
	dependent->allowed_nodes = work;
	work = NULL;

	} else {
	pcmk__rsc_info(dependent,
	"%s: Rolling back scores from %s (no available nodes)",
	dependent->id, primary->id);
	}

	if (work != NULL) {
	g_hash_table_destroy(work);
	}
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent for role purposes
	*
	* Update the priority of the dependent resource in a colocation, for the
	* purposes of selecting its role
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint
	*/
	void
	pcmk__apply_coloc_to_priority(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation)
	{
	const char *dependent_value = NULL;
	const char *primary_value = NULL;
	const char *attr = colocation->node_attribute;
	int score_multiplier = 1;

	const pcmk_resource_t *primary_role_rsc = NULL;

	CRM_ASSERT((dependent != NULL) && (primary != NULL) &&
	(colocation != NULL));

	if ((primary->allocated_to == NULL) \|\| (dependent->allocated_to == NULL)) {
	return;
	}

	dependent_value = pcmk__colocation_node_attr(dependent->allocated_to, attr,
	dependent);
	primary_value = pcmk__colocation_node_attr(primary->allocated_to, attr,
	primary);

	primary_role_rsc = get_resource_for_role(primary);

	if (!pcmk__str_eq(dependent_value, primary_value, pcmk__str_casei)) {
	if ((colocation->score == INFINITY)
	&& (colocation->dependent_role == pcmk_role_promoted)) {
	dependent->priority = -INFINITY;
	}
	return;
	}

	if ((colocation->primary_role != pcmk_role_unknown)
	&& (colocation->primary_role != primary_role_rsc->next_role)) {
	return;
	}

	if (colocation->dependent_role == pcmk_role_unpromoted) {
	score_multiplier = -1;
	}

	dependent->priority = pcmk__add_scores(score_multiplier * colocation->score,
	dependent->priority);
	pcmk__rsc_trace(dependent,
	"Applied %s to %s promotion priority (now %s after %s %s)",
	colocation->id, dependent->id,
	pcmk_readable_score(dependent->priority),
	((score_multiplier == 1)? "adding" : "subtracting"),
	pcmk_readable_score(colocation->score));
	}

	/*!
	* \internal
	* \brief Find score of highest-scored node that matches colocation attribute
	*
	* \param[in] rsc Resource whose allowed nodes should be searched
	* \param[in] attr Colocation attribute name (must not be NULL)
	* \param[in] value Colocation attribute value to require
	*/
	static int
	best_node_score_matching_attr(const pcmk_resource_t rsc, const char attr,
	const char *value)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	int best_score = -INFINITY;
	const char *best_node = NULL;

	// Find best allowed node with matching attribute
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {

	if ((node->weight > best_score)
	&& pcmk__node_available(node, false, false)
	&& pcmk__str_eq(value, pcmk__colocation_node_attr(node, attr, rsc),
	pcmk__str_casei)) {

	best_score = node->weight;
	best_node = node->details->uname;
	}
	}

	if (!pcmk__str_eq(attr, CRM_ATTR_UNAME, pcmk__str_none)) {
	if (best_node == NULL) {
	crm_info("No allowed node for %s matches node attribute %s=%s",
	rsc->id, attr, value);
	} else {
	crm_info("Allowed node %s for %s had best score (%d) "
	"of those matching node attribute %s=%s",
	best_node, rsc->id, best_score, attr, value);
	}
	}
	return best_score;
	}

	/*!
	* \internal
	* \brief Check whether a resource is allowed only on a single node
	*
	* \param[in] rsc Resource to check
	*
	* \return \c true if \p rsc is allowed only on one node, otherwise \c false
	*/
	static bool
	allowed_on_one(const pcmk_resource_t *rsc)
	{
	GHashTableIter iter;
	pcmk_node_t *allowed_node = NULL;
	int allowed_nodes = 0;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &allowed_node)) {
	if ((allowed_node->weight >= 0) && (++allowed_nodes > 1)) {
	pcmk__rsc_trace(rsc, "%s is allowed on multiple nodes", rsc->id);
	return false;
	}
	}
	pcmk__rsc_trace(rsc, "%s is allowed %s", rsc->id,
	((allowed_nodes == 1)? "on a single node" : "nowhere"));
	return (allowed_nodes == 1);
	}

	/*!
	* \internal
	* \brief Add resource's colocation matches to current node assignment scores
	*
	* For each node in a given table, if any of a given resource's allowed nodes
	* have a matching value for the colocation attribute, add the highest of those
	* nodes' scores to the node's score.
	*
	* \param[in,out] nodes Table of nodes with assignment scores so far
	* \param[in] source_rsc Resource whose node scores to add
	* \param[in] target_rsc Resource on whose behalf to update \p nodes
	* \param[in] colocation Original colocation constraint (used to get
	* configured primary resource's stickiness, and
	* to get colocation node attribute; pass NULL to
	* ignore stickiness and use default attribute)
	* \param[in] factor Factor by which to multiply scores being added
	* \param[in] only_positive Whether to add only positive scores
	*/
	static void
	add_node_scores_matching_attr(GHashTable *nodes,
	const pcmk_resource_t *source_rsc,
	const pcmk_resource_t *target_rsc,
	const pcmk__colocation_t *colocation,
	float factor, bool only_positive)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	const char *attr = colocation->node_attribute;

	// Iterate through each node
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	float delta_f = 0;
	int delta = 0;
	int score = 0;
	int new_score = 0;
	const char *value = pcmk__colocation_node_attr(node, attr, target_rsc);

	score = best_node_score_matching_attr(source_rsc, attr, value);

	if ((factor < 0) && (score < 0)) {
	/* If the dependent is anti-colocated, we generally don't want the
	* primary to prefer nodes that the dependent avoids. That could
	* lead to unnecessary shuffling of the primary when the dependent
	* hits its migration threshold somewhere, for example.
	*
	* However, there are cases when it is desirable. If the dependent
	* can't run anywhere but where the primary is, it would be
	* worthwhile to move the primary for the sake of keeping the
	* dependent active.
	*
	* We can't know that exactly at this point since we don't know
	* where the primary will be assigned, but we can limit considering
	* the preference to when the dependent is allowed only on one node.
	* This is less than ideal for multiple reasons:
	*
	* - the dependent could be allowed on more than one node but have
	* anti-colocation primaries on each;
	* - the dependent could be a clone or bundle with multiple
	* instances, and the dependent as a whole is allowed on multiple
	* nodes but some instance still can't run
	* - the dependent has considered node-specific criteria such as
	* location constraints and stickiness by this point, but might
	* have other factors that end up disallowing a node
	*
	* but the alternative is making the primary move when it doesn't
	* need to.
	*
	* We also consider the primary's stickiness and influence, so the
	* user has some say in the matter. (This is the configured primary,
	* not a particular instance of the primary, but that doesn't matter
	* unless stickiness uses a rule to vary by node, and that seems
	* acceptable to ignore.)
	*/
	if ((colocation->primary->stickiness >= -score)
	\|\| !pcmk__colocation_has_influence(colocation, NULL)
	\|\| !allowed_on_one(colocation->dependent)) {
	crm_trace("%s: Filtering %d + %f * %d "
	"(double negative disallowed)",
	pcmk__node_name(node), node->weight, factor, score);
	continue;
	}
	}

	if (node->weight == INFINITY_HACK) {
	crm_trace("%s: Filtering %d + %f * %d (node was marked unusable)",
	pcmk__node_name(node), node->weight, factor, score);
	continue;
	}

	delta_f = factor * score;

	// Round the number; see http://c-faq.com/fp/round.html
	delta = (int) ((delta_f < 0)? (delta_f - 0.5) : (delta_f + 0.5));

	/* Small factors can obliterate the small scores that are often actually
	* used in configurations. If the score and factor are nonzero, ensure
	* that the result is nonzero as well.
	*/
	if ((delta == 0) && (score != 0)) {
	if (factor > 0.0) {
	delta = 1;
	} else if (factor < 0.0) {
	delta = -1;
	}
	}

	new_score = pcmk__add_scores(delta, node->weight);

	if (only_positive && (new_score < 0) && (node->weight > 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d "
	"(negative disallowed, marking node unusable)",
	pcmk__node_name(node), node->weight, factor, score,
	new_score);
	node->weight = INFINITY_HACK;
	continue;
	}

	if (only_positive && (new_score < 0) && (node->weight == 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d (negative disallowed)",
	pcmk__node_name(node), node->weight, factor, score,
	new_score);
	continue;
	}

	crm_trace("%s: %d + %f * %d = %d", pcmk__node_name(node),
	node->weight, factor, score, new_score);
	node->weight = new_score;
	}
	}

	/*!
	* \internal
	* \brief Update nodes with scores of colocated resources' nodes
	*
	* Given a table of nodes and a resource, update the nodes' scores with the
	* scores of the best nodes matching the attribute used for each of the
	* resource's relevant colocations.
	*
	* \param[in,out] source_rsc Resource whose node scores to add
	* \param[in] target_rsc Resource on whose behalf to update \p *nodes
	* \param[in] log_id Resource ID for logs (if \c NULL, use
	* \p source_rsc ID)
	* \param[in,out] nodes Nodes to update (set initial contents to \c NULL
	* to copy allowed nodes from \p source_rsc)
	* \param[in] colocation Original colocation constraint (used to get
	* configured primary resource's stickiness, and
	* to get colocation node attribute; if \c NULL,
	* <tt>source_rsc</tt>'s own matching node scores
	* will not be added, and \p *nodes must be \c NULL
	* as well)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pcmk__coloc_select values
	*
	* \note \c NULL \p target_rsc, \c NULL \p *nodes, \c NULL \p colocation, and
	* the \c pcmk__coloc_select_this_with flag are used together (and only by
	* \c cmp_resources()).
	* \note The caller remains responsible for freeing \p *nodes.
	* \note This is the shared implementation of
	* \c pcmk_assignment_methods_t:add_colocated_node_scores().
	*/
	void
	pcmk__add_colocated_node_scores(pcmk_resource_t *source_rsc,
	const pcmk_resource_t *target_rsc,
	const char *log_id,
	GHashTable **nodes,
	const pcmk__colocation_t *colocation,
	float factor, uint32_t flags)
	{
	GHashTable *work = NULL;

	CRM_ASSERT((source_rsc != NULL) && (nodes != NULL)
	&& ((colocation != NULL)
	\|\| ((target_rsc == NULL) && (*nodes == NULL))));

	if (log_id == NULL) {
	log_id = source_rsc->id;
	}

	// Avoid infinite recursion
	if (pcmk_is_set(source_rsc->flags, pcmk_rsc_updating_nodes)) {
	pcmk__rsc_info(source_rsc, "%s: Breaking dependency loop at %s",
	log_id, source_rsc->id);
	return;
	}
	pcmk__set_rsc_flags(source_rsc, pcmk_rsc_updating_nodes);

	if (*nodes == NULL) {
	work = pcmk__copy_node_table(source_rsc->allowed_nodes);
	target_rsc = source_rsc;
	} else {
	const bool pos = pcmk_is_set(flags, pcmk__coloc_select_nonnegative);

	pcmk__rsc_trace(source_rsc, "%s: Merging %s scores from %s (at %.6f)",
	log_id, (pos? "positive" : "all"), source_rsc->id, factor);
	work = pcmk__copy_node_table(*nodes);
	add_node_scores_matching_attr(work, source_rsc, target_rsc, colocation,
	factor, pos);
	}

	if (work == NULL) {
	pcmk__clear_rsc_flags(source_rsc, pcmk_rsc_updating_nodes);
	return;
	}

	if (pcmk__any_node_available(work)) {
	GList *colocations = NULL;

	if (pcmk_is_set(flags, pcmk__coloc_select_this_with)) {
	colocations = pcmk__this_with_colocations(source_rsc);
	pcmk__rsc_trace(source_rsc,
	"Checking additional %d optional '%s with' "
	"constraints",
	g_list_length(colocations), source_rsc->id);
	} else {
	colocations = pcmk__with_this_colocations(source_rsc);
	pcmk__rsc_trace(source_rsc,
	"Checking additional %d optional 'with %s' "
	"constraints",
	g_list_length(colocations), source_rsc->id);
	}
	flags \|= pcmk__coloc_select_active;

	for (GList *iter = colocations; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *constraint = iter->data;

	pcmk_resource_t *other = NULL;
	float other_factor = factor * constraint->score / (float) INFINITY;

	if (pcmk_is_set(flags, pcmk__coloc_select_this_with)) {
	other = constraint->primary;
	} else if (!pcmk__colocation_has_influence(constraint, NULL)) {
	continue;
	} else {
	other = constraint->dependent;
	}

	pcmk__rsc_trace(source_rsc,
	"Optionally merging score of '%s' constraint "
	"(%s with %s)",
	constraint->id, constraint->dependent->id,
	constraint->primary->id);
	other->cmds->add_colocated_node_scores(other, target_rsc, log_id,
	&work, constraint,
	other_factor, flags);
	pe__show_node_scores(true, NULL, log_id, work, source_rsc->cluster);
	}
	g_list_free(colocations);

	} else if (pcmk_is_set(flags, pcmk__coloc_select_active)) {
	pcmk__rsc_info(source_rsc, "%s: Rolling back optional scores from %s",
	log_id, source_rsc->id);
	g_hash_table_destroy(work);
	pcmk__clear_rsc_flags(source_rsc, pcmk_rsc_updating_nodes);
	return;
	}


	if (pcmk_is_set(flags, pcmk__coloc_select_nonnegative)) {
	pcmk_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->weight == INFINITY_HACK) {
	node->weight = 1;
	}
	}
	}

	if (*nodes != NULL) {
	g_hash_table_destroy(*nodes);
	}
	*nodes = work;

	pcmk__clear_rsc_flags(source_rsc, pcmk_rsc_updating_nodes);
	}

	/*!
	* \internal
	* \brief Apply a "with this" colocation to a resource's allowed node scores
	*
	* \param[in,out] data Colocation to apply
	* \param[in,out] user_data Resource being assigned
	*/
	void
	pcmk__add_dependent_scores(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t *colocation = data;
	pcmk_resource_t *target_rsc = user_data;

	pcmk_resource_t *source_rsc = colocation->dependent;
	const float factor = colocation->score / (float) INFINITY;
	uint32_t flags = pcmk__coloc_select_active;

	if (!pcmk__colocation_has_influence(colocation, NULL)) {
	return;
	}
	if (target_rsc->variant == pcmk_rsc_variant_clone) {
	flags \|= pcmk__coloc_select_nonnegative;
	}
	pcmk__rsc_trace(target_rsc,
	"%s: Incorporating attenuated %s assignment scores due "
	"to colocation %s",
	target_rsc->id, source_rsc->id, colocation->id);
	source_rsc->cmds->add_colocated_node_scores(source_rsc, target_rsc,
	source_rsc->id,
	&target_rsc->allowed_nodes,
	colocation, factor, flags);
	}

	/*!
	* \internal
	* \brief Exclude nodes from a dependent's node table if not in a given list
	*
	* Given a dependent resource in a colocation and a list of nodes where the
	* primary resource will run, set a node's score to \c -INFINITY in the
	* dependent's node table if not found in the primary nodes list.
	*
	* \param[in,out] dependent Dependent resource
	* \param[in] primary Primary resource (for logging only)
	* \param[in] colocation Colocation constraint (for logging only)
	* \param[in] primary_nodes List of nodes where the primary will have
	* unblocked instances in a suitable role
	* \param[in] merge_scores If \c true and a node is found in both \p table
	* and \p list, add the node's score in \p list to
	* the node's score in \p table
	*/
	void
	pcmk__colocation_intersect_nodes(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	const GList *primary_nodes, bool merge_scores)
	{
	GHashTableIter iter;
	pcmk_node_t *dependent_node = NULL;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	g_hash_table_iter_init(&iter, dependent->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &dependent_node)) {
	const pcmk_node_t *primary_node = NULL;

	primary_node = pe_find_node_id(primary_nodes,
	dependent_node->details->id);
	if (primary_node == NULL) {
	dependent_node->weight = -INFINITY;
	pcmk__rsc_trace(dependent,
	"Banning %s from %s (no primary instance) for %s",
	dependent->id, pcmk__node_name(dependent_node),
	colocation->id);

	} else if (merge_scores) {
	dependent_node->weight = pcmk__add_scores(dependent_node->weight,
	primary_node->weight);
	pcmk__rsc_trace(dependent,
	"Added %s's score %s to %s's score for %s (now %s) "
	"for colocation %s",
	primary->id, pcmk_readable_score(primary_node->weight),
	dependent->id, pcmk__node_name(dependent_node),
	pcmk_readable_score(dependent_node->weight),
	colocation->id);
	}
	}
	}

	/*!
	* \internal
	* \brief Get all colocations affecting a resource as the primary
	*
	* \param[in] rsc Resource to get colocations for
	*
	* \return Newly allocated list of colocations affecting \p rsc as primary
	*
	* \note This is a convenience wrapper for the with_this_colocations() method.
	*/
	GList *
	pcmk__with_this_colocations(const pcmk_resource_t *rsc)
	{
	GList *list = NULL;

	rsc->cmds->with_this_colocations(rsc, rsc, &list);
	return list;
	}

	/*!
	* \internal
	* \brief Get all colocations affecting a resource as the dependent
	*
	* \param[in] rsc Resource to get colocations for
	*
	* \return Newly allocated list of colocations affecting \p rsc as dependent
	*
	* \note This is a convenience wrapper for the this_with_colocations() method.
	*/
	GList *
	pcmk__this_with_colocations(const pcmk_resource_t *rsc)
	{
	GList *list = NULL;

	rsc->cmds->this_with_colocations(rsc, rsc, &list);
	return list;
	}
	diff --git a/lib/pacemaker/pcmk_sched_instances.c b/lib/pacemaker/pcmk_sched_instances.c
	index 0c80c0f8ac..8d8da56c16 100644
	--- a/lib/pacemaker/pcmk_sched_instances.c
	+++ b/lib/pacemaker/pcmk_sched_instances.c
	@@ -1,1687 +1,1687 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	/* This file is intended for code usable with both clone instances and bundle
	* replica containers.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Check whether a node is allowed to run an instance
	*
	* \param[in] instance Clone instance or bundle container to check
	* \param[in] node Node to check
	* \param[in] max_per_node Maximum number of instances allowed to run on a node
	*
	* \return true if \p node is allowed to run \p instance, otherwise false
	*/
	static bool
	can_run_instance(const pcmk_resource_t instance, const pcmk_node_t node,
	int max_per_node)
	{
	pcmk_node_t *allowed_node = NULL;

	if (pcmk_is_set(instance->flags, pcmk_rsc_removed)) {
	pcmk__rsc_trace(instance, "%s cannot run on %s: orphaned",
	instance->id, pcmk__node_name(node));
	return false;
	}

	if (!pcmk__node_available(node, false, false)) {
	pcmk__rsc_trace(instance,
	"%s cannot run on %s: node cannot run resources",
	instance->id, pcmk__node_name(node));
	return false;
	}

	allowed_node = pcmk__top_allowed_node(instance, node);
	if (allowed_node == NULL) {
	crm_warn("%s cannot run on %s: node not allowed",
	instance->id, pcmk__node_name(node));
	return false;
	}

	if (allowed_node->weight < 0) {
	pcmk__rsc_trace(instance,
	"%s cannot run on %s: parent score is %s there",
	instance->id, pcmk__node_name(node),
	pcmk_readable_score(allowed_node->weight));
	return false;
	}

	if (allowed_node->count >= max_per_node) {
	pcmk__rsc_trace(instance,
	"%s cannot run on %s: node already has %d instance%s",
	instance->id, pcmk__node_name(node), max_per_node,
	pcmk__plural_s(max_per_node));
	return false;
	}

	pcmk__rsc_trace(instance, "%s can run on %s (%d already running)",
	instance->id, pcmk__node_name(node), allowed_node->count);
	return true;
	}

	/*!
	* \internal
	* \brief Ban a clone instance or bundle replica from unavailable allowed nodes
	*
	* \param[in,out] instance Clone instance or bundle replica to ban
	* \param[in] max_per_node Maximum instances allowed to run on a node
	*/
	static void
	ban_unavailable_allowed_nodes(pcmk_resource_t *instance, int max_per_node)
	{
	if (instance->allowed_nodes != NULL) {
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, instance->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (!can_run_instance(instance, node, max_per_node)) {
	pcmk__rsc_trace(instance, "Banning %s from unavailable node %s",
	instance->id, pcmk__node_name(node));
	node->weight = -INFINITY;
	for (GList *child_iter = instance->children;
	child_iter != NULL; child_iter = child_iter->next) {
	pcmk_resource_t *child = child_iter->data;
	pcmk_node_t *child_node = NULL;

	child_node = g_hash_table_lookup(child->allowed_nodes,
	node->details->id);
	if (child_node != NULL) {
	pcmk__rsc_trace(instance,
	"Banning %s child %s "
	"from unavailable node %s",
	instance->id, child->id,
	pcmk__node_name(node));
	child_node->weight = -INFINITY;
	}
	}
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Create a hash table with a single node in it
	*
	* \param[in] node Node to copy into new table
	*
	* \return Newly created hash table containing a copy of \p node
	* \note The caller is responsible for freeing the result with
	* g_hash_table_destroy().
	*/
	static GHashTable *
	new_node_table(pcmk_node_t *node)
	{
	GHashTable *table = pcmk__strkey_table(NULL, free);

	node = pe__copy_node(node);
	g_hash_table_insert(table, (gpointer) node->details->id, node);
	return table;
	}

	/*!
	* \internal
	* \brief Apply a resource's parent's colocation scores to a node table
	*
	* \param[in] rsc Resource whose colocations should be applied
	* \param[in,out] nodes Node table to apply colocations to
	*/
	static void
	apply_parent_colocations(const pcmk_resource_t rsc, GHashTable *nodes)
	{
	GList *colocations = pcmk__this_with_colocations(rsc);

	for (const GList *iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *colocation = iter->data;
	pcmk_resource_t *other = colocation->primary;
	float factor = colocation->score / (float) INFINITY;

	other->cmds->add_colocated_node_scores(other, rsc, rsc->id, nodes,
	colocation, factor,
	pcmk__coloc_select_default);
	}
	g_list_free(colocations);
	colocations = pcmk__with_this_colocations(rsc);

	for (const GList *iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *colocation = iter->data;
	pcmk_resource_t *other = colocation->dependent;
	float factor = colocation->score / (float) INFINITY;

	if (!pcmk__colocation_has_influence(colocation, rsc)) {
	continue;
	}
	other->cmds->add_colocated_node_scores(other, rsc, rsc->id, nodes,
	colocation, factor,
	pcmk__coloc_select_nonnegative);
	}
	g_list_free(colocations);
	}

	/*!
	* \internal
	* \brief Compare clone or bundle instances based on colocation scores
	*
	* Determine the relative order in which two clone or bundle instances should be
	* assigned to nodes, considering the scores of colocation constraints directly
	* or indirectly involving them.
	*
	* \param[in] instance1 First instance to compare
	* \param[in] instance2 Second instance to compare
	*
	* \return A negative number if \p instance1 should be assigned first,
	* a positive number if \p instance2 should be assigned first,
	* or 0 if assignment order doesn't matter
	*/
	static int
	cmp_instance_by_colocation(const pcmk_resource_t *instance1,
	const pcmk_resource_t *instance2)
	{
	int rc = 0;
	pcmk_node_t *node1 = NULL;
	pcmk_node_t *node2 = NULL;
	pcmk_node_t *current_node1 = pcmk__current_node(instance1);
	pcmk_node_t *current_node2 = pcmk__current_node(instance2);
	GHashTable *colocated_scores1 = NULL;
	GHashTable *colocated_scores2 = NULL;

	CRM_ASSERT((instance1 != NULL) && (instance1->parent != NULL)
	&& (instance2 != NULL) && (instance2->parent != NULL)
	&& (current_node1 != NULL) && (current_node2 != NULL));

	// Create node tables initialized with each node
	colocated_scores1 = new_node_table(current_node1);
	colocated_scores2 = new_node_table(current_node2);

	// Apply parental colocations
	apply_parent_colocations(instance1, &colocated_scores1);
	apply_parent_colocations(instance2, &colocated_scores2);

	// Find original nodes again, with scores updated for colocations
	node1 = g_hash_table_lookup(colocated_scores1, current_node1->details->id);
	node2 = g_hash_table_lookup(colocated_scores2, current_node2->details->id);

	// Compare nodes by updated scores
	if (node1->weight < node2->weight) {
	crm_trace("Assign %s (%d on %s) after %s (%d on %s)",
	instance1->id, node1->weight, pcmk__node_name(node1),
	instance2->id, node2->weight, pcmk__node_name(node2));
	rc = 1;

	} else if (node1->weight > node2->weight) {
	crm_trace("Assign %s (%d on %s) before %s (%d on %s)",
	instance1->id, node1->weight, pcmk__node_name(node1),
	instance2->id, node2->weight, pcmk__node_name(node2));
	rc = -1;
	}

	g_hash_table_destroy(colocated_scores1);
	g_hash_table_destroy(colocated_scores2);
	return rc;
	}

	/*!
	* \internal
	* \brief Check whether a resource or any of its children are failed
	*
	* \param[in] rsc Resource to check
	*
	* \return true if \p rsc or any of its children are failed, otherwise false
	*/
	static bool
	did_fail(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	return true;
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	if (did_fail((const pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node is allowed to run a resource
	*
	* \param[in] rsc Resource to check
	* \param[in,out] node Node to check (will be set NULL if not allowed)
	*
	* \return true if *node is either NULL or allowed for \p rsc, otherwise false
	*/
	static bool
	node_is_allowed(const pcmk_resource_t rsc, pcmk_node_t *node)
	{
	if (*node != NULL) {
	pcmk_node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes,
	(*node)->details->id);

	if ((allowed == NULL) \|\| (allowed->weight < 0)) {
	pcmk__rsc_trace(rsc, "%s: current location (%s) is unavailable",
	rsc->id, pcmk__node_name(*node));
	*node = NULL;
	return false;
	}
	}
	return true;
	}

	/*!
	* \internal
	* \brief Compare two clone or bundle instances' instance numbers
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a's instance number is lower,
	* a positive number if \p b's instance number is lower,
	* or 0 if their instance numbers are the same
	*/
	gint
	pcmk__cmp_instance_number(gconstpointer a, gconstpointer b)
	{
	const pcmk_resource_t instance1 = (const pcmk_resource_t ) a;
	const pcmk_resource_t instance2 = (const pcmk_resource_t ) b;
	char *div1 = NULL;
	char *div2 = NULL;

	CRM_ASSERT((instance1 != NULL) && (instance2 != NULL));

	// Clone numbers are after a colon, bundle numbers after a dash
	div1 = strrchr(instance1->id, ':');
	if (div1 == NULL) {
	div1 = strrchr(instance1->id, '-');
	}
	div2 = strrchr(instance2->id, ':');
	if (div2 == NULL) {
	div2 = strrchr(instance2->id, '-');
	}
	CRM_ASSERT((div1 != NULL) && (div2 != NULL));

	return (gint) (strtol(div1 + 1, NULL, 10) - strtol(div2 + 1, NULL, 10));
	}

	/*!
	* \internal
	* \brief Compare clone or bundle instances according to assignment order
	*
	* Compare two clone or bundle instances according to the order they should be
	* assigned to nodes, preferring (in order):
	*
	* - Active instance that is less multiply active
	* - Instance that is not active on a disallowed node
	* - Instance with higher configured priority
	* - Active instance whose current node can run resources
	* - Active instance whose parent is allowed on current node
	* - Active instance whose current node has fewer other instances
	* - Active instance
	* - Instance that isn't failed
	* - Instance whose colocations result in higher score on current node
	* - Instance with lower ID in lexicographic order
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a should be assigned first,
	* a positive number if \p b should be assigned first,
	* or 0 if assignment order doesn't matter
	*/
	gint
	pcmk__cmp_instance(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	pcmk_node_t *node1 = NULL;
	pcmk_node_t *node2 = NULL;
	unsigned int nnodes1 = 0;
	unsigned int nnodes2 = 0;

	bool can1 = true;
	bool can2 = true;

	const pcmk_resource_t instance1 = (const pcmk_resource_t ) a;
	const pcmk_resource_t instance2 = (const pcmk_resource_t ) b;

	CRM_ASSERT((instance1 != NULL) && (instance2 != NULL));

	node1 = instance1->fns->active_node(instance1, &nnodes1, NULL);
	node2 = instance2->fns->active_node(instance2, &nnodes2, NULL);

	/* If both instances are running and at least one is multiply
	* active, prefer instance that's running on fewer nodes.
	*/
	if ((nnodes1 > 0) && (nnodes2 > 0)) {
	if (nnodes1 < nnodes2) {
	crm_trace("Assign %s (active on %d) before %s (active on %d): "
	"less multiply active",
	instance1->id, nnodes1, instance2->id, nnodes2);
	return -1;

	} else if (nnodes1 > nnodes2) {
	crm_trace("Assign %s (active on %d) after %s (active on %d): "
	"more multiply active",
	instance1->id, nnodes1, instance2->id, nnodes2);
	return 1;
	}
	}

	/* An instance that is either inactive or active on an allowed node is
	* preferred over an instance that is active on a no-longer-allowed node.
	*/
	can1 = node_is_allowed(instance1, &node1);
	can2 = node_is_allowed(instance2, &node2);
	if (can1 && !can2) {
	crm_trace("Assign %s before %s: not active on a disallowed node",
	instance1->id, instance2->id);
	return -1;

	} else if (!can1 && can2) {
	crm_trace("Assign %s after %s: active on a disallowed node",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance with higher configured priority
	if (instance1->priority > instance2->priority) {
	crm_trace("Assign %s before %s: priority (%d > %d)",
	instance1->id, instance2->id,
	instance1->priority, instance2->priority);
	return -1;

	} else if (instance1->priority < instance2->priority) {
	crm_trace("Assign %s after %s: priority (%d < %d)",
	instance1->id, instance2->id,
	instance1->priority, instance2->priority);
	return 1;
	}

	// Prefer active instance
	if ((node1 == NULL) && (node2 == NULL)) {
	crm_trace("No assignment preference for %s vs. %s: inactive",
	instance1->id, instance2->id);
	return 0;

	} else if (node1 == NULL) {
	crm_trace("Assign %s after %s: active", instance1->id, instance2->id);
	return 1;

	} else if (node2 == NULL) {
	crm_trace("Assign %s before %s: active", instance1->id, instance2->id);
	return -1;
	}

	// Prefer instance whose current node can run resources
	can1 = pcmk__node_available(node1, false, false);
	can2 = pcmk__node_available(node2, false, false);
	if (can1 && !can2) {
	crm_trace("Assign %s before %s: current node can run resources",
	instance1->id, instance2->id);
	return -1;

	} else if (!can1 && can2) {
	crm_trace("Assign %s after %s: current node can't run resources",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance whose parent is allowed to run on instance's current node
	node1 = pcmk__top_allowed_node(instance1, node1);
	node2 = pcmk__top_allowed_node(instance2, node2);
	if ((node1 == NULL) && (node2 == NULL)) {
	crm_trace("No assignment preference for %s vs. %s: "
	"parent not allowed on either instance's current node",
	instance1->id, instance2->id);
	return 0;

	} else if (node1 == NULL) {
	crm_trace("Assign %s after %s: parent not allowed on current node",
	instance1->id, instance2->id);
	return 1;

	} else if (node2 == NULL) {
	crm_trace("Assign %s before %s: parent allowed on current node",
	instance1->id, instance2->id);
	return -1;
	}

	// Prefer instance whose current node is running fewer other instances
	if (node1->count < node2->count) {
	crm_trace("Assign %s before %s: fewer active instances on current node",
	instance1->id, instance2->id);
	return -1;

	} else if (node1->count > node2->count) {
	crm_trace("Assign %s after %s: more active instances on current node",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance that isn't failed
	can1 = did_fail(instance1);
	can2 = did_fail(instance2);
	if (!can1 && can2) {
	crm_trace("Assign %s before %s: not failed",
	instance1->id, instance2->id);
	return -1;
	} else if (can1 && !can2) {
	crm_trace("Assign %s after %s: failed",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance with higher cumulative colocation score on current node
	rc = cmp_instance_by_colocation(instance1, instance2);
	if (rc != 0) {
	return rc;
	}

	// Prefer instance with lower instance number
	rc = pcmk__cmp_instance_number(instance1, instance2);
	if (rc < 0) {
	crm_trace("Assign %s before %s: instance number",
	instance1->id, instance2->id);
	} else if (rc > 0) {
	crm_trace("Assign %s after %s: instance number",
	instance1->id, instance2->id);
	} else {
	crm_trace("No assignment preference for %s vs. %s",
	instance1->id, instance2->id);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Increment the parent's instance count after assigning an instance
	*
	* An instance's parent tracks how many instances have been assigned to each
	* node via its pcmk_node_t:count member. After assigning an instance to a node,
	* find the corresponding node in the parent's allowed table and increment it.
	*
	* \param[in,out] instance Instance whose parent to update
	* \param[in] assigned_to Node to which the instance was assigned
	*/
	static void
	increment_parent_count(pcmk_resource_t *instance,
	const pcmk_node_t *assigned_to)
	{
	pcmk_node_t *allowed = NULL;

	if (assigned_to == NULL) {
	return;
	}
	allowed = pcmk__top_allowed_node(instance, assigned_to);

	if (allowed == NULL) {
	/* The instance is allowed on the node, but its parent isn't. This
	* shouldn't be possible if the resource is managed, and we won't be
	* able to limit the number of instances assigned to the node.
	*/
	CRM_LOG_ASSERT(!pcmk_is_set(instance->flags, pcmk_rsc_managed));

	} else {
	allowed->count++;
	}
	}

	/*!
	* \internal
	* \brief Assign an instance to a node
	*
	* \param[in,out] instance Clone instance or bundle replica container
	* \param[in] prefer If not NULL, attempt early assignment to this
	* node, if still the best choice; otherwise,
	* perform final assignment
	* \param[in] max_per_node Assign at most this many instances to one node
	*
	* \return Node to which \p instance is assigned
	*/
	static const pcmk_node_t *
	assign_instance(pcmk_resource_t instance, const pcmk_node_t prefer,
	int max_per_node)
	{
	pcmk_node_t *chosen = NULL;

	pcmk__rsc_trace(instance, "Assigning %s (preferring %s)", instance->id,
	((prefer == NULL)? "no node" : prefer->details->uname));

	if (pcmk_is_set(instance->flags, pcmk_rsc_assigning)) {
	pcmk__rsc_debug(instance,
	"Assignment loop detected involving %s colocations",
	instance->id);
	return NULL;
	}
	ban_unavailable_allowed_nodes(instance, max_per_node);

	// Failed early assignments are reversible (stop_if_fail=false)
	chosen = instance->cmds->assign(instance, prefer, (prefer == NULL));
	increment_parent_count(instance, chosen);
	return chosen;
	}

	/*!
	* \internal
	* \brief Try to assign an instance to its current node early
	*
	* \param[in] rsc Clone or bundle being assigned (for logs only)
	* \param[in] instance Clone instance or bundle replica container
	* \param[in] current Instance's current node
	* \param[in] max_per_node Maximum number of instances per node
	* \param[in] available Number of instances still available for assignment
	*
	* \return \c true if \p instance was successfully assigned to its current node,
	* or \c false otherwise
	*/
	static bool
	assign_instance_early(const pcmk_resource_t rsc, pcmk_resource_t instance,
	const pcmk_node_t *current, int max_per_node,
	int available)
	{
	const pcmk_node_t *chosen = NULL;
	int reserved = 0;

	pcmk_resource_t *parent = instance->parent;
	GHashTable *allowed_orig = NULL;
	GHashTable *allowed_orig_parent = parent->allowed_nodes;
	const pcmk_node_t *allowed_node = NULL;

	pcmk__rsc_trace(instance, "Trying to assign %s to its current node %s",
	instance->id, pcmk__node_name(current));

	allowed_node = g_hash_table_lookup(instance->allowed_nodes,
	current->details->id);
	if (!pcmk__node_available(allowed_node, true, false)) {
	pcmk__rsc_info(instance,
	"Not assigning %s to current node %s: unavailable",
	instance->id, pcmk__node_name(current));
	return false;
	}

	/* On each iteration, if instance gets assigned to a node other than its
	* current one, we reserve one instance for the chosen node, unassign
	* instance, restore instance's original node tables, and try again. This
	* way, instances are proportionally assigned to nodes based on preferences,
	* but shuffling of specific instances is minimized. If a node will be
	* assigned instances at all, it preferentially receives instances that are
	* currently active there.
	*
	* parent->allowed_nodes tracks the number of instances assigned to each
	* node. If a node already has max_per_node instances assigned,
	* ban_unavailable_allowed_nodes() marks it as unavailable.
	*
	* In the end, we restore the original parent->allowed_nodes to undo the
	* changes to counts during tentative assignments. If we successfully
	* assigned instance to its current node, we increment that node's counter.
	*/

	// Back up the allowed node tables of instance and its children recursively
	pcmk__copy_node_tables(instance, &allowed_orig);

	// Update instances-per-node counts in a scratch table
	parent->allowed_nodes = pcmk__copy_node_table(parent->allowed_nodes);

	while (reserved < available) {
	chosen = assign_instance(instance, current, max_per_node);

	- if (pe__same_node(chosen, current)) {
	+ if (pcmk__same_node(chosen, current)) {
	// Successfully assigned to current node
	break;
	}

	// Assignment updates scores, so restore to original state
	pcmk__rsc_debug(instance, "Rolling back node scores for %s",
	instance->id);
	pcmk__restore_node_tables(instance, allowed_orig);

	if (chosen == NULL) {
	// Assignment failed, so give up
	pcmk__rsc_info(instance,
	"Not assigning %s to current node %s: unavailable",
	instance->id, pcmk__node_name(current));
	pcmk__set_rsc_flags(instance, pcmk_rsc_unassigned);
	break;
	}

	// We prefer more strongly to assign an instance to the chosen node
	pcmk__rsc_debug(instance,
	"Not assigning %s to current node %s: %s is better",
	instance->id, pcmk__node_name(current),
	pcmk__node_name(chosen));

	// Reserve one instance for the chosen node and try again
	if (++reserved >= available) {
	pcmk__rsc_info(instance,
	"Not assigning %s to current node %s: "
	"other assignments are more important",
	instance->id, pcmk__node_name(current));

	} else {
	pcmk__rsc_debug(instance,
	"Reserved an instance of %s for %s. Retrying "
	"assignment of %s to %s",
	rsc->id, pcmk__node_name(chosen), instance->id,
	pcmk__node_name(current));
	}

	// Clear this assignment (frees chosen); leave instance counts in parent
	pcmk__unassign_resource(instance);
	chosen = NULL;
	}

	g_hash_table_destroy(allowed_orig);

	// Restore original instances-per-node counts
	g_hash_table_destroy(parent->allowed_nodes);
	parent->allowed_nodes = allowed_orig_parent;

	if (chosen == NULL) {
	// Couldn't assign instance to current node
	return false;
	}
	pcmk__rsc_trace(instance, "Assigned %s to current node %s",
	instance->id, pcmk__node_name(current));
	increment_parent_count(instance, chosen);
	return true;
	}

	/*!
	* \internal
	* \brief Reset the node counts of a resource's allowed nodes to zero
	*
	* \param[in,out] rsc Resource to reset
	*
	* \return Number of nodes that are available to run resources
	*/
	static unsigned int
	reset_allowed_node_counts(pcmk_resource_t *rsc)
	{
	unsigned int available_nodes = 0;
	pcmk_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	node->count = 0;
	if (pcmk__node_available(node, false, false)) {
	available_nodes++;
	}
	}
	return available_nodes;
	}

	/*!
	* \internal
	* \brief Check whether an instance has a preferred node
	*
	* \param[in] instance Clone instance or bundle replica container
	* \param[in] optimal_per_node Optimal number of instances per node
	*
	* \return Instance's current node if still available, otherwise NULL
	*/
	static const pcmk_node_t *
	preferred_node(const pcmk_resource_t *instance, int optimal_per_node)
	{
	const pcmk_node_t *node = NULL;
	const pcmk_node_t *parent_node = NULL;

	// Check whether instance is active, healthy, and not yet assigned
	if ((instance->running_on == NULL)
	\|\| !pcmk_is_set(instance->flags, pcmk_rsc_unassigned)
	\|\| pcmk_is_set(instance->flags, pcmk_rsc_failed)) {
	return NULL;
	}

	// Check whether instance's current node can run resources
	node = pcmk__current_node(instance);
	if (!pcmk__node_available(node, true, false)) {
	pcmk__rsc_trace(instance, "Not assigning %s to %s early (unavailable)",
	instance->id, pcmk__node_name(node));
	return NULL;
	}

	// Check whether node already has optimal number of instances assigned
	parent_node = pcmk__top_allowed_node(instance, node);
	if ((parent_node != NULL) && (parent_node->count >= optimal_per_node)) {
	pcmk__rsc_trace(instance,
	"Not assigning %s to %s early "
	"(optimal instances already assigned)",
	instance->id, pcmk__node_name(node));
	return NULL;
	}

	return node;
	}

	/*!
	* \internal
	* \brief Assign collective instances to nodes
	*
	* \param[in,out] collective Clone or bundle resource being assigned
	* \param[in,out] instances List of clone instances or bundle containers
	* \param[in] max_total Maximum instances to assign in total
	* \param[in] max_per_node Maximum instances to assign to any one node
	*/
	void
	pcmk__assign_instances(pcmk_resource_t collective, GList instances,
	int max_total, int max_per_node)
	{
	// Reuse node count to track number of assigned instances
	unsigned int available_nodes = reset_allowed_node_counts(collective);

	int optimal_per_node = 0;
	int assigned = 0;
	GList *iter = NULL;
	pcmk_resource_t *instance = NULL;
	const pcmk_node_t *current = NULL;

	if (available_nodes > 0) {
	optimal_per_node = max_total / available_nodes;
	}
	if (optimal_per_node < 1) {
	optimal_per_node = 1;
	}

	pcmk__rsc_debug(collective,
	"Assigning up to %d %s instance%s to up to %u node%s "
	"(at most %d per host, %d optimal)",
	max_total, collective->id, pcmk__plural_s(max_total),
	available_nodes, pcmk__plural_s(available_nodes),
	max_per_node, optimal_per_node);

	// Assign as many instances as possible to their current location
	for (iter = instances; (iter != NULL) && (assigned < max_total);
	iter = iter->next) {
	int available = max_total - assigned;

	instance = iter->data;
	if (!pcmk_is_set(instance->flags, pcmk_rsc_unassigned)) {
	continue; // Already assigned
	}

	current = preferred_node(instance, optimal_per_node);
	if ((current != NULL)
	&& assign_instance_early(collective, instance, current,
	max_per_node, available)) {
	assigned++;
	}
	}

	pcmk__rsc_trace(collective, "Assigned %d of %d instance%s to current node",
	assigned, max_total, pcmk__plural_s(max_total));

	for (iter = instances; iter != NULL; iter = iter->next) {
	instance = (pcmk_resource_t *) iter->data;

	if (!pcmk_is_set(instance->flags, pcmk_rsc_unassigned)) {
	continue; // Already assigned
	}

	if (instance->running_on != NULL) {
	current = pcmk__current_node(instance);
	if (pcmk__top_allowed_node(instance, current) == NULL) {
	const char *unmanaged = "";

	if (!pcmk_is_set(instance->flags, pcmk_rsc_managed)) {
	unmanaged = "Unmanaged resource ";
	}
	crm_notice("%s%s is running on %s which is no longer allowed",
	unmanaged, instance->id, pcmk__node_name(current));
	}
	}

	if (assigned >= max_total) {
	pcmk__rsc_debug(collective,
	"Not assigning %s because maximum %d instances "
	"already assigned",
	instance->id, max_total);
	resource_location(instance, NULL, -INFINITY,
	"collective_limit_reached", collective->cluster);

	} else if (assign_instance(instance, NULL, max_per_node) != NULL) {
	assigned++;
	}
	}

	pcmk__rsc_debug(collective, "Assigned %d of %d possible instance%s of %s",
	assigned, max_total, pcmk__plural_s(max_total),
	collective->id);
	}

	enum instance_state {
	instance_starting = (1 << 0),
	instance_stopping = (1 << 1),

	/* This indicates that some instance is restarting. It's not the same as
	* instance_starting\|instance_stopping, which would indicate that some
	* instance is starting, and some instance (not necessarily the same one) is
	* stopping.
	*/
	instance_restarting = (1 << 2),

	instance_active = (1 << 3),

	instance_all = instance_starting\|instance_stopping
	\|instance_restarting\|instance_active,
	};

	/*!
	* \internal
	* \brief Check whether an instance is active, starting, and/or stopping
	*
	* \param[in] instance Clone instance or bundle replica container
	* \param[in,out] state Whether any instance is starting, stopping, etc.
	*/
	static void
	check_instance_state(const pcmk_resource_t instance, uint32_t state)
	{
	const GList *iter = NULL;
	uint32_t instance_state = 0; // State of just this instance

	// No need to check further if all conditions have already been detected
	if (pcmk_all_flags_set(*state, instance_all)) {
	return;
	}

	// If instance is a collective (a cloned group), check its children instead
	if (instance->variant > pcmk_rsc_variant_primitive) {
	for (iter = instance->children;
	(iter != NULL) && !pcmk_all_flags_set(*state, instance_all);
	iter = iter->next) {
	check_instance_state((const pcmk_resource_t *) iter->data, state);
	}
	return;
	}

	// If we get here, instance is a primitive

	if (instance->running_on != NULL) {
	instance_state \|= instance_active;
	}

	// Check each of the instance's actions for runnable start or stop
	for (iter = instance->actions;
	(iter != NULL) && !pcmk_all_flags_set(instance_state,
	instance_starting
	\|instance_stopping);
	iter = iter->next) {

	const pcmk_action_t action = (const pcmk_action_t ) iter->data;
	const bool optional = pcmk_is_set(action->flags, pcmk_action_optional);

	if (pcmk__str_eq(PCMK_ACTION_START, action->task, pcmk__str_none)) {
	if (!optional
	&& pcmk_is_set(action->flags, pcmk_action_runnable)) {

	pcmk__rsc_trace(instance, "Instance is starting due to %s",
	action->uuid);
	instance_state \|= instance_starting;
	} else {
	pcmk__rsc_trace(instance, "%s doesn't affect %s state (%s)",
	action->uuid, instance->id,
	(optional? "optional" : "unrunnable"));
	}

	} else if (pcmk__str_eq(PCMK_ACTION_STOP, action->task,
	pcmk__str_none)) {
	/* Only stop actions can be pseudo-actions for primitives. That
	* indicates that the node they are on is being fenced, so the stop
	* is implied rather than actually executed.
	*/
	if (!optional
	&& pcmk_any_flags_set(action->flags, pcmk_action_pseudo
	\|pcmk_action_runnable)) {
	pcmk__rsc_trace(instance, "Instance is stopping due to %s",
	action->uuid);
	instance_state \|= instance_stopping;
	} else {
	pcmk__rsc_trace(instance, "%s doesn't affect %s state (%s)",
	action->uuid, instance->id,
	(optional? "optional" : "unrunnable"));
	}
	}
	}

	if (pcmk_all_flags_set(instance_state,
	instance_starting\|instance_stopping)) {
	instance_state \|= instance_restarting;
	}
	*state \|= instance_state;
	}

	/*!
	* \internal
	* \brief Create actions for collective resource instances
	*
	* \param[in,out] collective Clone or bundle resource to create actions for
	* \param[in,out] instances List of clone instances or bundle containers
	*/
	void
	pcmk__create_instance_actions(pcmk_resource_t collective, GList instances)
	{
	uint32_t state = 0;

	pcmk_action_t *stop = NULL;
	pcmk_action_t *stopped = NULL;

	pcmk_action_t *start = NULL;
	pcmk_action_t *started = NULL;

	pcmk__rsc_trace(collective, "Creating collective instance actions for %s",
	collective->id);

	// Create actions for each instance appropriate to its variant
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->cmds->create_actions(instance);
	check_instance_state(instance, &state);
	}

	// Create pseudo-actions for rsc start and started
	start = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_START,
	!pcmk_is_set(state, instance_starting),
	true);
	started = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_RUNNING,
	!pcmk_is_set(state, instance_starting),
	false);
	started->priority = INFINITY;
	if (pcmk_any_flags_set(state, instance_active\|instance_starting)) {
	pcmk__set_action_flags(started, pcmk_action_runnable);
	}

	// Create pseudo-actions for rsc stop and stopped
	stop = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_STOP,
	!pcmk_is_set(state, instance_stopping),
	true);
	stopped = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_STOPPED,
	!pcmk_is_set(state, instance_stopping),
	true);
	stopped->priority = INFINITY;
	if (!pcmk_is_set(state, instance_restarting)) {
	pcmk__set_action_flags(stop, pcmk_action_migratable);
	}

	if (collective->variant == pcmk_rsc_variant_clone) {
	pe__create_clone_notif_pseudo_ops(collective, start, started, stop,
	stopped);
	}
	}

	/*!
	* \internal
	* \brief Get a list of clone instances or bundle replica containers
	*
	* \param[in] rsc Clone or bundle resource
	*
	* \return Clone instances if \p rsc is a clone, or a newly created list of
	* \p rsc's replica containers if \p rsc is a bundle
	* \note The caller must call free_instance_list() on the result when the list
	* is no longer needed.
	*/
	static inline GList *
	get_instance_list(const pcmk_resource_t *rsc)
	{
	if (rsc->variant == pcmk_rsc_variant_bundle) {
	return pe__bundle_containers(rsc);
	} else {
	return rsc->children;
	}
	}

	/*!
	* \internal
	* \brief Free any memory created by get_instance_list()
	*
	* \param[in] rsc Clone or bundle resource passed to get_instance_list()
	* \param[in,out] list Return value of get_instance_list() for \p rsc
	*/
	static inline void
	free_instance_list(const pcmk_resource_t rsc, GList list)
	{
	if (list != rsc->children) {
	g_list_free(list);
	}
	}

	/*!
	* \internal
	* \brief Check whether an instance is compatible with a role and node
	*
	* \param[in] instance Clone instance or bundle replica container
	* \param[in] node Instance must match this node
	* \param[in] role If not pcmk_role_unknown, instance must match this role
	* \param[in] current If true, compare instance's original node and role,
	* otherwise compare assigned next node and role
	*
	* \return true if \p instance is compatible with \p node and \p role,
	* otherwise false
	*/
	bool
	pcmk__instance_matches(const pcmk_resource_t instance, const pcmk_node_t node,
	enum rsc_role_e role, bool current)
	{
	pcmk_node_t *instance_node = NULL;

	CRM_CHECK((instance != NULL) && (node != NULL), return false);

	if ((role != pcmk_role_unknown)
	&& (role != instance->fns->state(instance, current))) {
	pcmk__rsc_trace(instance,
	"%s is not a compatible instance (role is not %s)",
	instance->id, role2text(role));
	return false;
	}

	if (!is_set_recursive(instance, pcmk_rsc_blocked, true)) {
	// We only want instances that haven't failed
	instance_node = instance->fns->location(instance, NULL, current);
	}

	if (instance_node == NULL) {
	pcmk__rsc_trace(instance,
	"%s is not a compatible instance "
	"(not assigned to a node)",
	instance->id);
	return false;
	}

	- if (!pe__same_node(instance_node, node)) {
	+ if (!pcmk__same_node(instance_node, node)) {
	pcmk__rsc_trace(instance,
	"%s is not a compatible instance "
	"(assigned to %s not %s)",
	instance->id, pcmk__node_name(instance_node),
	pcmk__node_name(node));
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Find an instance that matches a given resource by node and role
	*
	* \param[in] match_rsc Resource that instance must match (for logging only)
	* \param[in] rsc Clone or bundle resource to check for matching instance
	* \param[in] node Instance must match this node
	* \param[in] role If not pcmk_role_unknown, instance must match this role
	* \param[in] current If true, compare instance's original node and role,
	* otherwise compare assigned next node and role
	*
	* \return \p rsc instance matching \p node and \p role if any, otherwise NULL
	*/
	static pcmk_resource_t *
	find_compatible_instance_on_node(const pcmk_resource_t *match_rsc,
	const pcmk_resource_t *rsc,
	const pcmk_node_t *node, enum rsc_role_e role,
	bool current)
	{
	GList *instances = NULL;

	instances = get_instance_list(rsc);
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	if (pcmk__instance_matches(instance, node, role, current)) {
	pcmk__rsc_trace(match_rsc,
	"Found %s %s instance %s compatible with %s on %s",
	role == pcmk_role_unknown? "matching" : role2text(role),
	rsc->id, instance->id, match_rsc->id,
	pcmk__node_name(node));
	free_instance_list(rsc, instances); // Only frees list, not contents
	return instance;
	}
	}
	free_instance_list(rsc, instances);

	pcmk__rsc_trace(match_rsc,
	"No %s %s instance found compatible with %s on %s",
	((role == pcmk_role_unknown)? "matching" : role2text(role)),
	rsc->id, match_rsc->id, pcmk__node_name(node));
	return NULL;
	}

	/*!
	* \internal
	* \brief Find a clone instance or bundle container compatible with a resource
	*
	* \param[in] match_rsc Resource that instance must match
	* \param[in] rsc Clone or bundle resource to check for matching instance
	* \param[in] role If not pcmk_role_unknown, instance must match this role
	* \param[in] current If true, compare instance's original node and role,
	* otherwise compare assigned next node and role
	*
	* \return Compatible (by \p role and \p match_rsc location) instance of \p rsc
	* if any, otherwise NULL
	*/
	pcmk_resource_t *
	pcmk__find_compatible_instance(const pcmk_resource_t *match_rsc,
	const pcmk_resource_t *rsc, enum rsc_role_e role,
	bool current)
	{
	pcmk_resource_t *instance = NULL;
	GList *nodes = NULL;
	const pcmk_node_t *node = NULL;

	// If match_rsc has a node, check only that node
	node = match_rsc->fns->location(match_rsc, NULL, current);
	if (node != NULL) {
	return find_compatible_instance_on_node(match_rsc, rsc, node, role,
	current);
	}

	// Otherwise check for an instance matching any of match_rsc's allowed nodes
	nodes = pcmk__sort_nodes(g_hash_table_get_values(match_rsc->allowed_nodes),
	NULL);
	for (GList *iter = nodes; (iter != NULL) && (instance == NULL);
	iter = iter->next) {
	instance = find_compatible_instance_on_node(match_rsc, rsc,
	(pcmk_node_t *) iter->data,
	role, current);
	}

	if (instance == NULL) {
	pcmk__rsc_debug(rsc, "No %s instance found compatible with %s",
	rsc->id, match_rsc->id);
	}
	g_list_free(nodes);
	return instance;
	}

	/*!
	* \internal
	* \brief Unassign an instance if mandatory ordering has no interleave match
	*
	* \param[in] first 'First' action in an ordering
	* \param[in] then 'Then' action in an ordering
	* \param[in,out] then_instance 'Then' instance that has no interleave match
	* \param[in] type Group of enum pcmk__action_relation_flags
	* \param[in] current If true, "then" action is stopped or demoted
	*
	* \return true if \p then_instance was unassigned, otherwise false
	*/
	static bool
	unassign_if_mandatory(const pcmk_action_t first, const pcmk_action_t then,
	pcmk_resource_t *then_instance, uint32_t type,
	bool current)
	{
	// Allow "then" instance to go down even without an interleave match
	if (current) {
	pcmk__rsc_trace(then->rsc,
	"%s has no instance to order before stopping "
	"or demoting %s",
	first->rsc->id, then_instance->id);

	/* If the "first" action must be runnable, but there is no "first"
	* instance, the "then" instance must not be allowed to come up.
	*/
	} else if (pcmk_any_flags_set(type, pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_first_implies_then)) {
	pcmk__rsc_info(then->rsc,
	"Inhibiting %s from being active "
	"because there is no %s instance to interleave",
	then_instance->id, first->rsc->id);
	return pcmk__assign_resource(then_instance, NULL, true, true);
	}
	return false;
	}

	/*!
	* \internal
	* \brief Find first matching action for a clone instance or bundle container
	*
	* \param[in] action Action in an interleaved ordering
	* \param[in] instance Clone instance or bundle container being interleaved
	* \param[in] action_name Action to look for
	* \param[in] node If not NULL, require action to be on this node
	* \param[in] for_first If true, \p instance is the 'first' resource in the
	* ordering, otherwise it is the 'then' resource
	*
	* \return First action for \p instance (or in some cases if \p instance is a
	* bundle container, its containerized resource) that matches
	* \p action_name and \p node if any, otherwise NULL
	*/
	static pcmk_action_t *
	find_instance_action(const pcmk_action_t action, const pcmk_resource_t instance,
	const char action_name, const pcmk_node_t node,
	bool for_first)
	{
	const pcmk_resource_t *rsc = NULL;
	pcmk_action_t *matching_action = NULL;

	/* If instance is a bundle container, sometimes we should interleave the
	* action for the container itself, and sometimes for the containerized
	* resource.
	*
	* For example, given "start bundle A then bundle B", B likely requires the
	* service inside A's container to be active, rather than just the
	* container, so we should interleave the action for A's containerized
	* resource. On the other hand, it's possible B's container itself requires
	* something from A, so we should interleave the action for B's container.
	*
	* Essentially, for 'first', we should use the containerized resource for
	* everything except stop, and for 'then', we should use the container for
	* everything except promote and demote (which can only be performed on the
	* containerized resource).
	*/
	if ((for_first && !pcmk__str_any_of(action->task, PCMK_ACTION_STOP,
	PCMK_ACTION_STOPPED, NULL))

	\|\| (!for_first && pcmk__str_any_of(action->task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_PROMOTED,
	PCMK_ACTION_DEMOTE,
	PCMK_ACTION_DEMOTED, NULL))) {

	rsc = pe__get_rsc_in_container(instance);
	}
	if (rsc == NULL) {
	rsc = instance; // No containerized resource, use instance itself
	} else {
	node = NULL; // Containerized actions are on bundle-created guest
	}

	matching_action = find_first_action(rsc->actions, NULL, action_name, node);
	if (matching_action != NULL) {
	return matching_action;
	}

	if (pcmk_is_set(instance->flags, pcmk_rsc_removed)
	\|\| pcmk__str_any_of(action_name, PCMK_ACTION_STOP, PCMK_ACTION_DEMOTE,
	NULL)) {
	crm_trace("No %s action found for %s%s",
	action_name,
	pcmk_is_set(instance->flags, pcmk_rsc_removed)? "orphan " : "",
	instance->id);
	} else {
	crm_err("No %s action found for %s to interleave (bug?)",
	action_name, instance->id);
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get the original action name of a bundle or clone action
	*
	* Given an action for a bundle or clone, get the original action name,
	* mapping notify to the action being notified, and if the instances are
	* primitives, mapping completion actions to the action that was completed
	* (for example, stopped to stop).
	*
	* \param[in] action Clone or bundle action to check
	*
	* \return Original action name for \p action
	*/
	static const char *
	orig_action_name(const pcmk_action_t *action)
	{
	// Any instance will do
	const pcmk_resource_t *instance = action->rsc->children->data;

	char *action_type = NULL;
	const char *action_name = action->task;
	enum action_tasks orig_task = pcmk_action_unspecified;

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_NOTIFIED, NULL)) {
	// action->uuid is RSC_(confirmed-){pre,post}_notify_ACTION_INTERVAL
	CRM_CHECK(parse_op_key(action->uuid, NULL, &action_type, NULL),
	return task2text(pcmk_action_unspecified));
	action_name = strstr(action_type, "_notify_");
	CRM_CHECK(action_name != NULL,
	return task2text(pcmk_action_unspecified));
	action_name += strlen("_notify_");
	}
	orig_task = get_complex_task(instance, action_name);
	free(action_type);
	return task2text(orig_task);
	}

	/*!
	* \internal
	* \brief Update two interleaved actions according to an ordering between them
	*
	* Given information about an ordering of two interleaved actions, update the
	* actions' flags (and runnable_before members if appropriate) as appropriate
	* for the ordering. Effects may cascade to other orderings involving the
	* actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static uint32_t
	update_interleaved_actions(pcmk_action_t first, pcmk_action_t then,
	const pcmk_node_t *node, uint32_t filter,
	uint32_t type)
	{
	GList *instances = NULL;
	uint32_t changed = pcmk__updated_none;
	const char *orig_first_task = orig_action_name(first);

	// Stops and demotes must be interleaved with instance on current node
	bool current = pcmk__ends_with(first->uuid, "_" PCMK_ACTION_STOPPED "_0")
	\|\| pcmk__ends_with(first->uuid,
	"_" PCMK_ACTION_DEMOTED "_0");

	// Update the specified actions for each "then" instance individually
	instances = get_instance_list(then->rsc);
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pcmk_resource_t *first_instance = NULL;
	pcmk_resource_t *then_instance = iter->data;

	pcmk_action_t *first_action = NULL;
	pcmk_action_t *then_action = NULL;

	// Find a "first" instance to interleave with this "then" instance
	first_instance = pcmk__find_compatible_instance(then_instance,
	first->rsc,
	pcmk_role_unknown,
	current);

	if (first_instance == NULL) { // No instance can be interleaved
	if (unassign_if_mandatory(first, then, then_instance, type,
	current)) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	continue;
	}

	first_action = find_instance_action(first, first_instance,
	orig_first_task, node, true);
	if (first_action == NULL) {
	continue;
	}

	then_action = find_instance_action(then, then_instance, then->task,
	node, false);
	if (then_action == NULL) {
	continue;
	}

	if (order_actions(first_action, then_action, type)) {
	pcmk__set_updated_flags(changed, first,
	pcmk__updated_first\|pcmk__updated_then);
	}

	changed \|= then_instance->cmds->update_ordered_actions(
	first_action, then_action, node,
	first_instance->cmds->action_flags(first_action, node), filter,
	type, then->rsc->cluster);
	}
	free_instance_list(then->rsc, instances);
	return changed;
	}

	/*!
	* \internal
	* \brief Check whether two actions in an ordering can be interleaved
	*
	* \param[in] first 'First' action in the ordering
	* \param[in] then 'Then' action in the ordering
	*
	* \return true if \p first and \p then can be interleaved, otherwise false
	*/
	static bool
	can_interleave_actions(const pcmk_action_t first, const pcmk_action_t then)
	{
	bool interleave = false;
	pcmk_resource_t *rsc = NULL;

	if ((first->rsc == NULL) \|\| (then->rsc == NULL)) {
	crm_trace("Not interleaving %s with %s: not resource actions",
	first->uuid, then->uuid);
	return false;
	}

	if (first->rsc == then->rsc) {
	crm_trace("Not interleaving %s with %s: same resource",
	first->uuid, then->uuid);
	return false;
	}

	if ((first->rsc->variant < pcmk_rsc_variant_clone)
	\|\| (then->rsc->variant < pcmk_rsc_variant_clone)) {
	crm_trace("Not interleaving %s with %s: not clones or bundles",
	first->uuid, then->uuid);
	return false;
	}

	if (pcmk__ends_with(then->uuid, "_stop_0")
	\|\| pcmk__ends_with(then->uuid, "_demote_0")) {
	rsc = first->rsc;
	} else {
	rsc = then->rsc;
	}

	interleave = crm_is_true(g_hash_table_lookup(rsc->meta,
	PCMK_META_INTERLEAVE));
	pcmk__rsc_trace(rsc, "'%s then %s' will %sbe interleaved (based on %s)",
	first->uuid, then->uuid, (interleave? "" : "not "),
	rsc->id);
	return interleave;
	}

	/*!
	* \internal
	* \brief Update non-interleaved instance actions according to an ordering
	*
	* Given information about an ordering of two non-interleaved actions, update
	* the actions' flags (and runnable_before members if appropriate) as
	* appropriate for the ordering. Effects may cascade to other orderings
	* involving the actions as well.
	*
	* \param[in,out] instance Clone instance or bundle container
	* \param[in,out] first "First" action in ordering
	* \param[in] then "Then" action in ordering (for \p instance's parent)
	* \param[in] node If not NULL, limit scope of ordering to this node
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static uint32_t
	update_noninterleaved_actions(pcmk_resource_t instance, pcmk_action_t first,
	const pcmk_action_t then, const pcmk_node_t node,
	uint32_t flags, uint32_t filter, uint32_t type)
	{
	pcmk_action_t *instance_action = NULL;
	uint32_t instance_flags = 0;
	uint32_t changed = pcmk__updated_none;

	// Check whether instance has an equivalent of "then" action
	instance_action = find_first_action(instance->actions, NULL, then->task,
	node);
	if (instance_action == NULL) {
	return changed;
	}

	// Check whether action is runnable
	instance_flags = instance->cmds->action_flags(instance_action, node);
	if (!pcmk_is_set(instance_flags, pcmk_action_runnable)) {
	return changed;
	}

	// If so, update actions for the instance
	changed = instance->cmds->update_ordered_actions(first, instance_action,
	node, flags, filter, type,
	instance->cluster);

	// Propagate any changes to later actions
	if (pcmk_is_set(changed, pcmk__updated_then)) {
	for (GList *after_iter = instance_action->actions_after;
	after_iter != NULL; after_iter = after_iter->next) {
	pcmk__related_action_t *after = after_iter->data;

	pcmk__update_action_for_orderings(after->action, instance->cluster);
	}
	}

	return changed;
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two clone or bundle actions, update
	* the actions' flags (and runnable_before members if appropriate) as
	* appropriate for the ordering. Effects may cascade to other orderings
	* involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__instance_update_ordered_actions(pcmk_action_t first, pcmk_action_t then,
	const pcmk_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pcmk_scheduler_t *scheduler)
	{
	CRM_ASSERT((first != NULL) && (then != NULL) && (scheduler != NULL));

	if (then->rsc == NULL) {
	return pcmk__updated_none;

	} else if (can_interleave_actions(first, then)) {
	return update_interleaved_actions(first, then, node, filter, type);

	} else {
	uint32_t changed = pcmk__updated_none;
	GList *instances = get_instance_list(then->rsc);

	// Update actions for the clone or bundle resource itself
	changed \|= pcmk__update_ordered_actions(first, then, node, flags,
	filter, type, scheduler);

	// Update the 'then' clone instances or bundle containers individually
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pcmk_resource_t *instance = iter->data;

	changed \|= update_noninterleaved_actions(instance, first, then,
	node, flags, filter, type);
	}
	free_instance_list(then->rsc, instances);
	return changed;
	}
	}

	#define pe__clear_action_summary_flags(flags, action, flag) do { \
	flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Action summary", action->rsc->id, \
	flags, flag, #flag); \
	} while (0)

	/*!
	* \internal
	* \brief Return action flags for a given clone or bundle action
	*
	* \param[in,out] action Action for a clone or bundle
	* \param[in] instances Clone instances or bundle containers
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__collective_action_flags(pcmk_action_t action, const GList instances,
	const pcmk_node_t *node)
	{
	bool any_runnable = false;
	const char *action_name = orig_action_name(action);

	// Set original assumptions (optional and runnable may be cleared below)
	uint32_t flags = pcmk_action_optional
	\|pcmk_action_runnable
	\|pcmk_action_pseudo;

	for (const GList *iter = instances; iter != NULL; iter = iter->next) {
	const pcmk_resource_t *instance = iter->data;
	const pcmk_node_t *instance_node = NULL;
	pcmk_action_t *instance_action = NULL;
	uint32_t instance_flags;

	// Node is relevant only to primitive instances
	if (instance->variant == pcmk_rsc_variant_primitive) {
	instance_node = node;
	}

	instance_action = find_first_action(instance->actions, NULL,
	action_name, instance_node);
	if (instance_action == NULL) {
	pcmk__rsc_trace(action->rsc, "%s has no %s action on %s",
	instance->id, action_name, pcmk__node_name(node));
	continue;
	}

	pcmk__rsc_trace(action->rsc, "%s has %s for %s on %s",
	instance->id, instance_action->uuid, action_name,
	pcmk__node_name(node));

	instance_flags = instance->cmds->action_flags(instance_action, node);

	// If any instance action is mandatory, so is the collective action
	if (pcmk_is_set(flags, pcmk_action_optional)
	&& !pcmk_is_set(instance_flags, pcmk_action_optional)) {
	pcmk__rsc_trace(instance, "%s is mandatory because %s is",
	action->uuid, instance_action->uuid);
	pe__clear_action_summary_flags(flags, action,
	pcmk_action_optional);
	pcmk__clear_action_flags(action, pcmk_action_optional);
	}

	// If any instance action is runnable, so is the collective action
	if (pcmk_is_set(instance_flags, pcmk_action_runnable)) {
	any_runnable = true;
	}
	}

	if (!any_runnable) {
	pcmk__rsc_trace(action->rsc,
	"%s is not runnable because no instance can run %s",
	action->uuid, action_name);
	pe__clear_action_summary_flags(flags, action, pcmk_action_runnable);
	if (node == NULL) {
	pcmk__clear_action_flags(action, pcmk_action_runnable);
	}
	}

	return flags;
	}
	diff --git a/lib/pacemaker/pcmk_sched_nodes.c b/lib/pacemaker/pcmk_sched_nodes.c
	index d1c11c8f32..15576021c9 100644
	--- a/lib/pacemaker/pcmk_sched_nodes.c
	+++ b/lib/pacemaker/pcmk_sched_nodes.c
	@@ -1,435 +1,435 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Check whether a node is available to run resources
	*
	* \param[in] node Node to check
	* \param[in] consider_score If true, consider a negative score unavailable
	* \param[in] consider_guest If true, consider a guest node unavailable whose
	* resource will not be active
	*
	* \return true if node is online and not shutting down, unclean, or in standby
	* or maintenance mode, otherwise false
	*/
	bool
	pcmk__node_available(const pcmk_node_t *node, bool consider_score,
	bool consider_guest)
	{
	if ((node == NULL) \|\| (node->details == NULL) \|\| !node->details->online
	\|\| node->details->shutdown \|\| node->details->unclean
	\|\| node->details->standby \|\| node->details->maintenance) {
	return false;
	}

	if (consider_score && (node->weight < 0)) {
	return false;
	}

	// @TODO Go through all callers to see which should set consider_guest
	if (consider_guest && pe__is_guest_node(node)) {
	pcmk_resource_t *guest = node->details->remote_rsc->container;

	if (guest->fns->location(guest, NULL, FALSE) == NULL) {
	return false;
	}
	}

	return true;
	}

	/*!
	* \internal
	* \brief Copy a hash table of node objects
	*
	* \param[in] nodes Hash table to copy
	*
	* \return New copy of nodes (or NULL if nodes is NULL)
	*/
	GHashTable *
	pcmk__copy_node_table(GHashTable *nodes)
	{
	GHashTable *new_table = NULL;
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	if (nodes == NULL) {
	return NULL;
	}
	new_table = pcmk__strkey_table(NULL, free);
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	pcmk_node_t *new_node = pe__copy_node(node);

	g_hash_table_insert(new_table, (gpointer) new_node->details->id,
	new_node);
	}
	return new_table;
	}

	/*!
	* \internal
	* \brief Free a table of node tables
	*
	* \param[in,out] data Table to free
	*
	* \note This is a \c GDestroyNotify wrapper for \c g_hash_table_destroy().
	*/
	static void
	destroy_node_tables(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \internal
	* \brief Recursively copy the node tables of a resource
	*
	* Build a hash table containing copies of the allowed nodes tables of \p rsc
	* and its entire tree of descendants. The key is the resource ID, and the value
	* is a copy of the resource's node table.
	*
	* \param[in] rsc Resource whose node table to copy
	* \param[in,out] copy Where to store the copied node tables
	*
	* \note \p *copy should be \c NULL for the top-level call.
	* \note The caller is responsible for freeing \p copy using
	* \c g_hash_table_destroy().
	*/
	void
	pcmk__copy_node_tables(const pcmk_resource_t rsc, GHashTable *copy)
	{
	CRM_ASSERT((rsc != NULL) && (copy != NULL));

	if (*copy == NULL) {
	*copy = pcmk__strkey_table(NULL, destroy_node_tables);
	}

	g_hash_table_insert(*copy, rsc->id,
	pcmk__copy_node_table(rsc->allowed_nodes));

	for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk__copy_node_tables((const pcmk_resource_t *) iter->data, copy);
	}
	}

	/*!
	* \internal
	* \brief Recursively restore the node tables of a resource from backup
	*
	* Given a hash table containing backup copies of the allowed nodes tables of
	* \p rsc and its entire tree of descendants, replace the resources' current
	* node tables with the backed-up copies.
	*
	* \param[in,out] rsc Resource whose node tables to restore
	* \param[in] backup Table of backup node tables (created by
	* \c pcmk__copy_node_tables())
	*
	* \note This function frees the resources' current node tables.
	*/
	void
	pcmk__restore_node_tables(pcmk_resource_t rsc, GHashTable backup)
	{
	CRM_ASSERT((rsc != NULL) && (backup != NULL));

	g_hash_table_destroy(rsc->allowed_nodes);

	// Copy to avoid danger with multiple restores
	rsc->allowed_nodes = g_hash_table_lookup(backup, rsc->id);
	rsc->allowed_nodes = pcmk__copy_node_table(rsc->allowed_nodes);

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk__restore_node_tables((pcmk_resource_t *) iter->data, backup);
	}
	}

	/*!
	* \internal
	* \brief Copy a list of node objects
	*
	* \param[in] list List to copy
	* \param[in] reset Set copies' scores to 0
	*
	* \return New list of shallow copies of nodes in original list
	*/
	GList *
	pcmk__copy_node_list(const GList *list, bool reset)
	{
	GList *result = NULL;

	for (const GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk_node_t *new_node = NULL;
	pcmk_node_t *this_node = iter->data;

	new_node = pe__copy_node(this_node);
	if (reset) {
	new_node->weight = 0;
	}
	result = g_list_prepend(result, new_node);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Compare two nodes for assignment preference
	*
	* Given two nodes, check which one is more preferred by assignment criteria
	* such as node score and utilization.
	*
	* \param[in] a First node to compare
	* \param[in] b Second node to compare
	* \param[in] data Node to prefer if all else equal
	*
	* \return -1 if \p a is preferred, +1 if \p b is preferred, or 0 if they are
	* equally preferred
	*/
	static gint
	compare_nodes(gconstpointer a, gconstpointer b, gpointer data)
	{
	const pcmk_node_t node1 = (const pcmk_node_t ) a;
	const pcmk_node_t node2 = (const pcmk_node_t ) b;
	const pcmk_node_t preferred = (const pcmk_node_t ) data;

	int node1_score = -INFINITY;
	int node2_score = -INFINITY;

	int result = 0;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	// Compare node scores

	if (pcmk__node_available(node1, false, false)) {
	node1_score = node1->weight;
	}
	if (pcmk__node_available(node2, false, false)) {
	node2_score = node2->weight;
	}

	if (node1_score > node2_score) {
	crm_trace("%s before %s (score %d > %d)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	node1_score, node2_score);
	return -1;
	}

	if (node1_score < node2_score) {
	crm_trace("%s after %s (score %d < %d)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	node1_score, node2_score);
	return 1;
	}

	// If appropriate, compare node utilization

	if (pcmk__str_eq(node1->details->data_set->placement_strategy, "minimal",
	pcmk__str_casei)) {
	goto equal;
	}

	if (pcmk__str_eq(node1->details->data_set->placement_strategy, "balanced",
	pcmk__str_casei)) {
	result = pcmk__compare_node_capacities(node1, node2);
	if (result < 0) {
	crm_trace("%s before %s (greater capacity by %d attributes)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	result * -1);
	return -1;
	} else if (result > 0) {
	crm_trace("%s after %s (lower capacity by %d attributes)",
	pcmk__node_name(node1), pcmk__node_name(node2), result);
	return 1;
	}
	}

	// Compare number of resources already assigned to node

	if (node1->details->num_resources < node2->details->num_resources) {
	crm_trace("%s before %s (%d resources < %d)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	node1->details->num_resources, node2->details->num_resources);
	return -1;

	} else if (node1->details->num_resources > node2->details->num_resources) {
	crm_trace("%s after %s (%d resources > %d)",
	pcmk__node_name(node1), pcmk__node_name(node2),
	node1->details->num_resources, node2->details->num_resources);
	return 1;
	}

	// Check whether one node is already running desired resource

	if (preferred != NULL) {
	- if (pe__same_node(preferred, node1)) {
	+ if (pcmk__same_node(preferred, node1)) {
	crm_trace("%s before %s (preferred node)",
	pcmk__node_name(node1), pcmk__node_name(node2));
	return -1;
	- } else if (pe__same_node(preferred, node2)) {
	+ } else if (pcmk__same_node(preferred, node2)) {
	crm_trace("%s after %s (not preferred node)",
	pcmk__node_name(node1), pcmk__node_name(node2));
	return 1;
	}
	}

	// If all else is equal, prefer node with lowest-sorting name
	equal:
	result = strcmp(node1->details->uname, node2->details->uname);
	if (result < 0) {
	crm_trace("%s before %s (name)",
	pcmk__node_name(node1), pcmk__node_name(node2));
	return -1;
	} else if (result > 0) {
	crm_trace("%s after %s (name)",
	pcmk__node_name(node1), pcmk__node_name(node2));
	return 1;
	}

	crm_trace("%s == %s", pcmk__node_name(node1), pcmk__node_name(node2));
	return 0;
	}

	/*!
	* \internal
	* \brief Sort a list of nodes by assigment preference
	*
	* \param[in,out] nodes Node list to sort
	* \param[in] active_node Node where resource being assigned is active
	*
	* \return New head of sorted list
	*/
	GList *
	pcmk__sort_nodes(GList nodes, pcmk_node_t active_node)
	{
	return g_list_sort_with_data(nodes, compare_nodes, active_node);
	}

	/*!
	* \internal
	* \brief Check whether any node is available to run resources
	*
	* \param[in] nodes Nodes to check
	*
	* \return true if any node in \p nodes is available to run resources,
	* otherwise false
	*/
	bool
	pcmk__any_node_available(GHashTable *nodes)
	{
	GHashTableIter iter;
	const pcmk_node_t *node = NULL;

	if (nodes == NULL) {
	return false;
	}
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (pcmk__node_available(node, true, false)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Apply node health values for all nodes in cluster
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__apply_node_health(pcmk_scheduler_t *scheduler)
	{
	int base_health = 0;
	enum pcmk__health_strategy strategy;
	const char *strategy_str = pe_pref(scheduler->config_hash,
	PCMK_OPT_NODE_HEALTH_STRATEGY);

	strategy = pcmk__parse_health_strategy(strategy_str);
	if (strategy == pcmk__health_strategy_none) {
	return;
	}
	crm_info("Applying node health strategy '%s'", strategy_str);

	// The progressive strategy can use a base health score
	if (strategy == pcmk__health_strategy_progressive) {
	base_health = pe__health_score(PCMK_OPT_NODE_HEALTH_BASE, scheduler);
	}

	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	int health = pe__sum_node_health_scores(node, base_health);

	// An overall health score of 0 has no effect
	if (health == 0) {
	continue;
	}
	crm_info("Overall system health of %s is %d",
	pcmk__node_name(node), health);

	// Use node health as a location score for each resource on the node
	for (GList *r = scheduler->resources; r != NULL; r = r->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) r->data;

	bool constrain = true;

	if (health < 0) {
	/* Negative health scores do not apply to resources with
	* PCMK_META_ALLOW_UNHEALTHY_NODES=true.
	*/
	constrain = !crm_is_true(g_hash_table_lookup(rsc->meta,
	PCMK_META_ALLOW_UNHEALTHY_NODES));
	}
	if (constrain) {
	pcmk__new_location(strategy_str, rsc, health, NULL, node);
	} else {
	pcmk__rsc_trace(rsc, "%s is immune from health ban on %s",
	rsc->id, pcmk__node_name(node));
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Check for a node in a resource's parent's allowed nodes
	*
	* \param[in] rsc Resource whose parent should be checked
	* \param[in] node Node to check for
	*
	* \return Equivalent of \p node from \p rsc's parent's allowed nodes if any,
	* otherwise NULL
	*/
	pcmk_node_t *
	pcmk__top_allowed_node(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	GHashTable *allowed_nodes = NULL;

	if ((rsc == NULL) \|\| (node == NULL)) {
	return NULL;
	} else if (rsc->parent == NULL) {
	allowed_nodes = rsc->allowed_nodes;
	} else {
	allowed_nodes = rsc->parent->allowed_nodes;
	}
	return g_hash_table_lookup(allowed_nodes, node->details->id);
	}
	diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c
	index 98ae0a56c0..d6f643b70b 100644
	--- a/lib/pacemaker/pcmk_sched_ordering.c
	+++ b/lib/pacemaker/pcmk_sched_ordering.c
	@@ -1,1505 +1,1505 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <inttypes.h> // PRIx32
	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	enum pe_order_kind {
	pe_order_kind_optional,
	pe_order_kind_mandatory,
	pe_order_kind_serialize,
	};

	enum ordering_symmetry {
	ordering_asymmetric, // the only relation in an asymmetric ordering
	ordering_symmetric, // the normal relation in a symmetric ordering
	ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
	};

	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	__rsc = pcmk__find_constraint_resource(scheduler->resources, \
	__name); \
	if (__rsc == NULL) { \
	pcmk__config_err("%s: No resource found for %s", __set, __name);\
	return pcmk_rc_unpack_error; \
	} \
	} while (0)

	static const char *
	invert_action(const char *action)
	{
	if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
	return PCMK_ACTION_STOP;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
	return PCMK_ACTION_START;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
	return PCMK_ACTION_STOPPED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
	return PCMK_ACTION_RUNNING;
	}
	pcmk__config_warn("Unknown action '%s' specified in order constraint",
	action);
	return NULL;
	}

	static enum pe_order_kind
	get_ordering_type(const xmlNode *xml_obj)
	{
	enum pe_order_kind kind_e = pe_order_kind_mandatory;
	const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);

	if (kind == NULL) {
	const char *score = crm_element_value(xml_obj, PCMK_XA_SCORE);

	kind_e = pe_order_kind_mandatory;

	if (score) {
	// @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
	int score_i = char2score(score);

	if (score_i == 0) {
	kind_e = pe_order_kind_optional;
	}
	pcmk__warn_once(pcmk__wo_order_score,
	"Support for 'score' in rsc_order is deprecated "
	"and will be removed in a future release "
	"(use 'kind' instead)");
	}

	} else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_none)) {
	kind_e = pe_order_kind_mandatory;

	} else if (pcmk__str_eq(kind, "Optional", pcmk__str_none)) {
	kind_e = pe_order_kind_optional;

	} else if (pcmk__str_eq(kind, "Serialize", pcmk__str_none)) {
	kind_e = pe_order_kind_serialize;

	} else {
	pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
	"%s to 'Mandatory' because '%s' is not valid",
	pcmk__s(ID(xml_obj), "missing ID"), kind);
	}
	return kind_e;
	}

	/*!
	* \internal
	* \brief Get ordering symmetry from XML
	*
	* \param[in] xml_obj Ordering XML
	* \param[in] parent_kind Default ordering kind
	* \param[in] parent_symmetrical_s Parent element's \c PCMK_XA_SYMMETRICAL
	* setting, if any
	*
	* \retval ordering_symmetric Ordering is symmetric
	* \retval ordering_asymmetric Ordering is asymmetric
	*/
	static enum ordering_symmetry
	get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
	const char *parent_symmetrical_s)
	{
	int rc = pcmk_rc_ok;
	bool symmetric = false;
	enum pe_order_kind kind = parent_kind; // Default to parent's kind

	// Check ordering XML for explicit kind
	if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL)
	\|\| (crm_element_value(xml_obj, PCMK_XA_SCORE) != NULL)) {
	kind = get_ordering_type(xml_obj);
	}

	// Check ordering XML (and parent) for explicit PCMK_XA_SYMMETRICAL setting
	rc = pcmk__xe_get_bool_attr(xml_obj, PCMK_XA_SYMMETRICAL, &symmetric);

	if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
	symmetric = crm_is_true(parent_symmetrical_s);
	rc = pcmk_rc_ok;
	}

	if (rc == pcmk_rc_ok) {
	if (symmetric) {
	if (kind == pe_order_kind_serialize) {
	pcmk__config_warn("Ignoring " PCMK_XA_SYMMETRICAL
	" for '%s' because not valid with "
	XML_ORDER_ATTR_KIND " of 'Serialize'",
	ID(xml_obj));
	} else {
	return ordering_symmetric;
	}
	}
	return ordering_asymmetric;
	}

	// Use default symmetry
	if (kind == pe_order_kind_serialize) {
	return ordering_asymmetric;
	}
	return ordering_symmetric;
	}

	/*!
	* \internal
	* \brief Get ordering flags appropriate to ordering kind
	*
	* \param[in] kind Ordering kind
	* \param[in] first Action name for 'first' action
	* \param[in] symmetry This ordering's symmetry role
	*
	* \return Minimal ordering flags appropriate to \p kind
	*/
	static uint32_t
	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
	enum ordering_symmetry symmetry)
	{
	uint32_t flags = pcmk__ar_none; // so we trace-log all flags set

	switch (kind) {
	case pe_order_kind_optional:
	pcmk__set_relation_flags(flags, pcmk__ar_ordered);
	break;

	case pe_order_kind_serialize:
	/* This flag is not used anywhere directly but means the relation
	* will not match an equality comparison against pcmk__ar_none or
	* pcmk__ar_ordered.
	*/
	pcmk__set_relation_flags(flags, pcmk__ar_serialize);
	break;

	case pe_order_kind_mandatory:
	pcmk__set_relation_flags(flags, pcmk__ar_ordered);
	switch (symmetry) {
	case ordering_asymmetric:
	pcmk__set_relation_flags(flags, pcmk__ar_asymmetric);
	break;

	case ordering_symmetric:
	pcmk__set_relation_flags(flags,
	pcmk__ar_first_implies_then);
	if (pcmk__strcase_any_of(first, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	pcmk__set_relation_flags(flags,
	pcmk__ar_unrunnable_first_blocks);
	}
	break;

	case ordering_symmetric_inverse:
	pcmk__set_relation_flags(flags,
	pcmk__ar_then_implies_first);
	break;
	}
	break;
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Find resource corresponding to ID specified in ordering
	*
	* \param[in] xml Ordering XML
	* \param[in] resource_attr XML attribute name for resource ID
	* \param[in] instance_attr XML attribute name for instance number.
	* This option is deprecated and will be removed in a
	* future release.
	* \param[in] scheduler Scheduler data
	*
	* \return Resource corresponding to \p id, or NULL if none
	*/
	static pcmk_resource_t *
	get_ordering_resource(const xmlNode xml, const char resource_attr,
	const char *instance_attr,
	const pcmk_scheduler_t *scheduler)
	{
	// @COMPAT: instance_attr and instance_id variables deprecated since 2.1.5
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(xml, resource_attr);
	const char *instance_id = crm_element_value(xml, instance_attr);

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without %s",
	ID(xml), resource_attr);
	return NULL;
	}

	rsc = pcmk__find_constraint_resource(scheduler->resources, rsc_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", ID(xml), rsc_id);
	return NULL;
	}

	if (instance_id != NULL) {
	pcmk__warn_once(pcmk__wo_order_inst,
	"Support for " PCMK__XA_FIRST_INSTANCE " and "
	PCMK__XA_THEN_INSTANCE " is deprecated and will be "
	"removed in a future release.");

	if (!pe_rsc_is_clone(rsc)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	ID(xml), rsc_id, instance_id);
	return NULL;
	}
	rsc = find_clone_instance(rsc, instance_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", ID(xml), rsc_id, instance_id);
	return NULL;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine minimum number of 'first' instances required in ordering
	*
	* \param[in] rsc 'First' resource in ordering
	* \param[in] xml Ordering XML
	*
	* \return Minimum 'first' instances required (or 0 if not applicable)
	*/
	static int
	get_minimum_first_instances(const pcmk_resource_t rsc, const xmlNode xml)
	{
	const char *clone_min = NULL;
	bool require_all = false;

	if (!pe_rsc_is_clone(rsc)) {
	return 0;
	}

	clone_min = g_hash_table_lookup(rsc->meta, PCMK_META_CLONE_MIN);
	if (clone_min != NULL) {
	int clone_min_int = 0;

	pcmk__scan_min_int(clone_min, &clone_min_int, 0);
	return clone_min_int;
	}

	/* @COMPAT 1.1.13:
	* require-all=false is deprecated equivalent of PCMK_META_CLONE_MIN=1
	*/
	if (pcmk__xe_get_bool_attr(xml, "require-all", &require_all) != ENODATA) {
	pcmk__warn_once(pcmk__wo_require_all,
	"Support for require-all in ordering constraints "
	"is deprecated and will be removed in a future release "
	"(use " PCMK_META_CLONE_MIN " clone meta-attribute "
	"instead)");
	if (!require_all) {
	return 1;
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Create orderings for a constraint with \c PCMK_META_CLONE_MIN > 0
	*
	* \param[in] id Ordering ID
	* \param[in,out] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	* \param[in] flags Ordering flags
	* \param[in] clone_min Minimum required instances of 'first'
	*/
	static void
	clone_min_ordering(const char *id,
	pcmk_resource_t rsc_first, const char action_first,
	pcmk_resource_t rsc_then, const char action_then,
	uint32_t flags, int clone_min)
	{
	// Create a pseudo-action for when the minimum instances are active
	char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
	pcmk_action_t *clone_min_met = get_pseudo_op(task, rsc_first->cluster);

	free(task);

	/* Require the pseudo-action to have the required number of actions to be
	* considered runnable before allowing the pseudo-action to be runnable.
	*/
	clone_min_met->required_runnable_before = clone_min;
	pcmk__set_action_flags(clone_min_met, pcmk_action_min_runnable);

	// Order the actions for each clone instance before the pseudo-action
	for (GList *iter = rsc_first->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t *child = iter->data;

	pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
	NULL, NULL, NULL, clone_min_met,
	pcmk__ar_min_runnable
	\|pcmk__ar_first_implies_then_graphed,
	rsc_first->cluster);
	}

	// Order "then" action after the pseudo-action (if runnable)
	pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
	pcmk__op_key(rsc_then->id, action_then, 0),
	NULL, flags\|pcmk__ar_unrunnable_first_blocks,
	rsc_first->cluster);
	}

	/*!
	* \internal
	* \brief Update ordering flags for restart-type=restart
	*
	* \param[in] rsc 'Then' resource in ordering
	* \param[in] kind Ordering kind
	* \param[in] flag Ordering flag to set (when applicable)
	* \param[in,out] flags Ordering flag set to update
	*
	* \compat The \c PCMK__META_RESTART_TYPE resource meta-attribute is deprecated.
	* Eventually, it will be removed, and \c pe_restart_ignore will be the
	* only behavior, at which time this can just be removed entirely.
	*/
	#define handle_restart_type(rsc, kind, flag, flags) do { \
	if (((kind) == pe_order_kind_optional) \
	&& ((rsc)->restart_type == pe_restart_restart)) { \
	pcmk__set_relation_flags((flags), (flag)); \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Create new ordering for inverse of symmetric constraint
	*
	* \param[in] id Ordering ID (for logging only)
	* \param[in] kind Ordering kind
	* \param[in] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in,out] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	*/
	static void
	inverse_ordering(const char *id, enum pe_order_kind kind,
	pcmk_resource_t rsc_first, const char action_first,
	pcmk_resource_t rsc_then, const char action_then)
	{
	action_then = invert_action(action_then);
	action_first = invert_action(action_first);
	if ((action_then == NULL) \|\| (action_first == NULL)) {
	pcmk__config_warn("Cannot invert constraint '%s' "
	"(please specify inverse manually)", id);
	} else {
	uint32_t flags = ordering_flags_for_kind(kind, action_first,
	ordering_symmetric_inverse);

	handle_restart_type(rsc_then, kind, pcmk__ar_then_implies_first, flags);
	pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
	action_first, flags);
	}
	}

	static void
	unpack_simple_rsc_order(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *rsc_then = NULL;
	pcmk_resource_t *rsc_first = NULL;
	int min_required_before = 0;
	enum pe_order_kind kind = pe_order_kind_mandatory;
	uint32_t flags = pcmk__ar_none;
	enum ordering_symmetry symmetry;

	const char *action_then = NULL;
	const char *action_first = NULL;
	const char *id = NULL;

	CRM_CHECK(xml_obj != NULL, return);

	id = crm_element_value(xml_obj, PCMK_XA_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID,
	xml_obj->name);
	return;
	}

	rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST,
	PCMK__XA_FIRST_INSTANCE, scheduler);
	if (rsc_first == NULL) {
	return;
	}

	rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN,
	PCMK__XA_THEN_INSTANCE, scheduler);
	if (rsc_then == NULL) {
	return;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	if (action_first == NULL) {
	action_first = PCMK_ACTION_START;
	}

	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
	if (action_then == NULL) {
	action_then = action_first;
	}

	kind = get_ordering_type(xml_obj);

	symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
	flags = ordering_flags_for_kind(kind, action_first, symmetry);

	handle_restart_type(rsc_then, kind, pcmk__ar_first_implies_then, flags);

	/* If there is a minimum number of instances that must be runnable before
	* the 'then' action is runnable, we use a pseudo-action for convenience:
	* minimum number of clone instances have runnable actions ->
	* pseudo-action is runnable -> dependency is runnable.
	*/
	min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
	if (min_required_before > 0) {
	clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
	flags, min_required_before);
	} else {
	pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
	action_then, flags);
	}

	if (symmetry == ordering_symmetric) {
	inverse_ordering(id, kind, rsc_first, action_first,
	rsc_then, action_then);
	}
	}

	/*!
	* \internal
	* \brief Create a new ordering between two actions
	*
	* \param[in,out] first_rsc Resource for 'first' action (if NULL and
	* \p first_action is a resource action, that
	* resource will be used)
	* \param[in,out] first_action_task Action key for 'first' action (if NULL and
	* \p first_action is not NULL, its UUID will
	* be used)
	* \param[in,out] first_action 'first' action (if NULL, \p first_rsc and
	* \p first_action_task must be set)
	*
	* \param[in] then_rsc Resource for 'then' action (if NULL and
	* \p then_action is a resource action, that
	* resource will be used)
	* \param[in,out] then_action_task Action key for 'then' action (if NULL and
	* \p then_action is not NULL, its UUID will
	* be used)
	* \param[in] then_action 'then' action (if NULL, \p then_rsc and
	* \p then_action_task must be set)
	*
	* \param[in] flags Group of enum pcmk__action_relation_flags
	* \param[in,out] sched Scheduler data to add ordering to
	*
	* \note This function takes ownership of first_action_task and
	* then_action_task, which do not need to be freed by the caller.
	*/
	void
	pcmk__new_ordering(pcmk_resource_t first_rsc, char first_action_task,
	pcmk_action_t first_action, pcmk_resource_t then_rsc,
	char then_action_task, pcmk_action_t then_action,
	uint32_t flags, pcmk_scheduler_t *sched)
	{
	pcmk__action_relation_t *order = NULL;

	// One of action or resource must be specified for each side
	CRM_CHECK(((first_action != NULL) \|\| (first_rsc != NULL))
	&& ((then_action != NULL) \|\| (then_rsc != NULL)),
	free(first_action_task); free(then_action_task); return);

	if ((first_rsc == NULL) && (first_action != NULL)) {
	first_rsc = first_action->rsc;
	}
	if ((then_rsc == NULL) && (then_action != NULL)) {
	then_rsc = then_action->rsc;
	}

	order = calloc(1, sizeof(pcmk__action_relation_t));
	CRM_ASSERT(order != NULL);

	order->id = sched->order_id++;
	order->flags = flags;
	order->rsc1 = first_rsc;
	order->rsc2 = then_rsc;
	order->action1 = first_action;
	order->action2 = then_action;
	order->task1 = first_action_task;
	order->task2 = then_action_task;

	if ((order->task1 == NULL) && (first_action != NULL)) {
	order->task1 = strdup(first_action->uuid);
	}

	if ((order->task2 == NULL) && (then_action != NULL)) {
	order->task2 = strdup(then_action->uuid);
	}

	if ((order->rsc1 == NULL) && (first_action != NULL)) {
	order->rsc1 = first_action->rsc;
	}

	if ((order->rsc2 == NULL) && (then_action != NULL)) {
	order->rsc2 = then_action->rsc;
	}

	pcmk__rsc_trace(first_rsc, "Created ordering %d for %s then %s",
	(sched->order_id - 1),
	pcmk__s(order->task1, "an underspecified action"),
	pcmk__s(order->task2, "an underspecified action"));

	sched->ordering_constraints = g_list_prepend(sched->ordering_constraints,
	order);
	pcmk__order_migration_equivalents(order);
	}

	/*!
	* \brief Unpack a set in an ordering constraint
	*
	* \param[in] set Set XML to unpack
	* \param[in] parent_kind rsc_order XML "kind" attribute
	* \param[in] parent_symmetrical_s rsc_order XML \c PCMK_XA_SYMMETRICAL
	* attribute
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
	const char parent_symmetrical_s, pcmk_scheduler_t scheduler)
	{
	GList *set_iter = NULL;
	GList *resources = NULL;

	pcmk_resource_t *last = NULL;
	pcmk_resource_t *resource = NULL;

	int local_kind = parent_kind;
	bool sequential = false;
	uint32_t flags = pcmk__ar_ordered;
	enum ordering_symmetry symmetry;

	char *key = NULL;
	const char *id = ID(set);
	const char *action = crm_element_value(set, "action");
	const char *sequential_s = crm_element_value(set, "sequential");
	const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);

	if (action == NULL) {
	action = PCMK_ACTION_START;
	}

	if (kind_s) {
	local_kind = get_ordering_type(set);
	}
	if (sequential_s == NULL) {
	sequential_s = "1";
	}

	sequential = crm_is_true(sequential_s);

	symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
	flags = ordering_flags_for_kind(local_kind, action, symmetry);

	for (const xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
	resources = g_list_append(resources, resource);
	}

	if (pcmk__list_of_1(resources)) {
	crm_trace("Single set: %s", id);
	goto done;
	}

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pcmk_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = pcmk__op_key(resource->id, action, 0);

	if (local_kind == pe_order_kind_serialize) {
	/* Serialize before everything that comes after */

	for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
	pcmk_resource_t *then_rsc = iter->data;
	char *then_key = pcmk__op_key(then_rsc->id, action, 0);

	pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
	then_key, NULL, flags, scheduler);
	}

	} else if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(last, action, resource, action,
	flags);
	}
	last = resource;
	}
	free(key);
	}

	if (symmetry == ordering_asymmetric) {
	goto done;
	}

	last = NULL;
	action = invert_action(action);

	flags = ordering_flags_for_kind(local_kind, action,
	ordering_symmetric_inverse);

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pcmk_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(resource, action, last, action,
	flags);
	}
	last = resource;
	}
	}

	done:
	g_list_free(resources);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Order two resource sets relative to each other
	*
	* \param[in] id Ordering ID (for logging)
	* \param[in] set1 First listed set
	* \param[in] set2 Second listed set
	* \param[in] kind Ordering kind
	* \param[in,out] scheduler Scheduler data
	* \param[in] symmetry Which ordering symmetry applies to this relation
	*
	* \return Standard Pacemaker return code
	*/
	static int
	order_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	enum pe_order_kind kind, pcmk_scheduler_t *scheduler,
	enum ordering_symmetry symmetry)
	{

	const xmlNode *xml_rsc = NULL;
	const xmlNode *xml_rsc_2 = NULL;

	pcmk_resource_t *rsc_1 = NULL;
	pcmk_resource_t *rsc_2 = NULL;

	const char *action_1 = crm_element_value(set1, "action");
	const char *action_2 = crm_element_value(set2, "action");

	uint32_t flags = pcmk__ar_none;

	bool require_all = true;

	(void) pcmk__xe_get_bool_attr(set1, "require-all", &require_all);

	if (action_1 == NULL) {
	action_1 = PCMK_ACTION_START;
	}

	if (action_2 == NULL) {
	action_2 = PCMK_ACTION_START;
	}

	if (symmetry == ordering_symmetric_inverse) {
	action_1 = invert_action(action_1);
	action_2 = invert_action(action_2);
	}

	if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
	\|\| pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
	/* Assuming: A -> ( B \|\| C) -> D
	* The one-or-more logic only applies during the start/promote phase.
	* During shutdown neither B nor can shutdown until D is down, so simply
	* turn require_all back on.
	*/
	require_all = true;
	}

	flags = ordering_flags_for_kind(kind, action_1, symmetry);

	/* If we have an unordered set1, whether it is sequential or not is
	* irrelevant in regards to set2.
	*/
	if (!require_all) {
	char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s", ID(set1));
	pcmk_action_t *unordered_action = get_pseudo_op(task, scheduler);

	free(task);
	pcmk__set_action_flags(unordered_action, pcmk_action_min_runnable);

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	/* Add an ordering constraint between every element in set1 and the
	* pseudo action. If any action in set1 is runnable the pseudo
	* action will be runnable.
	*/
	pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
	NULL, NULL, NULL, unordered_action,
	pcmk__ar_min_runnable
	\|pcmk__ar_first_implies_then_graphed,
	scheduler);
	}
	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));

	/* Add an ordering constraint between the pseudo-action and every
	* element in set2. If the pseudo-action is runnable, every action
	* in set2 will be runnable.
	*/
	pcmk__new_ordering(NULL, NULL, unordered_action,
	rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
	NULL, flags\|pcmk__ar_unrunnable_first_blocks,
	scheduler);
	}

	return pcmk_rc_ok;
	}

	if (pcmk__xe_attr_is_true(set1, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the first one
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	}

	} else {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
	}
	}

	if (pcmk__xe_attr_is_true(set2, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);

	} else {
	// Get the first one
	xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	}
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);

	} else if (rsc_1 != NULL) {
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else if (rsc_2 != NULL) {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	for (xmlNode *xml_rsc_2 = first_named_child(set2,
	XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
	action_2, flags);
	}
	}
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief If an ordering constraint uses resource tags, expand them
	*
	* \param[in,out] xml_obj Ordering constraint XML
	* \param[out] expanded_xml Equivalent XML with tags expanded
	* \param[in] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
	* and pcmk_rc_unpack_error on invalid configuration)
	*/
	static int
	unpack_order_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	const pcmk_scheduler_t *scheduler)
	{
	const char *id_first = NULL;
	const char *id_then = NULL;
	const char *action_first = NULL;
	const char *action_then = NULL;

	pcmk_resource_t *rsc_first = NULL;
	pcmk_resource_t *rsc_then = NULL;
	pcmk_tag_t *tag_first = NULL;
	pcmk_tag_t *tag_then = NULL;

	xmlNode *rsc_set_first = NULL;
	xmlNode *rsc_set_then = NULL;
	bool any_sets = false;

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	return pcmk_rc_ok;
	}

	id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
	id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
	if ((id_first == NULL) \|\| (id_then == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id_first, &rsc_first,
	&tag_first)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_first);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id_then, &rsc_then,
	&tag_then)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_then);
	return pcmk_rc_unpack_error;
	}

	if ((rsc_first != NULL) && (rsc_then != NULL)) {
	// Neither side references a template or tag
	return pcmk_rc_ok;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);

	*expanded_xml = copy_xml(xml_obj);

	// Convert template/tag reference in "first" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_first != NULL) {
	if (action_first != NULL) {
	// Move "first-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_first, "action", action_first);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION);
	}
	any_sets = true;
	}

	// Convert template/tag reference in "then" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_then != NULL) {
	if (action_then != NULL) {
	// Move "then-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_then, "action", action_then);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Unpack ordering constraint XML
	*
	* \param[in,out] xml_obj Ordering constraint XML to unpack
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__unpack_ordering(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
	const char *invert = crm_element_value(xml_obj, PCMK_XA_SYMMETRICAL);
	enum pe_order_kind kind = get_ordering_type(xml_obj);

	enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
	NULL);

	// Expand any resource tags in the constraint XML
	if (unpack_order_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	// If the constraint has resource sets, unpack them
	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	set = expand_idref(set, scheduler->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_order_set(set, kind, invert, scheduler) != pcmk_rc_ok)) {

	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if (last != NULL) {

	if (order_rsc_sets(id, last, set, kind, scheduler,
	symmetry) != pcmk_rc_ok) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if ((symmetry == ordering_symmetric)
	&& (order_rsc_sets(id, set, last, kind, scheduler,
	ordering_symmetric_inverse) != pcmk_rc_ok)) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	// If the constraint has no resource sets, unpack it as a simple ordering
	if (last == NULL) {
	return unpack_simple_rsc_order(xml_obj, scheduler);
	}
	}

	static bool
	ordering_is_invalid(pcmk_action_t action, pcmk__related_action_t input)
	{
	/* Prevent user-defined ordering constraints between resources
	* running in a guest node and the resource that defines that node.
	*/
	if (!pcmk_is_set(input->type, pcmk__ar_guest_allowed)
	&& (input->action->rsc != NULL)
	&& pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {

	pcmk__config_warn("Invalid ordering constraint between %s and %s",
	input->action->rsc->id, action->rsc->id);
	return true;
	}

	/* If there's an order like
	* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
	*
	* then rscA is being migrated from node1 to node2, while rscB is being
	* migrated from node2 to node1. If there would be a graph loop,
	* break the order "load_stopped_node2" -> "rscA_migrate_to node1".
	*/
	if (((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target)
	&& (action->rsc != NULL)
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
	&& pcmk__graph_has_loop(action, action, input)) {
	return true;
	}

	return false;
	}

	void
	pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;
	pcmk__related_action_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = input_iter->data;
	if (ordering_is_invalid(action, input)) {
	input->type = (enum pe_ordering) pcmk__ar_none;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Order stops on a node before the node's shutdown
	*
	* \param[in,out] node Node being shut down
	* \param[in] shutdown_op Shutdown action for node
	*/
	void
	pcmk__order_stops_before_shutdown(pcmk_node_t node, pcmk_action_t shutdown_op)
	{
	for (GList *iter = node->details->data_set->actions;
	iter != NULL; iter = iter->next) {

	pcmk_action_t action = (pcmk_action_t ) iter->data;

	// Only stops on the node shutting down are relevant
	- if (!pe__same_node(action->node, node)
	+ if (!pcmk__same_node(action->node, node)
	\|\| !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	continue;
	}

	// Resources and nodes in maintenance mode won't be touched

	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)) {
	pcmk__rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource in maintenance mode",
	action->uuid, pcmk__node_name(node));
	continue;

	} else if (node->details->maintenance) {
	pcmk__rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"node in maintenance mode",
	action->uuid, pcmk__node_name(node));
	continue;
	}

	/* Don't touch a resource that is unmanaged or blocked, to avoid
	* blocking the shutdown (though if another action depends on this one,
	* we may still end up blocking)
	*/
	if (!pcmk_any_flags_set(action->rsc->flags,
	pcmk_rsc_managed\|pcmk_rsc_blocked)) {
	pcmk__rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource is unmanaged or blocked",
	action->uuid, pcmk__node_name(node));
	continue;
	}

	pcmk__rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
	action->uuid, pcmk__node_name(node));
	pcmk__clear_action_flags(action, pcmk_action_optional);
	pcmk__new_ordering(action->rsc, NULL, action, NULL,
	strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
	pcmk__ar_ordered\|pcmk__ar_unrunnable_first_blocks,
	node->details->data_set);
	}
	}

	/*!
	* \brief Find resource actions matching directly or as child
	*
	* \param[in] rsc Resource to check
	* \param[in] original_key Action key to search for (possibly referencing
	* parent of \rsc)
	*
	* \return Newly allocated list of matching actions
	* \note It is the caller's responsibility to free the result with g_list_free()
	*/
	static GList *
	find_actions_by_task(const pcmk_resource_t rsc, const char original_key)
	{
	// Search under given task key directly
	GList *list = find_actions(rsc->actions, original_key, NULL);

	if (list == NULL) {
	// Search again using this resource's ID
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	CRM_CHECK(parse_op_key(original_key, NULL, &task, &interval_ms),
	return NULL);
	key = pcmk__op_key(rsc->id, task, interval_ms);
	list = find_actions(rsc->actions, key, NULL);
	free(key);
	free(task);
	}
	return list;
	}

	/*!
	* \internal
	* \brief Order relevant resource actions after a given action
	*
	* \param[in,out] first_action Action to order after (or NULL if none runnable)
	* \param[in] rsc Resource whose actions should be ordered
	* \param[in,out] order Ordering constraint being applied
	*/
	static void
	order_resource_actions_after(pcmk_action_t *first_action,
	const pcmk_resource_t *rsc,
	pcmk__action_relation_t *order)
	{
	GList *then_actions = NULL;
	uint32_t flags = pcmk__ar_none;

	CRM_CHECK((rsc != NULL) && (order != NULL), return);

	flags = order->flags;
	pcmk__rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
	order->id, rsc->id);

	if (order->action2 != NULL) {
	then_actions = g_list_prepend(NULL, order->action2);

	} else {
	then_actions = find_actions_by_task(rsc, order->task2);
	}

	if (then_actions == NULL) {
	pcmk__rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
	order->id, order->task2, rsc->id);
	return;
	}

	if ((first_action != NULL) && (first_action->rsc == rsc)
	&& pcmk_is_set(first_action->flags, pcmk_action_migration_abort)) {

	pcmk__rsc_trace(rsc,
	"Detected dangling migration ordering (%s then %s %s)",
	first_action->uuid, order->task2, rsc->id);
	pcmk__clear_relation_flags(flags, pcmk__ar_first_implies_then);
	}

	if ((first_action == NULL)
	&& !pcmk_is_set(flags, pcmk__ar_first_implies_then)) {

	pcmk__rsc_debug(rsc,
	"Ignoring ordering %d for %s: No first action found",
	order->id, rsc->id);
	g_list_free(then_actions);
	return;
	}

	for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
	pcmk_action_t then_action_iter = (pcmk_action_t ) iter->data;

	if (first_action != NULL) {
	order_actions(first_action, then_action_iter, flags);
	} else {
	pcmk__clear_action_flags(then_action_iter, pcmk_action_runnable);
	crm_warn("%s of %s is unrunnable because there is no %s of %s "
	"to order it after", then_action_iter->task, rsc->id,
	order->task1, order->rsc1->id);
	}
	}

	g_list_free(then_actions);
	}

	static void
	rsc_order_first(pcmk_resource_t first_rsc, pcmk__action_relation_t order)
	{
	GList *first_actions = NULL;
	pcmk_action_t *first_action = order->action1;
	pcmk_resource_t *then_rsc = order->rsc2;

	CRM_ASSERT(first_rsc != NULL);
	pcmk__rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
	order->id, first_rsc->id);

	if (first_action != NULL) {
	first_actions = g_list_prepend(NULL, first_action);

	} else {
	first_actions = find_actions_by_task(first_rsc, order->task1);
	}

	if ((first_actions == NULL) && (first_rsc == then_rsc)) {
	pcmk__rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->task1, first_rsc->id);

	} else if (first_actions == NULL) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;

	parse_op_key(order->task1, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(first_rsc->id, op_type, interval_ms);

	if ((first_rsc->fns->state(first_rsc, TRUE) == pcmk_role_stopped)
	&& pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
	free(key);
	pcmk__rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) "
	"not found",
	order->id, order->task1, first_rsc->id);

	} else if ((first_rsc->fns->state(first_rsc,
	TRUE) == pcmk_role_unpromoted)
	&& pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	free(key);
	pcmk__rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) "
	"not found",
	order->id, order->task1, first_rsc->id);

	} else {
	pcmk__rsc_trace(first_rsc,
	"Creating first (%s for %s) for constraint %d ",
	order->task1, first_rsc->id, order->id);
	first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
	first_rsc->cluster);
	first_actions = g_list_prepend(NULL, first_action);
	}

	free(op_type);
	}

	if (then_rsc == NULL) {
	if (order->action2 == NULL) {
	pcmk__rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
	order->id);
	return;
	}
	then_rsc = order->action2->rsc;
	}
	for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
	first_action = iter->data;

	if (then_rsc == NULL) {
	order_actions(first_action, order->action2, order->flags);

	} else {
	order_resource_actions_after(first_action, then_rsc, order);
	}
	}

	g_list_free(first_actions);
	}

	// GFunc to call pcmk__block_colocation_dependents()
	static void
	block_colocation_dependents(gpointer data, gpointer user_data)
	{
	pcmk__block_colocation_dependents(data);
	}

	// GFunc to call pcmk__update_action_for_orderings()
	static void
	update_action_for_orderings(gpointer data, gpointer user_data)
	{
	pcmk__update_action_for_orderings((pcmk_action_t *) data,
	(pcmk_scheduler_t *) user_data);
	}

	/*!
	* \internal
	* \brief Apply all ordering constraints
	*
	* \param[in,out] sched Scheduler data
	*/
	void
	pcmk__apply_orderings(pcmk_scheduler_t *sched)
	{
	crm_trace("Applying ordering constraints");

	/* Ordering constraints need to be processed in the order they were created.
	* rsc_order_first() and order_resource_actions_after() require the relevant
	* actions to already exist in some cases, but rsc_order_first() will create
	* the 'first' action in certain cases. Thus calling rsc_order_first() can
	* change the behavior of later-created orderings.
	*
	* Also, g_list_append() should be avoided for performance reasons, so we
	* prepend orderings when creating them and reverse the list here.
	*
	* @TODO This is brittle and should be carefully redesigned so that the
	* order of creation doesn't matter, and the reverse becomes unneeded.
	*/
	sched->ordering_constraints = g_list_reverse(sched->ordering_constraints);

	for (GList *iter = sched->ordering_constraints;
	iter != NULL; iter = iter->next) {

	pcmk__action_relation_t *order = iter->data;
	pcmk_resource_t *rsc = order->rsc1;

	if (rsc != NULL) {
	rsc_order_first(rsc, order);
	continue;
	}

	rsc = order->rsc2;
	if (rsc != NULL) {
	order_resource_actions_after(order->action1, rsc, order);

	} else {
	crm_trace("Applying ordering constraint %d (non-resource actions)",
	order->id);
	order_actions(order->action1, order->action2, order->flags);
	}
	}

	g_list_foreach(sched->actions, block_colocation_dependents, NULL);

	crm_trace("Ordering probes");
	pcmk__order_probes(sched);

	crm_trace("Updating %d actions", g_list_length(sched->actions));
	g_list_foreach(sched->actions, update_action_for_orderings, sched);

	pcmk__disable_invalid_orderings(sched);
	}

	/*!
	* \internal
	* \brief Order a given action after each action in a given list
	*
	* \param[in,out] after "After" action
	* \param[in,out] list List of "before" actions
	*/
	void
	pcmk__order_after_each(pcmk_action_t after, GList list)
	{
	const char *after_desc = (after->task == NULL)? after->uuid : after->task;

	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk_action_t before = (pcmk_action_t ) iter->data;
	const char *before_desc = before->task? before->task : before->uuid;

	crm_debug("Ordering %s on %s before %s on %s",
	before_desc, pcmk__node_name(before->node),
	after_desc, pcmk__node_name(after->node));
	order_actions(before, after, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Order promotions and demotions for restarts of a clone or bundle
	*
	* \param[in,out] rsc Clone or bundle to order
	*/
	void
	pcmk__promotable_restart_ordering(pcmk_resource_t *rsc)
	{
	// Order start and promote after all instances are stopped
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order stop, start, and promote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order promote after all instances are started
	pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order demote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
	rsc, PCMK_ACTION_DEMOTED,
	pcmk__ar_ordered);
	}
	diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c
	index d7dd53787e..1a1a41d833 100644
	--- a/lib/pacemaker/pcmk_sched_primitive.c
	+++ b/lib/pacemaker/pcmk_sched_primitive.c
	@@ -1,1672 +1,1672 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <stdint.h> // uint8_t, uint32_t

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	static void stop_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void start_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void demote_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void promote_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void assert_role_error(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);

	#define RSC_ROLE_MAX (pcmk_role_promoted + 1)

	static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the immediate next role when transitioning from one role
	* to a target role. For example, when going from Stopped to Promoted, the
	* next role is Unpromoted, because the resource must be started before it
	* can be promoted. The current state then becomes Started, which is fed
	* into this array again, giving a next role of Promoted.
	*
	* Current role Immediate next role Final target role
	* ------------ ------------------- -----------------
	*/
	/* Unknown / { pcmk_role_unknown, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_stopped, /* Started */
	pcmk_role_stopped, /* Unpromoted */
	pcmk_role_stopped, /* Promoted */
	},
	/* Stopped / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_started, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_unpromoted, /* Promoted */
	},
	/* Started / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_started, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	/* Unpromoted / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_stopped, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	/* Promoted / { pcmk_role_stopped, / Unknown */
	pcmk_role_unpromoted, /* Stopped */
	pcmk_role_unpromoted, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Function to schedule actions needed for a role change
	*
	* \param[in,out] rsc Resource whose role is changing
	* \param[in,out] node Node where resource will be in its next role
	* \param[in] optional Whether scheduled actions should be optional
	*/
	typedef void (rsc_transition_fn)(pcmk_resource_t rsc, pcmk_node_t *node,
	bool optional);

	static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the function needed to transition directly from one role
	* to another. NULL indicates that nothing is needed.
	*
	* Current role Transition function Next role
	* ------------ ------------------- ----------
	*/
	/* Unknown / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	assert_role_error, /* Started */
	assert_role_error, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Stopped / { assert_role_error, / Unknown */
	NULL, /* Stopped */
	start_resource, /* Started */
	start_resource, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Started / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	NULL, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Unpromoted / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	stop_resource, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Promoted / { assert_role_error, / Unknown */
	demote_resource, /* Stopped */
	demote_resource, /* Started */
	demote_resource, /* Unpromoted */
	NULL, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Get a list of a resource's allowed nodes sorted by node score
	*
	* \param[in] rsc Resource to check
	*
	* \return List of allowed nodes sorted by node score
	*/
	static GList *
	sorted_allowed_nodes(const pcmk_resource_t *rsc)
	{
	if (rsc->allowed_nodes != NULL) {
	GList *nodes = g_hash_table_get_values(rsc->allowed_nodes);

	if (nodes != NULL) {
	return pcmk__sort_nodes(nodes, pcmk__current_node(rsc));
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Assign a resource to its best allowed node, if possible
	*
	* \param[in,out] rsc Resource to choose a node for
	* \param[in] prefer If not \c NULL, prefer this node when all else
	* equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return true if \p rsc could be assigned to a node, otherwise false
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	static bool
	assign_best_node(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *nodes = NULL;
	pcmk_node_t *chosen = NULL;
	pcmk_node_t *best = NULL;
	const pcmk_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc);

	if (prefer == NULL) {
	prefer = most_free_node;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	// We've already finished assignment of resources to nodes
	return rsc->allocated_to != NULL;
	}

	// Sort allowed nodes by score
	nodes = sorted_allowed_nodes(rsc);
	if (nodes != NULL) {
	best = (pcmk_node_t *) nodes->data; // First node has best score
	}

	if ((prefer != NULL) && (nodes != NULL)) {
	// Get the allowed node version of prefer
	chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);

	if (chosen == NULL) {
	pcmk__rsc_trace(rsc, "Preferred node %s for %s was unknown",
	pcmk__node_name(prefer), rsc->id);

	/* Favor the preferred node as long as its score is at least as good as
	* the best allowed node's.
	*
	* An alternative would be to favor the preferred node even if the best
	* node is better, when the best node's score is less than INFINITY.
	*/
	} else if (chosen->weight < best->weight) {
	pcmk__rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
	pcmk__node_name(chosen), rsc->id);
	chosen = NULL;

	} else if (!pcmk__node_available(chosen, true, false)) {
	pcmk__rsc_trace(rsc, "Preferred node %s for %s was unavailable",
	pcmk__node_name(chosen), rsc->id);
	chosen = NULL;

	} else {
	pcmk__rsc_trace(rsc,
	"Chose preferred node %s for %s "
	"(ignoring %d candidates)",
	pcmk__node_name(chosen), rsc->id,
	g_list_length(nodes));
	}
	}

	if ((chosen == NULL) && (best != NULL)) {
	/* Either there is no preferred node, or the preferred node is not
	* suitable, but another node is allowed to run the resource.
	*/

	chosen = best;

	if (!pe_rsc_is_unique_clone(rsc->parent)
	&& (chosen->weight > 0) // Zero not acceptable
	&& pcmk__node_available(chosen, false, false)) {
	/* If the resource is already running on a node, prefer that node if
	* it is just as good as the chosen node.
	*
	* We don't do this for unique clone instances, because
	* pcmk__assign_instances() has already assigned instances to their
	* running nodes when appropriate, and if we get here, we don't want
	* remaining unassigned instances to prefer a node that's already
	* running another instance.
	*/
	pcmk_node_t *running = pcmk__current_node(rsc);

	if (running == NULL) {
	// Nothing to do

	} else if (!pcmk__node_available(running, true, false)) {
	pcmk__rsc_trace(rsc,
	"Current node for %s (%s) can't run resources",
	rsc->id, pcmk__node_name(running));

	} else {
	int nodes_with_best_score = 1;

	for (GList *iter = nodes->next; iter; iter = iter->next) {
	pcmk_node_t allowed = (pcmk_node_t ) iter->data;

	if (allowed->weight != chosen->weight) {
	// The nodes are sorted by score, so no more are equal
	break;
	}
	- if (pe__same_node(allowed, running)) {
	+ if (pcmk__same_node(allowed, running)) {
	// Scores are equal, so prefer the current node
	chosen = allowed;
	}
	nodes_with_best_score++;
	}

	if (nodes_with_best_score > 1) {
	uint8_t log_level = LOG_INFO;

	if (chosen->weight >= INFINITY) {
	log_level = LOG_WARNING;
	}
	do_crm_log(log_level,
	"Chose %s for %s from %d nodes with score %s",
	pcmk__node_name(chosen), rsc->id,
	nodes_with_best_score,
	pcmk_readable_score(chosen->weight));
	}
	}
	}

	pcmk__rsc_trace(rsc, "Chose %s for %s from %d candidates",
	pcmk__node_name(chosen), rsc->id, g_list_length(nodes));
	}

	pcmk__assign_resource(rsc, chosen, false, stop_if_fail);
	g_list_free(nodes);
	return rsc->allocated_to != NULL;
	}

	/*!
	* \internal
	* \brief Apply a "this with" colocation to a node's allowed node scores
	*
	* \param[in,out] colocation Colocation to apply
	* \param[in,out] rsc Resource being assigned
	*/
	static void
	apply_this_with(pcmk__colocation_t colocation, pcmk_resource_t rsc)
	{
	GHashTable *archive = NULL;
	pcmk_resource_t *other = colocation->primary;

	// In certain cases, we will need to revert the node scores
	if ((colocation->dependent_role >= pcmk_role_promoted)
	\|\| ((colocation->score < 0) && (colocation->score > -INFINITY))) {
	archive = pcmk__copy_node_table(rsc->allowed_nodes);
	}

	if (pcmk_is_set(other->flags, pcmk_rsc_unassigned)) {
	pcmk__rsc_trace(rsc,
	"%s: Assigning colocation %s primary %s first"
	"(score=%d role=%s)",
	rsc->id, colocation->id, other->id,
	colocation->score, role2text(colocation->dependent_role));
	other->cmds->assign(other, NULL, true);
	}

	// Apply the colocation score to this resource's allowed node scores
	rsc->cmds->apply_coloc_score(rsc, other, colocation, true);
	if ((archive != NULL)
	&& !pcmk__any_node_available(rsc->allowed_nodes)) {
	pcmk__rsc_info(rsc,
	"%s: Reverting scores from colocation with %s "
	"because no nodes allowed",
	rsc->id, other->id);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive != NULL) {
	g_hash_table_destroy(archive);
	}
	}

	/*!
	* \internal
	* \brief Update a Pacemaker Remote node once its connection has been assigned
	*
	* \param[in] connection Connection resource that has been assigned
	*/
	static void
	remote_connection_assigned(const pcmk_resource_t *connection)
	{
	pcmk_node_t *remote_node = pe_find_node(connection->cluster->nodes,
	connection->id);

	CRM_CHECK(remote_node != NULL, return);

	if ((connection->allocated_to != NULL)
	&& (connection->next_role != pcmk_role_stopped)) {

	crm_trace("Pacemaker Remote node %s will be online",
	remote_node->details->id);
	remote_node->details->online = TRUE;
	if (remote_node->details->unseen) {
	// Avoid unnecessary fence, since we will attempt connection
	remote_node->details->unclean = FALSE;
	}

	} else {
	crm_trace("Pacemaker Remote node %s will be shut down "
	"(%sassigned connection's next role is %s)",
	remote_node->details->id,
	((connection->allocated_to == NULL)? "un" : ""),
	role2text(connection->next_role));
	remote_node->details->shutdown = TRUE;
	}
	}

	/*!
	* \internal
	* \brief Assign a primitive resource to a node
	*
	* \param[in,out] rsc Resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pcmk_node_t *
	pcmk__primitive_assign(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *this_with_colocations = NULL;
	GList *with_this_colocations = NULL;
	GList *iter = NULL;
	pcmk__colocation_t *colocation = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	// Never assign a child without parent being assigned first
	if ((rsc->parent != NULL)
	&& !pcmk_is_set(rsc->parent->flags, pcmk_rsc_assigning)) {
	pcmk__rsc_debug(rsc, "%s: Assigning parent %s first",
	rsc->id, rsc->parent->id);
	rsc->parent->cmds->assign(rsc->parent, prefer, stop_if_fail);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	// Assignment has already been done
	const char *node_name = "no node";

	if (rsc->allocated_to != NULL) {
	node_name = pcmk__node_name(rsc->allocated_to);
	}
	pcmk__rsc_debug(rsc, "%s: pre-assigned to %s", rsc->id, node_name);
	return rsc->allocated_to;
	}

	// Ensure we detect assignment loops
	if (pcmk_is_set(rsc->flags, pcmk_rsc_assigning)) {
	pcmk__rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
	return NULL;
	}
	pcmk__set_rsc_flags(rsc, pcmk_rsc_assigning);

	pe__show_node_scores(true, rsc, "Pre-assignment", rsc->allowed_nodes,
	rsc->cluster);

	this_with_colocations = pcmk__this_with_colocations(rsc);
	with_this_colocations = pcmk__with_this_colocations(rsc);

	// Apply mandatory colocations first, to satisfy as many as possible
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -CRM_SCORE_INFINITY)
	\|\| (colocation->score >= CRM_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -CRM_SCORE_INFINITY)
	\|\| (colocation->score >= CRM_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	pe__show_node_scores(true, rsc, "Mandatory-colocations",
	rsc->allowed_nodes, rsc->cluster);

	// Then apply optional colocations
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -CRM_SCORE_INFINITY)
	&& (colocation->score < CRM_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -CRM_SCORE_INFINITY)
	&& (colocation->score < CRM_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	g_list_free(this_with_colocations);
	g_list_free(with_this_colocations);

	if (rsc->next_role == pcmk_role_stopped) {
	pcmk__rsc_trace(rsc,
	"Banning %s from all nodes because it will be stopped",
	rsc->id);
	resource_location(rsc, NULL, -INFINITY, PCMK_META_TARGET_ROLE,
	rsc->cluster);

	} else if ((rsc->next_role > rsc->role)
	&& !pcmk_is_set(rsc->cluster->flags, pcmk_sched_quorate)
	&& (rsc->cluster->no_quorum_policy == pcmk_no_quorum_freeze)) {
	crm_notice("Resource %s cannot be elevated from %s to %s due to "
	PCMK_OPT_NO_QUORUM_POLICY "=freeze",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role));
	pe__set_next_role(rsc, rsc->role, PCMK_OPT_NO_QUORUM_POLICY "=freeze");
	}

	pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->allowed_nodes, rsc->cluster);

	// Unmanage resource if fencing is enabled but no device is configured
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(rsc->cluster->flags, pcmk_sched_have_fencing)) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	// Unmanaged resources stay on their current node
	const char *reason = NULL;
	pcmk_node_t *assign_to = NULL;

	pe__set_next_role(rsc, rsc->role, "unmanaged");
	assign_to = pcmk__current_node(rsc);
	if (assign_to == NULL) {
	reason = "inactive";
	} else if (rsc->role == pcmk_role_promoted) {
	reason = "promoted";
	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	reason = "failed";
	} else {
	reason = "active";
	}
	pcmk__rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id,
	(assign_to? assign_to->details->uname : "no node"),
	reason);
	pcmk__assign_resource(rsc, assign_to, true, stop_if_fail);

	} else if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_stop_all)) {
	// Must stop at some point, but be consistent with stop_if_fail
	if (stop_if_fail) {
	pcmk__rsc_debug(rsc,
	"Forcing %s to stop: " PCMK_OPT_STOP_ALL_RESOURCES,
	rsc->id);
	}
	pcmk__assign_resource(rsc, NULL, true, stop_if_fail);

	} else if (!assign_best_node(rsc, prefer, stop_if_fail)) {
	// Assignment failed
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	pcmk__rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
	} else if ((rsc->running_on != NULL) && stop_if_fail) {
	pcmk__rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
	}
	}

	pcmk__clear_rsc_flags(rsc, pcmk_rsc_assigning);

	if (rsc->is_remote_node) {
	remote_connection_assigned(rsc);
	}

	return rsc->allocated_to;
	}

	/*!
	* \internal
	* \brief Schedule actions to bring resource down and back to current role
	*
	* \param[in,out] rsc Resource to restart
	* \param[in,out] current Node that resource should be brought down on
	* \param[in] need_stop Whether the resource must be stopped
	* \param[in] need_promote Whether the resource must be promoted
	*
	* \return Role that resource would have after scheduled actions are taken
	*/
	static void
	schedule_restart_actions(pcmk_resource_t rsc, pcmk_node_t current,
	bool need_stop, bool need_promote)
	{
	enum rsc_role_e role = rsc->role;
	enum rsc_role_e next_role;
	rsc_transition_fn fn = NULL;

	pcmk__set_rsc_flags(rsc, pcmk_rsc_restarting);

	// Bring resource down to a stop on its current node
	while (role != pcmk_role_stopped) {
	next_role = rsc_state_matrix[role][pcmk_role_stopped];
	pcmk__rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
	(need_stop? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, current, !need_stop);
	role = next_role;
	}

	// Bring resource up to its next role on its next node
	while ((rsc->role <= rsc->next_role) && (role != rsc->role)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	bool required = need_stop;

	next_role = rsc_state_matrix[role][rsc->role];
	if ((next_role == pcmk_role_promoted) && need_promote) {
	required = true;
	}
	pcmk__rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
	(required? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->allocated_to, !required);
	role = next_role;
	}

	pcmk__clear_rsc_flags(rsc, pcmk_rsc_restarting);
	}

	/*!
	* \internal
	* \brief If a resource's next role is not explicitly specified, set a default
	*
	* \param[in,out] rsc Resource to set next role for
	*
	* \return "explicit" if next role was explicitly set, otherwise "implicit"
	*/
	static const char *
	set_default_next_role(pcmk_resource_t *rsc)
	{
	if (rsc->next_role != pcmk_role_unknown) {
	return "explicit";
	}

	if (rsc->allocated_to == NULL) {
	pe__set_next_role(rsc, pcmk_role_stopped, "assignment");
	} else {
	pe__set_next_role(rsc, pcmk_role_started, "assignment");
	}
	return "implicit";
	}

	/*!
	* \internal
	* \brief Create an action to represent an already pending start
	*
	* \param[in,out] rsc Resource to create start action for
	*/
	static void
	create_pending_start(pcmk_resource_t *rsc)
	{
	pcmk_action_t *start = NULL;

	pcmk__rsc_trace(rsc,
	"Creating action for %s to represent already pending start",
	rsc->id);
	start = start_action(rsc, rsc->allocated_to, TRUE);
	pcmk__set_action_flags(start, pcmk_action_always_in_graph);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to take a resource to its next role
	*
	* \param[in,out] rsc Resource to schedule actions for
	*/
	static void
	schedule_role_transition_actions(pcmk_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	while (role != rsc->next_role) {
	enum rsc_role_e next_role = rsc_state_matrix[role][rsc->next_role];
	rsc_transition_fn fn = NULL;

	pcmk__rsc_trace(rsc,
	"Creating action to take %s from %s to %s "
	"(ending at %s)",
	rsc->id, role2text(role), role2text(next_role),
	role2text(rsc->next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->allocated_to, false);
	role = next_role;
	}
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given primitive resource
	*
	* \param[in,out] rsc Primitive resource to create actions for
	*/
	void
	pcmk__primitive_create_actions(pcmk_resource_t *rsc)
	{
	bool need_stop = false;
	bool need_promote = false;
	bool is_moving = false;
	bool allow_migrate = false;
	bool multiply_active = false;

	pcmk_node_t *current = NULL;
	unsigned int num_all_active = 0;
	unsigned int num_clean_active = 0;
	const char *next_role_source = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	next_role_source = set_default_next_role(rsc);
	pcmk__rsc_trace(rsc,
	"Creating all actions for %s transition from %s to %s "
	"(%s) on %s",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role),
	next_role_source, pcmk__node_name(rsc->allocated_to));

	current = rsc->fns->active_node(rsc, &num_all_active, &num_clean_active);

	g_list_foreach(rsc->dangling_migrations, pcmk__abort_dangling_migration,
	rsc);

	if ((current != NULL) && (rsc->allocated_to != NULL)
	- && !pe__same_node(current, rsc->allocated_to)
	+ && !pcmk__same_node(current, rsc->allocated_to)
	&& (rsc->next_role >= pcmk_role_started)) {

	pcmk__rsc_trace(rsc, "Moving %s from %s to %s",
	rsc->id, pcmk__node_name(current),
	pcmk__node_name(rsc->allocated_to));
	is_moving = true;
	allow_migrate = pcmk__rsc_can_migrate(rsc, current);

	// This is needed even if migrating (though I'm not sure why ...)
	need_stop = true;
	}

	// Check whether resource is partially migrated and/or multiply active
	if ((rsc->partial_migration_source != NULL)
	&& (rsc->partial_migration_target != NULL)
	&& allow_migrate && (num_all_active == 2)
	- && pe__same_node(current, rsc->partial_migration_source)
	- && pe__same_node(rsc->allocated_to, rsc->partial_migration_target)) {
	+ && pcmk__same_node(current, rsc->partial_migration_source)
	+ && pcmk__same_node(rsc->allocated_to, rsc->partial_migration_target)) {
	/* A partial migration is in progress, and the migration target remains
	* the same as when the migration began.
	*/
	pcmk__rsc_trace(rsc,
	"Partial migration of %s from %s to %s will continue",
	rsc->id, pcmk__node_name(rsc->partial_migration_source),
	pcmk__node_name(rsc->partial_migration_target));

	} else if ((rsc->partial_migration_source != NULL)
	\|\| (rsc->partial_migration_target != NULL)) {
	// A partial migration is in progress but can't be continued

	if (num_all_active > 2) {
	// The resource is migrating and multiply active!
	crm_notice("Forcing recovery of %s because it is migrating "
	"from %s to %s and possibly active elsewhere",
	rsc->id, pcmk__node_name(rsc->partial_migration_source),
	pcmk__node_name(rsc->partial_migration_target));
	} else {
	// The migration source or target isn't available
	crm_notice("Forcing recovery of %s because it can no longer "
	"migrate from %s to %s",
	rsc->id, pcmk__node_name(rsc->partial_migration_source),
	pcmk__node_name(rsc->partial_migration_target));
	}
	need_stop = true;
	rsc->partial_migration_source = rsc->partial_migration_target = NULL;
	allow_migrate = false;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	multiply_active = (num_all_active > 1);
	} else {
	/* If a resource has "requires" set to nothing or quorum, don't consider
	* it active on unclean nodes (similar to how all resources behave when
	* PCMK_OPT_STONITH_ENABLED is false). We can start such resources
	* elsewhere before fencing completes, and if we considered the resource
	* active on the failed node, we would attempt recovery for being active
	* on multiple nodes.
	*/
	multiply_active = (num_clean_active > 1);
	}

	if (multiply_active) {
	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);

	// Resource was (possibly) incorrectly multiply active
	pcmk__sched_err("%s resource %s might be active on %u nodes (%s)",
	pcmk__s(class, "Untyped"), rsc->id, num_all_active,
	pcmk_multiply_active_text(rsc->recovery_type));
	crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ"
	"#Resource_is_Too_Active for more information");

	switch (rsc->recovery_type) {
	case pcmk_multiply_active_restart:
	need_stop = true;
	break;
	case pcmk_multiply_active_unexpected:
	need_stop = true; // stop_resource() will skip expected node
	pcmk__set_rsc_flags(rsc, pcmk_rsc_stop_unexpected);
	break;
	default:
	break;
	}

	} else {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_stop_unexpected);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_start_pending)) {
	create_pending_start(rsc);
	}

	if (is_moving) {
	// Remaining tests are only for resources staying where they are

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_stop_if_failed)) {
	need_stop = true;
	pcmk__rsc_trace(rsc, "Recovering %s", rsc->id);
	} else {
	pcmk__rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
	if (rsc->next_role == pcmk_role_promoted) {
	need_promote = true;
	}
	}

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	pcmk__rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
	need_stop = true;

	} else if ((rsc->role > pcmk_role_started) && (current != NULL)
	&& (rsc->allocated_to != NULL)) {
	pcmk_action_t *start = NULL;

	pcmk__rsc_trace(rsc, "Creating start action for promoted resource %s",
	rsc->id);
	start = start_action(rsc, rsc->allocated_to, TRUE);
	if (!pcmk_is_set(start->flags, pcmk_action_optional)) {
	// Recovery of a promoted resource
	pcmk__rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
	need_stop = true;
	}
	}

	// Create any actions needed to bring resource down and back up to same role
	schedule_restart_actions(rsc, current, need_stop, need_promote);

	// Create any actions needed to take resource from this role to the next
	schedule_role_transition_actions(rsc);

	pcmk__create_recurring_actions(rsc);

	if (allow_migrate) {
	pcmk__create_migration_actions(rsc, current);
	}
	}

	/*!
	* \internal
	* \brief Ban a resource from any allowed nodes that are Pacemaker Remote nodes
	*
	* \param[in] rsc Resource to check
	*/
	static void
	rsc_avoids_remote_nodes(const pcmk_resource_t *rsc)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (node->details->remote_rsc != NULL) {
	node->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Return allowed nodes as (possibly sorted) list
	*
	* Convert a resource's hash table of allowed nodes to a list. If printing to
	* stdout, sort the list, to keep action ID numbers consistent for regression
	* test output (while avoiding the performance hit on a live cluster).
	*
	* \param[in] rsc Resource to check for allowed nodes
	*
	* \return List of resource's allowed nodes
	* \note Callers should take care not to rely on the list being sorted.
	*/
	static GList *
	allowed_nodes_as_list(const pcmk_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;

	if (rsc->allowed_nodes) {
	allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	if (!pcmk__is_daemon) {
	allowed_nodes = g_list_sort(allowed_nodes, pe__cmp_node_name);
	}

	return allowed_nodes;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a primitive resource
	*
	* \param[in,out] rsc Primitive resource to create implicit constraints for
	*/
	void
	pcmk__primitive_internal_constraints(pcmk_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;
	bool check_unfencing = false;
	bool check_utilization = false;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__rsc_trace(rsc,
	"Skipping implicit constraints for unmanaged resource "
	"%s", rsc->id);
	return;
	}

	// Whether resource requires unfencing
	check_unfencing = !pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)
	&& pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_enable_unfencing)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_needs_unfencing);

	// Whether a non-default placement strategy is used
	check_utilization = (g_hash_table_size(rsc->utilization) > 0)
	&& !pcmk__str_eq(rsc->cluster->placement_strategy,
	"default", pcmk__str_casei);

	// Order stops before starts (i.e. restart)
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0), NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0), NULL,
	pcmk__ar_ordered
	\|pcmk__ar_first_implies_then
	\|pcmk__ar_intermediate_stop,
	rsc->cluster);

	// Promotable ordering: demote before stop, start before promote
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)
	\|\| (rsc->role > pcmk_role_unpromoted)) {

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
	NULL,
	pcmk__ar_promoted_then_implies_first, rsc->cluster);

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0),
	NULL,
	pcmk__ar_unrunnable_first_blocks, rsc->cluster);
	}

	// Don't clear resource history if probing on same node
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	pcmk__ar_if_on_same_node\|pcmk__ar_then_cancels_first,
	rsc->cluster);

	// Certain checks need allowed nodes
	if (check_unfencing \|\| check_utilization \|\| (rsc->container != NULL)) {
	allowed_nodes = allowed_nodes_as_list(rsc);
	}

	if (check_unfencing) {
	g_list_foreach(allowed_nodes, pcmk__order_restart_vs_unfence, rsc);
	}

	if (check_utilization) {
	pcmk__create_utilization_constraints(rsc, allowed_nodes);
	}

	if (rsc->container != NULL) {
	pcmk_resource_t *remote_rsc = NULL;

	if (rsc->is_remote_node) {
	// rsc is the implicit remote connection for a guest or bundle node

	/* Guest resources are not allowed to run on Pacemaker Remote nodes,
	* to avoid nesting remotes. However, bundles are allowed.
	*/
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_remote_nesting_allowed)) {
	rsc_avoids_remote_nodes(rsc->container);
	}

	/* If someone cleans up a guest or bundle node's container, we will
	* likely schedule a (re-)probe of the container and recovery of the
	* connection. Order the connection stop after the container probe,
	* so that if we detect the container running, we will trigger a new
	* transition and avoid the unnecessary recovery.
	*/
	pcmk__order_resource_actions(rsc->container, PCMK_ACTION_MONITOR,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);

	/* A user can specify that a resource must start on a Pacemaker Remote
	* node by explicitly configuring it with the container=NODENAME
	* meta-attribute. This is of questionable merit, since location
	* constraints can accomplish the same thing. But we support it, so here
	* we check whether a resource (that is not itself a remote connection)
	* has container set to a remote node or guest node resource.
	*/
	} else if (rsc->container->is_remote_node) {
	remote_rsc = rsc->container;
	} else {
	remote_rsc = pe__resource_contains_guest_node(rsc->cluster,
	rsc->container);
	}

	if (remote_rsc != NULL) {
	/* Force the resource on the Pacemaker Remote node instead of
	* colocating the resource with the container resource.
	*/
	for (GList *item = allowed_nodes; item; item = item->next) {
	pcmk_node_t *node = item->data;

	if (node->details->remote_rsc != remote_rsc) {
	node->weight = -INFINITY;
	}
	}

	} else {
	/* This resource is either a filler for a container that does NOT
	* represent a Pacemaker Remote node, or a Pacemaker Remote
	* connection resource for a guest node or bundle.
	*/
	int score;

	crm_trace("Order and colocate %s relative to its container %s",
	rsc->id, rsc->container->id);

	pcmk__new_ordering(rsc->container,
	pcmk__op_key(rsc->container->id,
	PCMK_ACTION_START, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_first_implies_then
	\|pcmk__ar_unrunnable_first_blocks,
	rsc->cluster);

	pcmk__new_ordering(rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
	NULL,
	rsc->container,
	pcmk__op_key(rsc->container->id,
	PCMK_ACTION_STOP, 0),
	NULL, pcmk__ar_then_implies_first, rsc->cluster);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_remote_nesting_allowed)) {
	score = 10000; /* Highly preferred but not essential */
	} else {
	score = INFINITY; /* Force them to run on the same host */
	}
	pcmk__new_colocation("#resource-with-container", NULL, score, rsc,
	rsc->container, NULL, NULL,
	pcmk__coloc_influence);
	}
	}

	if (rsc->is_remote_node
	\|\| pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	/* Remote connections and fencing devices are not allowed to run on
	* Pacemaker Remote nodes
	*/
	rsc_avoids_remote_nodes(rsc);
	}
	g_list_free(allowed_nodes);
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__primitive_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	enum pcmk__coloc_affects filter_results;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (for_dependent) {
	// Always process on behalf of primary resource
	primary->cmds->apply_coloc_score(dependent, primary, colocation, false);
	return;
	}

	filter_results = pcmk__colocation_affects(dependent, primary, colocation,
	false);
	pcmk__rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
	((colocation->score > 0)? "Colocating" : "Anti-colocating"),
	dependent->id, primary->id, colocation->id,
	colocation->score,
	filter_results);

	switch (filter_results) {
	case pcmk__coloc_affects_role:
	pcmk__apply_coloc_to_priority(dependent, primary, colocation);
	break;
	case pcmk__coloc_affects_location:
	pcmk__apply_coloc_to_scores(dependent, primary, colocation);
	break;
	default: // pcmk__coloc_affects_nothing
	return;
	}
	}

	/* Primitive implementation of
	* pcmk_assignment_methods_t:with_this_colocations()
	*/
	void
	pcmk__with_primitive_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (list != NULL));

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);
	if (rsc->parent != NULL) {
	rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc, list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->rsc_cons_lhs; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_with_this(list, colocation, orig_rsc);
	}
	}
	}
	}

	/* Primitive implementation of
	* pcmk_assignment_methods_t:this_with_colocations()
	*/
	void
	pcmk__primitive_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (list != NULL));

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);
	if (rsc->parent != NULL) {
	rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc, list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->rsc_cons; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_this_with(list, colocation, orig_rsc);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given primitive resource action
	*
	* \param[in,out] action Action to get flags for
	* \param[in] node If not NULL, limit effects to this node (ignored)
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__primitive_action_flags(pcmk_action_t action, const pcmk_node_t node)
	{
	CRM_ASSERT(action != NULL);
	return (uint32_t) action->flags;
	}

	/*!
	* \internal
	* \brief Check whether a node is a multiply active resource's expected node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return \c true if \p rsc is multiply active with
	* \c PCMK_META_MULTIPLE_ACTIVE set to \c stop_unexpected, and \p node
	* is the node where it will remain active
	* \note This assumes that the resource's next role cannot be changed to stopped
	* after this is called, which should be reasonable if status has already
	* been unpacked and resources have been assigned to nodes.
	*/
	static bool
	is_expected_node(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	return pcmk_all_flags_set(rsc->flags,
	pcmk_rsc_stop_unexpected\|pcmk_rsc_restarting)
	&& (rsc->next_role > pcmk_role_stopped)
	- && pe__same_node(rsc->allocated_to, node);
	+ && pcmk__same_node(rsc->allocated_to, node);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to stop a resource wherever it is active
	*
	* \param[in,out] rsc Resource being stopped
	* \param[in] node Node where resource is being stopped (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	stop_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t current = (pcmk_node_t ) iter->data;
	pcmk_action_t *stop = NULL;

	if (is_expected_node(rsc, current)) {
	/* We are scheduling restart actions for a multiply active resource
	* with PCMK_META_MULTIPLE_ACTIVE=stop_unexpected, and this is where
	* it should not be stopped.
	*/
	pcmk__rsc_trace(rsc,
	"Skipping stop of multiply active resource %s "
	"on expected node %s",
	rsc->id, pcmk__node_name(current));
	continue;
	}

	if (rsc->partial_migration_target != NULL) {
	// Continue migration if node originally was and remains target
	- if (pe__same_node(current, rsc->partial_migration_target)
	- && pe__same_node(current, rsc->allocated_to)) {
	+ if (pcmk__same_node(current, rsc->partial_migration_target)
	+ && pcmk__same_node(current, rsc->allocated_to)) {
	pcmk__rsc_trace(rsc,
	"Skipping stop of %s on %s "
	"because partial migration there will continue",
	rsc->id, pcmk__node_name(current));
	continue;
	} else {
	pcmk__rsc_trace(rsc,
	"Forcing stop of %s on %s "
	"because migration target changed",
	rsc->id, pcmk__node_name(current));
	optional = false;
	}
	}

	pcmk__rsc_trace(rsc, "Scheduling stop of %s on %s",
	rsc->id, pcmk__node_name(current));
	stop = stop_action(rsc, current, optional);

	if (rsc->allocated_to == NULL) {
	pe_action_set_reason(stop, "node availability", true);
	} else if (pcmk_all_flags_set(rsc->flags, pcmk_rsc_restarting
	\|pcmk_rsc_stop_unexpected)) {
	/* We are stopping a multiply active resource on a node that is
	* not its expected node, and we are still scheduling restart
	* actions, so the stop is for being multiply active.
	*/
	pe_action_set_reason(stop, "being multiply active", true);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__clear_action_flags(stop, pcmk_action_runnable);
	}

	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_remove_after_stop)) {
	pcmk__schedule_cleanup(rsc, current, optional);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_unfencing)) {
	pcmk_action_t *unfence = pe_fence_op(current, PCMK_ACTION_ON, true,
	NULL, false, rsc->cluster);

	order_actions(stop, unfence, pcmk__ar_then_implies_first);
	if (!pcmk__node_unfenced(current)) {
	pcmk__sched_err("Stopping %s until %s can be unfenced",
	rsc->id, pcmk__node_name(current));
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to start a resource on a node
	*
	* \param[in,out] rsc Resource being started
	* \param[in,out] node Node where resource should be started
	* \param[in] optional Whether actions should be optional
	*/
	static void
	start_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	pcmk_action_t *start = NULL;

	CRM_ASSERT(node != NULL);

	pcmk__rsc_trace(rsc, "Scheduling %s start of %s on %s (score %d)",
	(optional? "optional" : "required"), rsc->id,
	pcmk__node_name(node), node->weight);
	start = start_action(rsc, node, TRUE);

	pcmk__order_vs_unfence(rsc, node, start, pcmk__ar_first_implies_then);

	if (pcmk_is_set(start->flags, pcmk_action_runnable) && !optional) {
	pcmk__clear_action_flags(start, pcmk_action_optional);
	}

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the start becomes necessary for other
	* reasons later.
	*/
	pcmk__rsc_trace(rsc,
	"Start of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pcmk__node_name(node));
	pcmk__set_action_flags(start, pcmk_action_pseudo);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to promote a resource on a node
	*
	* \param[in,out] rsc Resource being promoted
	* \param[in] node Node where resource should be promoted
	* \param[in] optional Whether actions should be optional
	*/
	static void
	promote_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	GList *iter = NULL;
	GList *action_list = NULL;
	bool runnable = true;

	CRM_ASSERT(node != NULL);

	// Any start must be runnable for promotion to be runnable
	action_list = pe__resource_actions(rsc, node, PCMK_ACTION_START, true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pcmk_action_t start = (pcmk_action_t ) iter->data;

	if (!pcmk_is_set(start->flags, pcmk_action_runnable)) {
	runnable = false;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	pcmk_action_t *promote = promote_action(rsc, node, optional);

	pcmk__rsc_trace(rsc, "Scheduling %s promotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pcmk__node_name(node));

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the promote becomes necessary for
	* other reasons later.
	*/
	pcmk__rsc_trace(rsc,
	"Promotion of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pcmk__node_name(node));
	pcmk__set_action_flags(promote, pcmk_action_pseudo);
	}
	} else {
	pcmk__rsc_trace(rsc, "Not promoting %s on %s: start unrunnable",
	rsc->id, pcmk__node_name(node));
	action_list = pe__resource_actions(rsc, node, PCMK_ACTION_PROMOTE,
	true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pcmk_action_t promote = (pcmk_action_t ) iter->data;

	pcmk__clear_action_flags(promote, pcmk_action_runnable);
	}
	g_list_free(action_list);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to demote a resource wherever it is active
	*
	* \param[in,out] rsc Resource being demoted
	* \param[in] node Node where resource should be demoted (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	demote_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	/* Since this will only be called for a primitive (possibly as an instance
	* of a collective resource), the resource is multiply active if it is
	* running on more than one node, so we want to demote on all of them as
	* part of recovery, regardless of which one is the desired node.
	*/
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t current = (pcmk_node_t ) iter->data;

	if (is_expected_node(rsc, current)) {
	pcmk__rsc_trace(rsc,
	"Skipping demote of multiply active resource %s "
	"on expected node %s",
	rsc->id, pcmk__node_name(current));
	} else {
	pcmk__rsc_trace(rsc, "Scheduling %s demotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pcmk__node_name(current));
	demote_action(rsc, current, optional);
	}
	}
	}

	static void
	assert_role_error(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	CRM_ASSERT(false);
	}

	/*!
	* \internal
	* \brief Schedule cleanup of a resource
	*
	* \param[in,out] rsc Resource to clean up
	* \param[in] node Node to clean up on
	* \param[in] optional Whether clean-up should be optional
	*/
	void
	pcmk__schedule_cleanup(pcmk_resource_t rsc, const pcmk_node_t node,
	bool optional)
	{
	/* If the cleanup is required, its orderings are optional, because they're
	* relevant only if both actions are required. Conversely, if the cleanup is
	* optional, the orderings make the then action required if the first action
	* becomes required.
	*/
	uint32_t flag = optional? pcmk__ar_first_implies_then : pcmk__ar_ordered;

	CRM_CHECK((rsc != NULL) && (node != NULL), return);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: resource failed",
	rsc->id, pcmk__node_name(node));
	return;
	}

	if (node->details->unclean \|\| !node->details->online) {
	pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: node unavailable",
	rsc->id, pcmk__node_name(node));
	return;
	}

	crm_notice("Scheduling clean-up of %s on %s",
	rsc->id, pcmk__node_name(node));
	delete_action(rsc, node, optional);

	// stop -> clean-up -> start
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_DELETE, flag);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DELETE,
	rsc, PCMK_ACTION_START, flag);
	}

	/*!
	* \internal
	* \brief Add primitive meta-attributes relevant to graph actions to XML
	*
	* \param[in] rsc Primitive resource whose meta-attributes should be added
	* \param[in,out] xml Transition graph action attributes XML to add to
	*/
	void
	pcmk__primitive_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml)
	{
	char *name = NULL;
	char *value = NULL;
	const pcmk_resource_t *parent = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (xml != NULL));

	/* Clone instance numbers get set internally as meta-attributes, and are
	* needed in the transition graph (for example, to tell unique clone
	* instances apart).
	*/
	value = g_hash_table_lookup(rsc->meta, PCMK__META_CLONE_INSTANCE_NUM);
	if (value != NULL) {
	name = crm_meta_name(PCMK__META_CLONE_INSTANCE_NUM);
	crm_xml_add(xml, name, value);
	free(name);
	}

	// Not sure if this one is really needed ...
	value = g_hash_table_lookup(rsc->meta, PCMK_META_REMOTE_NODE);
	if (value != NULL) {
	name = crm_meta_name(PCMK_META_REMOTE_NODE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	/* The container meta-attribute can be set on the primitive itself or one of
	* its parents (for example, a group inside a container resource), so check
	* them all, and keep the highest one found.
	*/
	for (parent = rsc; parent != NULL; parent = parent->parent) {
	if (parent->container != NULL) {
	crm_xml_add(xml, CRM_META "_" PCMK__META_CONTAINER,
	parent->container->id);
	}
	}

	/* Bundle replica children will get their external-ip set internally as a
	* meta-attribute. The graph action needs it, but under a different naming
	* convention than other meta-attributes.
	*/
	value = g_hash_table_lookup(rsc->meta, "external-ip");
	if (value != NULL) {
	crm_xml_add(xml, "pcmk_external_ip", value);
	}
	}

	// Primitive implementation of pcmk_assignment_methods_t:add_utilization()
	void
	pcmk__primitive_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList all_rscs, GHashTable utilization)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (orig_rsc != NULL) && (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return;
	}

	pcmk__rsc_trace(orig_rsc,
	"%s: Adding primitive %s as colocated utilization",
	orig_rsc->id, rsc->id);
	pcmk__release_node_capacity(utilization, rsc);
	}

	/*!
	* \internal
	* \brief Get epoch time of node's shutdown attribute (or now if none)
	*
	* \param[in,out] node Node to check
	*
	* \return Epoch time corresponding to shutdown attribute if set or now if not
	*/
	static time_t
	shutdown_time(pcmk_node_t *node)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
	time_t result = 0;

	if (shutdown != NULL) {
	long long result_ll;

	if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
	result = (time_t) result_ll;
	}
	}
	return (result == 0)? get_effective_time(node->details->data_set) : result;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it's not locked to the node
	*
	* \param[in] data Node to check
	* \param[in,out] user_data Resource to check
	*/
	static void
	ban_if_not_locked(gpointer data, gpointer user_data)
	{
	const pcmk_node_t node = (const pcmk_node_t ) data;
	pcmk_resource_t rsc = (pcmk_resource_t ) user_data;

	if (strcmp(node->details->uname, rsc->lock_node->details->uname) != 0) {
	resource_location(rsc, node, -CRM_SCORE_INFINITY,
	PCMK_OPT_SHUTDOWN_LOCK, rsc->cluster);
	}
	}

	// Primitive implementation of pcmk_assignment_methods_t:shutdown_lock()
	void
	pcmk__primitive_shutdown_lock(pcmk_resource_t *rsc)
	{
	const char *class = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	class = crm_element_value(rsc->xml, PCMK_XA_CLASS);

	// Fence devices and remote connections can't be locked
	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
	\|\| pe__resource_is_remote_conn(rsc)) {
	return;
	}

	if (rsc->lock_node != NULL) {
	// The lock was obtained from resource history

	if (rsc->running_on != NULL) {
	/* The resource was started elsewhere even though it is now
	* considered locked. This shouldn't be possible, but as a
	* failsafe, we don't want to disturb the resource now.
	*/
	pcmk__rsc_info(rsc,
	"Cancelling shutdown lock "
	"because %s is already active", rsc->id);
	pe__clear_resource_history(rsc, rsc->lock_node);
	rsc->lock_node = NULL;
	rsc->lock_time = 0;
	}

	// Only a resource active on exactly one node can be locked
	} else if (pcmk__list_of_1(rsc->running_on)) {
	pcmk_node_t *node = rsc->running_on->data;

	if (node->details->shutdown) {
	if (node->details->unclean) {
	pcmk__rsc_debug(rsc,
	"Not locking %s to unclean %s for shutdown",
	rsc->id, pcmk__node_name(node));
	} else {
	rsc->lock_node = node;
	rsc->lock_time = shutdown_time(node);
	}
	}
	}

	if (rsc->lock_node == NULL) {
	// No lock needed
	return;
	}

	if (rsc->cluster->shutdown_lock > 0) {
	time_t lock_expiration = rsc->lock_time + rsc->cluster->shutdown_lock;

	pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
	rsc->id, pcmk__node_name(rsc->lock_node),
	(long long) lock_expiration);
	pe__update_recheck_time(++lock_expiration, rsc->cluster,
	"shutdown lock expiration");
	} else {
	pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown",
	rsc->id, pcmk__node_name(rsc->lock_node));
	}

	// If resource is locked to one node, ban it from all other nodes
	g_list_foreach(rsc->cluster->nodes, ban_if_not_locked, rsc);
	}
	diff --git a/lib/pacemaker/pcmk_sched_probes.c b/lib/pacemaker/pcmk_sched_probes.c
	index 0de1dfb073..69d20aa832 100644
	--- a/lib/pacemaker/pcmk_sched_probes.c
	+++ b/lib/pacemaker/pcmk_sched_probes.c
	@@ -1,905 +1,905 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add the expected result to a newly created probe
	*
	* \param[in,out] probe Probe action to add expected result to
	* \param[in] rsc Resource that probe is for
	* \param[in] node Node that probe will run on
	*/
	static void
	add_expected_result(pcmk_action_t probe, const pcmk_resource_t rsc,
	const pcmk_node_t *node)
	{
	// Check whether resource is currently active on node
	pcmk_node_t *running = pe_find_node_id(rsc->running_on, node->details->id);

	// The expected result is what we think the resource's current state is
	if (running == NULL) {
	pe__add_action_expected_result(probe, CRM_EX_NOT_RUNNING);

	} else if (rsc->role == pcmk_role_promoted) {
	pe__add_action_expected_result(probe, CRM_EX_PROMOTED);
	}
	}

	/*!
	* \internal
	* \brief Create any needed robes on a node for a list of resources
	*
	* \param[in,out] rscs List of resources to create probes for
	* \param[in,out] node Node to create probes on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_resource_list(GList rscs, pcmk_node_t node)
	{
	bool any_created = false;

	for (GList *iter = rscs; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->cmds->create_probe(rsc, node)) {
	any_created = true;
	}
	}
	return any_created;
	}

	/*!
	* \internal
	* \brief Order one resource's start after another's start-up probe
	*
	* \param[in,out] rsc1 Resource that might get start-up probe
	* \param[in] rsc2 Resource that might be started
	*/
	static void
	probe_then_start(pcmk_resource_t rsc1, pcmk_resource_t rsc2)
	{
	if ((rsc1->allocated_to != NULL)
	&& (g_hash_table_lookup(rsc1->known_on,
	rsc1->allocated_to->details->id) == NULL)) {

	pcmk__new_ordering(rsc1,
	pcmk__op_key(rsc1->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	rsc2, pcmk__op_key(rsc2->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_ordered, rsc1->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether a guest resource will stop
	*
	* \param[in] node Guest node to check
	*
	* \return true if guest resource will likely stop, otherwise false
	*/
	static bool
	guest_resource_will_stop(const pcmk_node_t *node)
	{
	const pcmk_resource_t *guest_rsc = node->details->remote_rsc->container;

	/* Ideally, we'd check whether the guest has a required stop, but that
	* information doesn't exist yet, so approximate it ...
	*/
	return node->details->remote_requires_reset
	\|\| node->details->unclean
	\|\| pcmk_is_set(guest_rsc->flags, pcmk_rsc_failed)
	\|\| (guest_rsc->next_role == pcmk_role_stopped)

	// Guest is moving
	\|\| ((guest_rsc->role > pcmk_role_stopped)
	&& (guest_rsc->allocated_to != NULL)
	&& (pe_find_node(guest_rsc->running_on,
	guest_rsc->allocated_to->details->uname) == NULL));
	}

	/*!
	* \internal
	* \brief Create a probe action for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return Newly created probe action
	*/
	static pcmk_action_t *
	probe_action(pcmk_resource_t rsc, pcmk_node_t node)
	{
	pcmk_action_t *probe = NULL;
	char *key = pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0);

	crm_debug("Scheduling probe of %s %s on %s",
	role2text(rsc->role), rsc->id, pcmk__node_name(node));

	probe = custom_action(rsc, key, PCMK_ACTION_MONITOR, node, FALSE,
	rsc->cluster);
	pcmk__clear_action_flags(probe, pcmk_action_optional);

	pcmk__order_vs_unfence(rsc, node, probe, pcmk__ar_ordered);
	add_expected_result(probe, rsc, node);
	return probe;
	}

	/*!
	* \internal
	* \brief Create probes for a resource on a node, if needed
	*
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_rsc_on_node(pcmk_resource_t rsc, pcmk_node_t node)
	{
	uint32_t flags = pcmk__ar_ordered;
	pcmk_action_t *probe = NULL;
	pcmk_node_t *allowed = NULL;
	pcmk_resource_t *top = uber_parent(rsc);
	const char *reason = NULL;

	CRM_ASSERT((rsc != NULL) && (node != NULL));

	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_probe_resources)) {
	reason = "start-up probes are disabled";
	goto no_probe;
	}

	if (pe__is_guest_or_remote_node(node)) {
	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_none)) {
	reason = "Pacemaker Remote nodes cannot run stonith agents";
	goto no_probe;

	} else if (pe__is_guest_node(node)
	&& pe__resource_contains_guest_node(rsc->cluster, rsc)) {
	reason = "guest nodes cannot run resources containing guest nodes";
	goto no_probe;

	} else if (rsc->is_remote_node) {
	reason = "Pacemaker Remote nodes cannot host remote connections";
	goto no_probe;
	}
	}

	// If this is a collective resource, probes are created for its children
	if (rsc->children != NULL) {
	return pcmk__probe_resource_list(rsc->children, node);
	}

	if ((rsc->container != NULL) && !rsc->is_remote_node) {
	reason = "resource is inside a container";
	goto no_probe;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	reason = "resource is orphaned";
	goto no_probe;

	} else if (g_hash_table_lookup(rsc->known_on, node->details->id) != NULL) {
	reason = "resource state is already known";
	goto no_probe;
	}

	allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (rsc->exclusive_discover \|\| top->exclusive_discover) {
	// Exclusive discovery is enabled ...

	if (allowed == NULL) {
	// ... but this node is not allowed to run the resource
	reason = "resource has exclusive discovery but is not allowed "
	"on node";
	goto no_probe;

	} else if (allowed->rsc_discover_mode != pcmk_probe_exclusive) {
	// ... but no constraint marks this node for discovery of resource
	reason = "resource has exclusive discovery but is not enabled "
	"on node";
	goto no_probe;
	}
	}

	if (allowed == NULL) {
	allowed = node;
	}
	if (allowed->rsc_discover_mode == pcmk_probe_never) {
	reason = "node has discovery disabled";
	goto no_probe;
	}

	if (pe__is_guest_node(node)) {
	pcmk_resource_t *guest = node->details->remote_rsc->container;

	if (guest->role == pcmk_role_stopped) {
	// The guest is stopped, so we know no resource is active there
	reason = "node's guest is stopped";
	probe_then_start(guest, top);
	goto no_probe;

	} else if (guest_resource_will_stop(node)) {
	reason = "node's guest will stop";

	// Order resource start after guest stop (in case it's restarting)
	pcmk__new_ordering(guest,
	pcmk__op_key(guest->id, PCMK_ACTION_STOP, 0),
	NULL, top,
	pcmk__op_key(top->id, PCMK_ACTION_START, 0),
	NULL, pcmk__ar_ordered, rsc->cluster);
	goto no_probe;
	}
	}

	// We've eliminated all cases where a probe is not needed, so now it is
	probe = probe_action(rsc, node);

	/* Below, we will order the probe relative to start or reload. If this is a
	* clone instance, the start or reload is for the entire clone rather than
	* just the instance. Otherwise, the start or reload is for the resource
	* itself.
	*/
	if (!pe_rsc_is_clone(top)) {
	top = rsc;
	}

	/* Prevent a start if the resource can't be probed, but don't cause the
	* resource or entire clone to stop if already active.
	*/
	if (!pcmk_is_set(probe->flags, pcmk_action_runnable)
	&& (top->running_on == NULL)) {
	pcmk__set_relation_flags(flags, pcmk__ar_unrunnable_first_blocks);
	}

	// Start or reload after probing the resource
	pcmk__new_ordering(rsc, NULL, probe,
	top, pcmk__op_key(top->id, PCMK_ACTION_START, 0), NULL,
	flags, rsc->cluster);
	pcmk__new_ordering(rsc, NULL, probe, top, reload_key(rsc), NULL,
	pcmk__ar_ordered, rsc->cluster);

	return true;

	no_probe:
	pcmk__rsc_trace(rsc,
	"Skipping probe for %s on %s because %s",
	rsc->id, node->details->id, reason);
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a probe should be ordered before another action
	*
	* \param[in] probe Probe action to check
	* \param[in] then Other action to check
	*
	* \return true if \p probe should be ordered before \p then, otherwise false
	*/
	static bool
	probe_needed_before_action(const pcmk_action_t *probe,
	const pcmk_action_t *then)
	{
	// Probes on a node are performed after unfencing it, not before
	if (pcmk__str_eq(then->task, PCMK_ACTION_STONITH, pcmk__str_none)
	- && pe__same_node(probe->node, then->node)) {
	+ && pcmk__same_node(probe->node, then->node)) {
	const char *op = g_hash_table_lookup(then->meta, "stonith_action");

	if (pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	return false;
	}
	}

	// Probes should be done on a node before shutting it down
	if (pcmk__str_eq(then->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)
	&& (probe->node != NULL) && (then->node != NULL)
	- && !pe__same_node(probe->node, then->node)) {
	+ && !pcmk__same_node(probe->node, then->node)) {
	return false;
	}

	// Otherwise probes should always be done before any other action
	return true;
	}

	/*!
	* \internal
	* \brief Add implicit "probe then X" orderings for "stop then X" orderings
	*
	* If the state of a resource is not known yet, a probe will be scheduled,
	* expecting a "not running" result. If the probe fails, a stop will not be
	* scheduled until the next transition. Thus, if there are ordering constraints
	* like "stop this resource then do something else that's not for the same
	* resource", add implicit "probe this resource then do something" equivalents
	* so the relation is upheld until we know whether a stop is needed.
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	add_probe_orderings_for_stops(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->ordering_constraints; iter != NULL;
	iter = iter->next) {

	pcmk__action_relation_t *order = iter->data;
	uint32_t order_flags = pcmk__ar_ordered;
	GList *probes = NULL;
	GList *then_actions = NULL;
	pcmk_action_t *first = NULL;
	pcmk_action_t *then = NULL;

	// Skip disabled orderings
	if (order->flags == pcmk__ar_none) {
	continue;
	}

	// Skip non-resource orderings, and orderings for the same resource
	if ((order->rsc1 == NULL) \|\| (order->rsc1 == order->rsc2)) {
	continue;
	}

	// Skip invalid orderings (shouldn't be possible)
	first = order->action1;
	then = order->action2;
	if (((first == NULL) && (order->task1 == NULL))
	\|\| ((then == NULL) && (order->task2 == NULL))) {
	continue;
	}

	// Skip orderings for first actions other than stop
	if ((first != NULL) && !pcmk__str_eq(first->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((first == NULL)
	&& !pcmk__ends_with(order->task1,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}

	/* Do not imply a probe ordering for a resource inside of a stopping
	* container. Otherwise, it might introduce a transition loop, since a
	* probe could be scheduled after the container starts again.
	*/
	if ((order->rsc2 != NULL) && (order->rsc1->container == order->rsc2)) {

	if ((then != NULL) && pcmk__str_eq(then->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((then == NULL)
	&& pcmk__ends_with(order->task2,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}
	}

	// Preserve certain order options for future filtering
	if (pcmk_is_set(order->flags, pcmk__ar_if_first_unmigratable)) {
	pcmk__set_relation_flags(order_flags,
	pcmk__ar_if_first_unmigratable);
	}
	if (pcmk_is_set(order->flags, pcmk__ar_if_on_same_node)) {
	pcmk__set_relation_flags(order_flags, pcmk__ar_if_on_same_node);
	}

	// Preserve certain order types for future filtering
	if ((order->flags == pcmk__ar_if_required_on_same_node)
	\|\| (order->flags == pcmk__ar_if_on_same_node_or_target)) {
	order_flags = order->flags;
	}

	// List all scheduled probes for the first resource
	probes = pe__resource_actions(order->rsc1, NULL, PCMK_ACTION_MONITOR,
	FALSE);
	if (probes == NULL) { // There aren't any
	continue;
	}

	// List all relevant "then" actions
	if (then != NULL) {
	then_actions = g_list_prepend(NULL, then);

	} else if (order->rsc2 != NULL) {
	then_actions = find_actions(order->rsc2->actions, order->task2,
	NULL);
	if (then_actions == NULL) { // There aren't any
	g_list_free(probes);
	continue;
	}
	}

	crm_trace("Implying 'probe then' orderings for '%s then %s' "
	"(id=%d, type=%.6x)",
	((first == NULL)? order->task1 : first->uuid),
	((then == NULL)? order->task2 : then->uuid),
	order->id, order->flags);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	for (GList *then_iter = then_actions; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk_action_t then = (pcmk_action_t ) then_iter->data;

	if (probe_needed_before_action(probe, then)) {
	order_actions(probe, then, order_flags);
	}
	}
	}

	g_list_free(then_actions);
	g_list_free(probes);
	}
	}

	/*!
	* \internal
	* \brief Add necessary orderings between probe and starts of clone instances
	*
	* , in additon to the ordering with the parent resource added upon creating
	* the probe.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action wrapper in the ordering
	*/
	static void
	add_start_orderings_for_probe(pcmk_action_t *probe,
	pcmk__related_action_t *after)
	{
	uint32_t flags = pcmk__ar_ordered\|pcmk__ar_unrunnable_first_blocks;

	/* Although the ordering between the probe of the clone instance and the
	* start of its parent has been added in pcmk__probe_rsc_on_node(), we
	* avoided enforcing `pcmk__ar_unrunnable_first_blocks` order type for that
	* as long as any of the clone instances are running to prevent them from
	* being unexpectedly stopped.
	*
	* On the other hand, we still need to prevent any inactive instances from
	* starting unless the probe is runnable so that we don't risk starting too
	* many instances before we know the state on all nodes.
	*/
	if ((after->action->rsc->variant <= pcmk_rsc_variant_group)
	\|\| pcmk_is_set(probe->flags, pcmk_action_runnable)
	// The order type is already enforced for its parent.
	\|\| pcmk_is_set(after->type, pcmk__ar_unrunnable_first_blocks)
	\|\| (pe__const_top_resource(probe->rsc, false) != after->action->rsc)
	\|\| !pcmk__str_eq(after->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	return;
	}

	crm_trace("Adding probe start orderings for 'unrunnable %s@%s "
	"then instances of %s@%s'",
	probe->uuid, pcmk__node_name(probe->node),
	after->action->uuid, pcmk__node_name(after->action->node));

	for (GList *then_iter = after->action->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	if (then->action->rsc->running_on
	\|\| (pe__const_top_resource(then->action->rsc, false)
	!= after->action->rsc)
	\|\| !pcmk__str_eq(then->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	continue;
	}

	crm_trace("Adding probe start ordering for 'unrunnable %s@%s "
	"then %s@%s' (type=%#.6x)",
	probe->uuid, pcmk__node_name(probe->node),
	then->action->uuid, pcmk__node_name(then->action->node),
	flags);

	/* Prevent the instance from starting if the instance can't, but don't
	* cause any other intances to stop if already active.
	*/
	order_actions(probe, then->action, flags);
	}

	return;
	}

	/*!
	* \internal
	* \brief Order probes before restarts and re-promotes
	*
	* If a given ordering is a "probe then start" or "probe then promote" ordering,
	* add an implicit "probe then stop/demote" ordering in case the action is part
	* of a restart/re-promote, and do the same recursively for all actions ordered
	* after the "then" action.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action in the ordering
	*/
	static void
	add_restart_orderings_for_probe(pcmk_action_t probe, pcmk_action_t after)
	{
	GList *iter = NULL;
	bool interleave = false;
	pcmk_resource_t *compatible_rsc = NULL;

	// Validate that this is a resource probe followed by some action
	if ((after == NULL) \|\| (probe == NULL) \|\| (probe->rsc == NULL)
	\|\| (probe->rsc->variant != pcmk_rsc_variant_primitive)
	\|\| !pcmk__str_eq(probe->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	return;
	}

	// Avoid running into any possible loop
	if (pcmk_is_set(after->flags, pcmk_action_detect_loop)) {
	return;
	}
	pcmk__set_action_flags(after, pcmk_action_detect_loop);

	crm_trace("Adding probe restart orderings for '%s@%s then %s@%s'",
	probe->uuid, pcmk__node_name(probe->node),
	after->uuid, pcmk__node_name(after->node));

	/* Add restart orderings if "then" is for a different primitive.
	* Orderings for collective resources will be added later.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant == pcmk_rsc_variant_primitive)
	&& (probe->rsc != after->rsc)) {

	GList *then_actions = NULL;

	if (pcmk__str_eq(after->task, PCMK_ACTION_START, pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_STOP, FALSE);

	} else if (pcmk__str_eq(after->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_DEMOTE, FALSE);
	}

	for (iter = then_actions; iter != NULL; iter = iter->next) {
	pcmk_action_t then = (pcmk_action_t ) iter->data;

	// Skip pseudo-actions (for example, those implied by fencing)
	if (!pcmk_is_set(then->flags, pcmk_action_pseudo)) {
	order_actions(probe, then, pcmk__ar_ordered);
	}
	}
	g_list_free(then_actions);
	}

	/* Detect whether "then" is an interleaved clone action. For these, we want
	* to add orderings only for the relevant instance.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant > pcmk_rsc_variant_group)) {
	const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
	PCMK_META_INTERLEAVE);

	interleave = crm_is_true(interleave_s);
	if (interleave) {
	compatible_rsc = pcmk__find_compatible_instance(probe->rsc,
	after->rsc,
	pcmk_role_unknown,
	false);
	}
	}

	/* Now recursively do the same for all actions ordered after "then". This
	* also handles collective resources since the collective action will be
	* ordered before its individual instances' actions.
	*/
	for (iter = after->actions_after; iter != NULL; iter = iter->next) {
	pcmk__related_action_t *after_wrapper = iter->data;

	/* pcmk__ar_first_implies_then is the reason why a required A.start
	* implies/enforces B.start to be required too, which is the cause of
	* B.restart/re-promote.
	*
	* Not sure about pcmk__ar_first_implies_same_node_then though. It's now
	* only used for unfencing case, which tends to introduce transition
	* loops...
	*/
	if (!pcmk_is_set(after_wrapper->type, pcmk__ar_first_implies_then)) {
	/* The order type between a group/clone and its child such as
	* B.start-> B_child.start is:
	* pcmk__ar_then_implies_first_graphed
	* \|pcmk__ar_unrunnable_first_blocks
	*
	* Proceed through the ordering chain and build dependencies with
	* its children.
	*/
	if ((after->rsc == NULL)
	\|\| (after->rsc->variant < pcmk_rsc_variant_group)
	\|\| (probe->rsc->parent == after->rsc)
	\|\| (after_wrapper->action->rsc == NULL)
	\|\| (after_wrapper->action->rsc->variant > pcmk_rsc_variant_group)
	\|\| (after->rsc != after_wrapper->action->rsc->parent)) {
	continue;
	}

	/* Proceed to the children of a group or a non-interleaved clone.
	* For an interleaved clone, proceed only to the relevant child.
	*/
	if ((after->rsc->variant > pcmk_rsc_variant_group) && interleave
	&& ((compatible_rsc == NULL)
	\|\| (compatible_rsc != after_wrapper->action->rsc))) {
	continue;
	}
	}

	crm_trace("Recursively adding probe restart orderings for "
	"'%s@%s then %s@%s' (type=%#.6x)",
	after->uuid, pcmk__node_name(after->node),
	after_wrapper->action->uuid,
	pcmk__node_name(after_wrapper->action->node),
	after_wrapper->type);

	add_restart_orderings_for_probe(probe, after_wrapper->action);
	}
	}

	/*!
	* \internal
	* \brief Clear the tracking flag on all scheduled actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	clear_actions_tracking_flag(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t *action = iter->data;

	pcmk__clear_action_flags(action, pcmk_action_detect_loop);
	}
	}

	/*!
	* \internal
	* \brief Add start and restart orderings for probes scheduled for a resource
	*
	* \param[in,out] data Resource whose probes should be ordered
	* \param[in] user_data Unused
	*/
	static void
	add_start_restart_orderings_for_rsc(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	GList *probes = NULL;

	// For collective resources, order each instance recursively
	if (rsc->variant != pcmk_rsc_variant_primitive) {
	g_list_foreach(rsc->children, add_start_restart_orderings_for_rsc,
	NULL);
	return;
	}

	// Find all probes for given resource
	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	// Add probe restart orderings for each probe found
	for (GList *iter = probes; iter != NULL; iter = iter->next) {
	pcmk_action_t probe = (pcmk_action_t ) iter->data;

	for (GList *then_iter = probe->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	add_start_orderings_for_probe(probe, then);
	add_restart_orderings_for_probe(probe, then->action);
	clear_actions_tracking_flag(rsc->cluster);
	}
	}

	g_list_free(probes);
	}

	/*!
	* \internal
	* \brief Add "A then probe B" orderings for "A then B" orderings
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This function is currently disabled (see next comment).
	*/
	static void
	order_then_probes(pcmk_scheduler_t *scheduler)
	{
	#if 0
	/* Given an ordering "A then B", we would prefer to wait for A to be started
	* before probing B.
	*
	* For example, if A is a filesystem which B can't even run without, it
	* would be helpful if the author of B's agent could assume that A is
	* running before B.monitor will be called.
	*
	* However, we can't _only_ probe after A is running, otherwise we wouldn't
	* detect the state of B if A could not be started. We can't even do an
	* opportunistic version of this, because B may be moving:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> B.start
	*
	* and if we add B.stop -> A.stop here, we get a loop:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> A.stop
	*
	* We could kill the "B.probe -> B.stop" dependency, but that could mean
	* stopping B "too" soon, because B.start must wait for the probe, and
	* we don't want to stop B if we can't start it.
	*
	* We could add the ordering only if A is an anonymous clone with
	* clone-max == node-max (since we'll never be moving it). However, we could
	* still be stopping one instance at the same time as starting another.
	*
	* The complexity of checking for allowed conditions combined with the ever
	* narrowing use case suggests that this code should remain disabled until
	* someone gets smarter.
	*/
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	pcmk_action_t *start = NULL;
	GList *actions = NULL;
	GList *probes = NULL;

	actions = pe__resource_actions(rsc, NULL, PCMK_ACTION_START, FALSE);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if (start == NULL) {
	crm_debug("No start action for %s", rsc->id);
	continue;
	}

	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	for (actions = start->actions_before; actions != NULL;
	actions = actions->next) {

	pcmk__related_action_t *before = actions->data;

	pcmk_action_t *first = before->action;
	pcmk_resource_t *first_rsc = first->rsc;

	if (first->required_runnable_before) {
	for (GList *clone_actions = first->actions_before;
	clone_actions != NULL;
	clone_actions = clone_actions->next) {

	before = clone_actions->data;

	crm_trace("Testing '%s then %s' for %s",
	first->uuid, before->action->uuid, start->uuid);

	CRM_ASSERT(before->action->rsc != NULL);
	first_rsc = before->action->rsc;
	break;
	}

	} else if (!pcmk__str_eq(first->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if (first_rsc == NULL) {
	continue;

	} else if (pe__const_top_resource(first_rsc, false)
	== pe__const_top_resource(start->rsc, false)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if (!pe_rsc_is_clone(pe__const_top_resource(first_rsc,
	false))) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	crm_debug("Applying %s before %s %d", first->uuid, start->uuid,
	pe__const_top_resource(first_rsc, false)->variant);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	crm_debug("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pcmk__ar_ordered);
	}
	}
	}
	#endif
	}

	void
	pcmk__order_probes(pcmk_scheduler_t *scheduler)
	{
	// Add orderings for "probe then X"
	g_list_foreach(scheduler->resources, add_start_restart_orderings_for_rsc,
	NULL);
	add_probe_orderings_for_stops(scheduler);

	order_then_probes(scheduler);
	}

	/*!
	* \internal
	* \brief Schedule any probes needed
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This may also schedule fencing of failed remote nodes.
	*/
	void
	pcmk__schedule_probes(pcmk_scheduler_t *scheduler)
	{
	// Schedule probes on each node in the cluster as needed
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	const char *probed = NULL;

	if (!node->details->online) { // Don't probe offline nodes
	if (pcmk__is_failed_remote_node(node)) {
	pe_fence_node(scheduler, node,
	"the connection is unrecoverable", FALSE);
	}
	continue;

	} else if (node->details->unclean) { // ... or nodes that need fencing
	continue;

	} else if (!node->details->rsc_discovery_enabled) {
	// The user requested that probes not be done on this node
	continue;
	}

	/* This is no longer needed for live clusters, since the probe_complete
	* node attribute will never be in the CIB. However this is still useful
	* for processing old saved CIBs (< 1.1.14), including the
	* reprobe-target_rc regression test.
	*/
	probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
	if (probed != NULL && crm_is_true(probed) == FALSE) {
	pcmk_action_t *probe_op = NULL;

	probe_op = custom_action(NULL,
	crm_strdup_printf("%s-%s", CRM_OP_REPROBE,
	node->details->uname),
	CRM_OP_REPROBE, node, FALSE, scheduler);
	add_hash_param(probe_op->meta, PCMK__META_OP_NO_WAIT,
	PCMK_VALUE_TRUE);
	continue;
	}

	// Probe each resource in the cluster on this node, as needed
	pcmk__probe_resource_list(scheduler->resources, node);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_remote.c b/lib/pacemaker/pcmk_sched_remote.c
	index 9707d1f7f7..6efb13a362 100644
	--- a/lib/pacemaker/pcmk_sched_remote.c
	+++ b/lib/pacemaker/pcmk_sched_remote.c
	@@ -1,735 +1,735 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	enum remote_connection_state {
	remote_state_unknown = 0,
	remote_state_alive = 1,
	remote_state_resting = 2,
	remote_state_failed = 3,
	remote_state_stopped = 4
	};

	static const char *
	state2text(enum remote_connection_state state)
	{
	switch (state) {
	case remote_state_unknown:
	return "unknown";
	case remote_state_alive:
	return "alive";
	case remote_state_resting:
	return "resting";
	case remote_state_failed:
	return "failed";
	case remote_state_stopped:
	return "stopped";
	}

	return "impossible";
	}

	/* We always use pcmk__ar_guest_allowed with these convenience functions to
	* exempt internally generated constraints from the prohibition of user
	* constraints involving remote connection resources.
	*
	* The start ordering additionally uses pcmk__ar_unrunnable_first_blocks so that
	* the specified action is not runnable if the start is not runnable.
	*/

	static inline void
	order_start_then_action(pcmk_resource_t first_rsc, pcmk_action_t then_action,
	uint32_t extra)
	{
	if ((first_rsc != NULL) && (then_action != NULL)) {
	pcmk__new_ordering(first_rsc, start_key(first_rsc), NULL,
	then_action->rsc, NULL, then_action,
	pcmk__ar_guest_allowed
	\|pcmk__ar_unrunnable_first_blocks
	\|extra,
	first_rsc->cluster);
	}
	}

	static inline void
	order_action_then_stop(pcmk_action_t first_action, pcmk_resource_t then_rsc,
	uint32_t extra)
	{
	if ((first_action != NULL) && (then_rsc != NULL)) {
	pcmk__new_ordering(first_action->rsc, NULL, first_action,
	then_rsc, stop_key(then_rsc), NULL,
	pcmk__ar_guest_allowed\|extra, then_rsc->cluster);
	}
	}

	static enum remote_connection_state
	get_remote_node_state(const pcmk_node_t *node)
	{
	const pcmk_resource_t *remote_rsc = NULL;
	const pcmk_node_t *cluster_node = NULL;

	CRM_ASSERT(node != NULL);

	remote_rsc = node->details->remote_rsc;
	CRM_ASSERT(remote_rsc != NULL);

	cluster_node = pcmk__current_node(remote_rsc);

	/* If the cluster node the remote connection resource resides on
	* is unclean or went offline, we can't process any operations
	* on that remote node until after it starts elsewhere.
	*/
	if ((remote_rsc->next_role == pcmk_role_stopped)
	\|\| (remote_rsc->allocated_to == NULL)) {

	// The connection resource is not going to run anywhere

	if ((cluster_node != NULL) && cluster_node->details->unclean) {
	/* The remote connection is failed because its resource is on a
	* failed node and can't be recovered elsewhere, so we must fence.
	*/
	return remote_state_failed;
	}

	if (!pcmk_is_set(remote_rsc->flags, pcmk_rsc_failed)) {
	/* Connection resource is cleanly stopped */
	return remote_state_stopped;
	}

	/* Connection resource is failed */

	if ((remote_rsc->next_role == pcmk_role_stopped)
	&& remote_rsc->remote_reconnect_ms
	&& node->details->remote_was_fenced
	&& !pe__shutdown_requested(node)) {

	/* We won't know whether the connection is recoverable until the
	* reconnect interval expires and we reattempt connection.
	*/
	return remote_state_unknown;
	}

	/* The remote connection is in a failed state. If there are any
	* resources known to be active on it (stop) or in an unknown state
	* (probe), we must assume the worst and fence it.
	*/
	return remote_state_failed;

	} else if (cluster_node == NULL) {
	/* Connection is recoverable but not currently running anywhere, so see
	* if we can recover it first
	*/
	return remote_state_unknown;

	} else if (cluster_node->details->unclean
	\|\| !(cluster_node->details->online)) {
	// Connection is running on a dead node, see if we can recover it first
	return remote_state_resting;

	} else if (pcmk__list_of_multiple(remote_rsc->running_on)
	&& (remote_rsc->partial_migration_source != NULL)
	&& (remote_rsc->partial_migration_target != NULL)) {
	/* We're in the middle of migrating a connection resource, so wait until
	* after the migration completes before performing any actions.
	*/
	return remote_state_resting;

	}
	return remote_state_alive;
	}

	/*!
	* \internal
	* \brief Order actions on remote node relative to actions for the connection
	*
	* \param[in,out] action An action scheduled on a Pacemaker Remote node
	*/
	static void
	apply_remote_ordering(pcmk_action_t *action)
	{
	pcmk_resource_t *remote_rsc = NULL;
	enum action_tasks task = text2task(action->task);
	enum remote_connection_state state = get_remote_node_state(action->node);

	uint32_t order_opts = pcmk__ar_none;

	if (action->rsc == NULL) {
	return;
	}

	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc != NULL);

	crm_trace("Order %s action %s relative to %s%s (state: %s)",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pcmk_rsc_failed)? "failed " : "",
	remote_rsc->id, state2text(state));

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* Migration ops map to pcmk_action_unspecified, but we need to apply
	* the same ordering as for stop or demote (see get_router_node()).
	*/
	task = pcmk_action_stop;
	}

	switch (task) {
	case pcmk_action_start:
	case pcmk_action_promote:
	order_opts = pcmk__ar_none;

	if (state == remote_state_failed) {
	/* Force recovery, by making this action required */
	pcmk__set_relation_flags(order_opts,
	pcmk__ar_first_implies_then);
	}

	/* Ensure connection is up before running this action */
	order_start_then_action(remote_rsc, action, order_opts);
	break;

	case pcmk_action_stop:
	if (state == remote_state_alive) {
	order_action_then_stop(action, remote_rsc,
	pcmk__ar_then_implies_first);

	} else if (state == remote_state_failed) {
	/* The resource is active on the node, but since we don't have a
	* valid connection, the only way to stop the resource is by
	* fencing the node. There is no need to order the stop relative
	* to the remote connection, since the stop will become implied
	* by the fencing.
	*/
	pe_fence_node(remote_rsc->cluster, action->node,
	"resources are active but "
	"connection is unrecoverable",
	FALSE);

	} else if (remote_rsc->next_role == pcmk_role_stopped) {
	/* State must be remote_state_unknown or remote_state_stopped.
	* Since the connection is not coming back up in this
	* transition, stop this resource first.
	*/
	order_action_then_stop(action, remote_rsc,
	pcmk__ar_then_implies_first);

	} else {
	/* The connection is going to be started somewhere else, so
	* stop this resource after that completes.
	*/
	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	}
	break;

	case pcmk_action_demote:
	/* Only order this demote relative to the connection start if the
	* connection isn't being torn down. Otherwise, the demote would be
	* blocked because the connection start would not be allowed.
	*/
	if ((state == remote_state_resting)
	\|\| (state == remote_state_unknown)) {

	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	} /* Otherwise we can rely on the stop ordering */
	break;

	default:
	/* Wait for the connection resource to be up */
	if (pcmk__action_is_recurring(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	order_start_then_action(remote_rsc, action,
	pcmk__ar_first_implies_then);

	} else {
	pcmk_node_t *cluster_node = pcmk__current_node(remote_rsc);

	if ((task == pcmk_action_monitor) && (state == remote_state_failed)) {
	/* We would only be here if we do not know the state of the
	* resource on the remote node. Since we have no way to find
	* out, it is necessary to fence the node.
	*/
	pe_fence_node(remote_rsc->cluster, action->node,
	"resources are in unknown state "
	"and connection is unrecoverable", FALSE);
	}

	if ((cluster_node != NULL) && (state == remote_state_stopped)) {
	/* The connection is currently up, but is going down
	* permanently. Make sure we check services are actually
	* stopped _before_ we let the connection get closed.
	*/
	order_action_then_stop(action, remote_rsc,
	pcmk__ar_unrunnable_first_blocks);

	} else {
	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	}
	}
	break;
	}
	}

	static void
	apply_container_ordering(pcmk_action_t *action)
	{
	/* VMs are also classified as containers for these purposes... in
	* that they both involve a 'thing' running on a real or remote
	* cluster node.
	*
	* This allows us to be smarter about the type and extent of
	* recovery actions required in various scenarios
	*/
	pcmk_resource_t *remote_rsc = NULL;
	pcmk_resource_t *container = NULL;
	enum action_tasks task = text2task(action->task);

	CRM_ASSERT(action->rsc != NULL);
	CRM_ASSERT(action->node != NULL);
	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc != NULL);

	container = remote_rsc->container;
	CRM_ASSERT(container != NULL);

	if (pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	pe_fence_node(action->rsc->cluster, action->node, "container failed",
	FALSE);
	}

	crm_trace("Order %s action %s relative to %s%s for %s%s",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pcmk_rsc_failed)? "failed " : "",
	remote_rsc->id,
	pcmk_is_set(container->flags, pcmk_rsc_failed)? "failed " : "",
	container->id);

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* Migration ops map to pcmk_action_unspecified, but we need to apply
	* the same ordering as for stop or demote (see get_router_node()).
	*/
	task = pcmk_action_stop;
	}

	switch (task) {
	case pcmk_action_start:
	case pcmk_action_promote:
	// Force resource recovery if the container is recovered
	order_start_then_action(container, action,
	pcmk__ar_first_implies_then);

	// Wait for the connection resource to be up, too
	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	break;

	case pcmk_action_stop:
	case pcmk_action_demote:
	if (pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	/* When the container representing a guest node fails, any stop
	* or demote actions for resources running on the guest node
	* are implied by the container stopping. This is similar to
	* how fencing operations work for cluster nodes and remote
	* nodes.
	*/
	} else {
	/* Ensure the operation happens before the connection is brought
	* down.
	*
	* If we really wanted to, we could order these after the
	* connection start, IFF the container's current role was
	* stopped (otherwise we re-introduce an ordering loop when the
	* connection is restarting).
	*/
	order_action_then_stop(action, remote_rsc, pcmk__ar_none);
	}
	break;

	default:
	/* Wait for the connection resource to be up */
	if (pcmk__action_is_recurring(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	if (task != pcmk_action_unspecified) {
	order_start_then_action(remote_rsc, action,
	pcmk__ar_first_implies_then);
	}
	} else {
	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	}
	break;
	}
	}

	/*!
	* \internal
	* \brief Order all relevant actions relative to remote connection actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__order_remote_connection_actions(pcmk_scheduler_t *scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {
	return;
	}

	crm_trace("Creating remote connection orderings");

	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t *action = iter->data;
	pcmk_resource_t *remote = NULL;

	// We are only interested in resource actions
	if (action->rsc == NULL) {
	continue;
	}

	/* Special case: If we are clearing the failcount of an actual
	* remote connection resource, then make sure this happens before
	* any start of the resource in this transition.
	*/
	if (action->rsc->is_remote_node &&
	pcmk__str_eq(action->task, PCMK_ACTION_CLEAR_FAILCOUNT,
	pcmk__str_none)) {

	pcmk__new_ordering(action->rsc, NULL, action, action->rsc,
	pcmk__op_key(action->rsc->id, PCMK_ACTION_START,
	0),
	NULL, pcmk__ar_ordered, scheduler);

	continue;
	}

	// We are only interested in actions assigned to a node
	if (action->node == NULL) {
	continue;
	}

	if (!pe__is_guest_or_remote_node(action->node)) {
	continue;
	}

	/* We are only interested in real actions.
	*
	* @TODO This is probably wrong; pseudo-actions might be converted to
	* real actions and vice versa later in update_actions() at the end of
	* pcmk__apply_orderings().
	*/
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	continue;
	}

	remote = action->node->details->remote_rsc;
	if (remote == NULL) {
	// Orphaned
	continue;
	}

	/* Another special case: if a resource is moving to a Pacemaker Remote
	* node, order the stop on the original node after any start of the
	* remote connection. This ensures that if the connection fails to
	* start, we leave the resource running on the original node.
	*/
	if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_none)) {
	for (GList *item = action->rsc->actions; item != NULL;
	item = item->next) {
	pcmk_action_t *rsc_action = item->data;

	- if (!pe__same_node(rsc_action->node, action->node)
	+ if (!pcmk__same_node(rsc_action->node, action->node)
	&& pcmk__str_eq(rsc_action->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	pcmk__new_ordering(remote, start_key(remote), NULL,
	action->rsc, NULL, rsc_action,
	pcmk__ar_ordered, scheduler);
	}
	}
	}

	/* The action occurs across a remote connection, so create
	* ordering constraints that guarantee the action occurs while the node
	* is active (after start, before stop ... things like that).
	*
	* This is somewhat brittle in that we need to make sure the results of
	* this ordering are compatible with the result of get_router_node().
	* It would probably be better to add PCMK__XA_ROUTER_NODE as part of
	* this logic rather than create_graph_action().
	*/
	if (remote->container) {
	crm_trace("Container ordering for %s", action->uuid);
	apply_container_ordering(action);

	} else {
	crm_trace("Remote ordering for %s", action->uuid);
	apply_remote_ordering(action);
	}
	}
	}

	/*!
	* \internal
	* \brief Check whether a node is a failed remote node
	*
	* \param[in] node Node to check
	*
	* \return true if \p node is a failed remote node, false otherwise
	*/
	bool
	pcmk__is_failed_remote_node(const pcmk_node_t *node)
	{
	return pe__is_remote_node(node) && (node->details->remote_rsc != NULL)
	&& (get_remote_node_state(node) == remote_state_failed);
	}

	/*!
	* \internal
	* \brief Check whether a given resource corresponds to a given node as guest
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is a guest node and \p rsc is its containing
	* resource, otherwise false
	*/
	bool
	pcmk__rsc_corresponds_to_guest(const pcmk_resource_t *rsc,
	const pcmk_node_t *node)
	{
	return (rsc != NULL) && (rsc->fillers != NULL) && (node != NULL)
	&& (node->details->remote_rsc != NULL)
	&& (node->details->remote_rsc->container == rsc);
	}

	/*!
	* \internal
	* \brief Get proper connection host that a remote action must be routed through
	*
	* A remote connection resource might be starting, stopping, or migrating in the
	* same transition that an action needs to be executed on its Pacemaker Remote
	* node. Determine the proper node that the remote action should be routed
	* through.
	*
	* \param[in] action (Potentially remote) action to route
	*
	* \return Connection host that action should be routed through if remote,
	* otherwise NULL
	*/
	pcmk_node_t *
	pcmk__connection_host_for_action(const pcmk_action_t *action)
	{
	pcmk_node_t *began_on = NULL;
	pcmk_node_t *ended_on = NULL;
	bool partial_migration = false;
	const char *task = action->task;

	if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_none)
	\|\| !pe__is_guest_or_remote_node(action->node)) {
	return NULL;
	}

	CRM_ASSERT(action->node->details->remote_rsc != NULL);

	began_on = pcmk__current_node(action->node->details->remote_rsc);
	ended_on = action->node->details->remote_rsc->allocated_to;
	if (action->node->details->remote_rsc
	&& (action->node->details->remote_rsc->container == NULL)
	&& action->node->details->remote_rsc->partial_migration_target) {
	partial_migration = true;
	}

	if (began_on == NULL) {
	crm_trace("Routing %s for %s through remote connection's "
	"next node %s (starting)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(ended_on? ended_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return ended_on;
	}

	if (ended_on == NULL) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (stopping)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	- if (pe__same_node(began_on, ended_on)) {
	+ if (pcmk__same_node(began_on, ended_on)) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (not moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	/* If we get here, the remote connection is moving during this transition.
	* This means some actions for resources behind the connection will get
	* routed through the cluster node the connection resource is currently on,
	* and others are routed through the cluster node the connection will end up
	* on.
	*/

	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	task = g_hash_table_lookup(action->meta, "notify_operation");
	}

	/*
	* Stop, demote, and migration actions must occur before the connection can
	* move (these actions are required before the remote resource can stop). In
	* this case, we know these actions have to be routed through the initial
	* cluster node the connection resource lived on before the move takes
	* place.
	*
	* The exception is a partial migration of a (non-guest) remote connection
	* resource; in that case, all actions (even these) will be ordered after
	* the connection's pseudo-start on the migration target, so the target is
	* the router node.
	*/
	if (pcmk__strcase_any_of(task, PCMK_ACTION_CANCEL, PCMK_ACTION_STOP,
	PCMK_ACTION_DEMOTE, PCMK_ACTION_MIGRATE_FROM,
	PCMK_ACTION_MIGRATE_TO, NULL)
	&& !partial_migration) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	/* Everything else (start, promote, monitor, probe, refresh,
	* clear failcount, delete, ...) must occur after the connection starts on
	* the node it is moving to.
	*/
	crm_trace("Routing %s for %s through remote connection's "
	"next node %s (moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(ended_on? ended_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return ended_on;
	}

	/*!
	* \internal
	* \brief Replace remote connection's addr="#uname" with actual address
	*
	* REMOTE_CONTAINER_HACK: If a given resource is a remote connection resource
	* with its "addr" parameter set to "#uname", pull the actual value from the
	* parameters evaluated without a node (which was put there earlier in
	* pcmk__create_graph() when the bundle's expand() method was called).
	*
	* \param[in,out] rsc Resource to check
	* \param[in,out] params Resource parameters evaluated per node
	*/
	void
	pcmk__substitute_remote_addr(pcmk_resource_t rsc, GHashTable params)
	{
	const char *remote_addr = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);

	if (pcmk__str_eq(remote_addr, "#uname", pcmk__str_none)) {
	GHashTable *base = pe_rsc_params(rsc, NULL, rsc->cluster);

	remote_addr = g_hash_table_lookup(base, PCMK_REMOTE_RA_ADDR);
	if (remote_addr != NULL) {
	g_hash_table_insert(params, strdup(PCMK_REMOTE_RA_ADDR),
	strdup(remote_addr));
	}
	}
	}

	/*!
	* \brief Add special guest node meta-attributes to XML
	*
	* If a given action will be executed on a guest node, add the following as XML
	* attributes (using meta-attribute naming):
	* * The resource's \c PCMK_META_CONTAINER_ATTR_TARGET meta-attribute (usually
	* set only for bundles), as \c PCMK_META_CONTAINER_ATTR_TARGET
	* * The guest's physical host (current host for "down" actions, next host for
	* "up" actions), as \c PCMK__META_PHYSICAL_HOST
	*
	* If the guest node has no physical host, then don't add either attribute.
	*
	* \param[in,out] args_xml XML to add attributes to
	* \param[in] action Action to check
	*/
	void
	pcmk__add_guest_meta_to_xml(xmlNode args_xml, const pcmk_action_t action)
	{
	const pcmk_node_t *guest = action->node;
	const pcmk_node_t *host = NULL;
	enum action_tasks task;

	if (!pe__is_guest_node(guest)) {
	return;
	}

	task = text2task(action->task);
	if ((task == pcmk_action_notify) \|\| (task == pcmk_action_notified)) {
	task = text2task(g_hash_table_lookup(action->meta, "notify_operation"));
	}

	switch (task) {
	case pcmk_action_stop:
	case pcmk_action_stopped:
	case pcmk_action_demote:
	case pcmk_action_demoted:
	// "Down" actions take place on guest's current host
	host = pcmk__current_node(guest->details->remote_rsc->container);
	break;

	case pcmk_action_start:
	case pcmk_action_started:
	case pcmk_action_monitor:
	case pcmk_action_promote:
	case pcmk_action_promoted:
	// "Up" actions take place on guest's next host
	host = guest->details->remote_rsc->container->allocated_to;
	break;

	default:
	break;
	}

	if (host != NULL) {
	gpointer target = g_hash_table_lookup(action->rsc->meta,
	PCMK_META_CONTAINER_ATTR_TARGET);

	hash2metafield((gpointer) PCMK_META_CONTAINER_ATTR_TARGET,
	target,
	(gpointer) args_xml);
	hash2metafield((gpointer) PCMK__META_PHYSICAL_HOST,
	(gpointer) host->details->uname,
	(gpointer) args_xml);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c
	index 9a76685b78..c311fb7819 100644
	--- a/lib/pacemaker/pcmk_sched_resource.c
	+++ b/lib/pacemaker/pcmk_sched_resource.c
	@@ -1,774 +1,774 @@
	/*
	* Copyright 2014-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <string.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Resource assignment methods by resource variant
	static pcmk_assignment_methods_t assignment_methods[] = {
	{
	pcmk__primitive_assign,
	pcmk__primitive_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__primitive_internal_constraints,
	pcmk__primitive_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_primitive_colocations,
	pcmk__primitive_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__apply_location,
	pcmk__primitive_action_flags,
	pcmk__update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__primitive_add_graph_meta,
	pcmk__primitive_add_utilization,
	pcmk__primitive_shutdown_lock,
	},
	{
	pcmk__group_assign,
	pcmk__group_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__group_internal_constraints,
	pcmk__group_apply_coloc_score,
	pcmk__group_colocated_resources,
	pcmk__with_group_colocations,
	pcmk__group_with_colocations,
	pcmk__group_add_colocated_node_scores,
	pcmk__group_apply_location,
	pcmk__group_action_flags,
	pcmk__group_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__group_add_utilization,
	pcmk__group_shutdown_lock,
	},
	{
	pcmk__clone_assign,
	pcmk__clone_create_actions,
	pcmk__clone_create_probe,
	pcmk__clone_internal_constraints,
	pcmk__clone_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_clone_colocations,
	pcmk__clone_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__clone_apply_location,
	pcmk__clone_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__clone_add_actions_to_graph,
	pcmk__clone_add_graph_meta,
	pcmk__clone_add_utilization,
	pcmk__clone_shutdown_lock,
	},
	{
	pcmk__bundle_assign,
	pcmk__bundle_create_actions,
	pcmk__bundle_create_probe,
	pcmk__bundle_internal_constraints,
	pcmk__bundle_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_bundle_colocations,
	pcmk__bundle_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__bundle_apply_location,
	pcmk__bundle_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_bundle_actions,
	pcmk__bundle_add_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__bundle_add_utilization,
	pcmk__bundle_shutdown_lock,
	}
	};

	/*!
	* \internal
	* \brief Check whether a resource's agent standard, provider, or type changed
	*
	* \param[in,out] rsc Resource to check
	* \param[in,out] node Node needing unfencing if agent changed
	* \param[in] rsc_entry XML with previously known agent information
	* \param[in] active_on_node Whether \p rsc is active on \p node
	*
	* \return true if agent for \p rsc changed, otherwise false
	*/
	bool
	pcmk__rsc_agent_changed(pcmk_resource_t rsc, pcmk_node_t node,
	const xmlNode *rsc_entry, bool active_on_node)
	{
	bool changed = false;
	const char *attr_list[] = {
	PCMK_XA_TYPE,
	PCMK_XA_CLASS,
	PCMK_XA_PROVIDER,
	};

	for (int i = 0; i < PCMK__NELEM(attr_list); i++) {
	const char *value = crm_element_value(rsc->xml, attr_list[i]);
	const char *old_value = crm_element_value(rsc_entry, attr_list[i]);

	if (!pcmk__str_eq(value, old_value, pcmk__str_none)) {
	changed = true;
	trigger_unfencing(rsc, node, "Device definition changed", NULL,
	rsc->cluster);
	if (active_on_node) {
	crm_notice("Forcing restart of %s on %s "
	"because %s changed from '%s' to '%s'",
	rsc->id, pcmk__node_name(node), attr_list[i],
	pcmk__s(old_value, ""), pcmk__s(value, ""));
	}
	}
	}
	if (changed && active_on_node) {
	// Make sure the resource is restarted
	custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE,
	rsc->cluster);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_start_pending);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Add resource (and any matching children) to list if it matches ID
	*
	* \param[in] result List to add resource to
	* \param[in] rsc Resource to check
	* \param[in] id ID to match
	*
	* \return (Possibly new) head of list
	*/
	static GList *
	add_rsc_if_matching(GList result, pcmk_resource_t rsc, const char *id)
	{
	if ((strcmp(rsc->id, id) == 0)
	\|\| ((rsc->clone_name != NULL) && (strcmp(rsc->clone_name, id) == 0))) {
	result = g_list_prepend(result, rsc);
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	result = add_rsc_if_matching(result, child, id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Find all resources matching a given ID by either ID or clone name
	*
	* \param[in] id Resource ID to check
	* \param[in] scheduler Scheduler data
	*
	* \return List of all resources that match \p id
	* \note The caller is responsible for freeing the return value with
	* g_list_free().
	*/
	GList *
	pcmk__rscs_matching_id(const char id, const pcmk_scheduler_t scheduler)
	{
	GList *result = NULL;

	CRM_CHECK((id != NULL) && (scheduler != NULL), return NULL);
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	result = add_rsc_if_matching(result, (pcmk_resource_t *) iter->data,
	id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for a resource
	*
	* \param[in,out] data Resource to set assignment methods for
	* \param[in] user_data Ignored
	*/
	static void
	set_assignment_methods_for_rsc(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;

	rsc->cmds = &assignment_methods[rsc->variant];
	g_list_foreach(rsc->children, set_assignment_methods_for_rsc, NULL);
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for all resources
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__set_assignment_methods(pcmk_scheduler_t *scheduler)
	{
	g_list_foreach(scheduler->resources, set_assignment_methods_for_rsc, NULL);
	}

	/*!
	* \internal
	* \brief Wrapper for colocated_resources() method for readability
	*
	* \param[in] rsc Resource to add to colocated list
	* \param[in] orig_rsc Resource originally requested
	* \param[in,out] list Pointer to list to add to
	*
	* \return (Possibly new) head of list
	*/
	static inline void
	add_colocated_resources(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	list = rsc->cmds->colocated_resources(rsc, orig_rsc, list);
	}

	// Shared implementation of pcmk_assignment_methods_t:colocated_resources()
	GList *
	pcmk__colocated_resources(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList *colocated_rscs)
	{
	const GList *iter = NULL;
	GList *colocations = NULL;

	if (orig_rsc == NULL) {
	orig_rsc = rsc;
	}

	if ((rsc == NULL) \|\| (g_list_find(colocated_rscs, rsc) != NULL)) {
	return colocated_rscs;
	}

	pcmk__rsc_trace(orig_rsc, "%s is in colocation chain with %s",
	rsc->id, orig_rsc->id);
	colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);

	// Follow colocations where this resource is the dependent resource
	colocations = pcmk__this_with_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pcmk_resource_t *primary = constraint->primary;

	if (primary == orig_rsc) {
	continue; // Break colocation loop
	}

	if ((constraint->score == INFINITY) &&
	(pcmk__colocation_affects(rsc, primary, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(primary, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	// Follow colocations where this resource is the primary resource
	colocations = pcmk__with_this_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pcmk_resource_t *dependent = constraint->dependent;

	if (dependent == orig_rsc) {
	continue; // Break colocation loop
	}

	if (pe_rsc_is_clone(rsc) && !pe_rsc_is_clone(dependent)) {
	continue; // We can't be sure whether dependent will be colocated
	}

	if ((constraint->score == INFINITY) &&
	(pcmk__colocation_affects(dependent, rsc, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(dependent, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	return colocated_rscs;
	}

	// No-op function for variants that don't need to implement add_graph_meta()
	void
	pcmk__noop_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml)
	{
	}

	/*!
	* \internal
	* \brief Output a summary of scheduled actions for a resource
	*
	* \param[in,out] rsc Resource to output actions for
	*/
	void
	pcmk__output_resource_actions(pcmk_resource_t *rsc)
	{
	pcmk_node_t *next = NULL;
	pcmk_node_t *current = NULL;
	pcmk__output_t *out = NULL;

	CRM_ASSERT(rsc != NULL);

	out = rsc->cluster->priv;
	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	child->cmds->output_actions(child);
	}
	return;
	}

	next = rsc->allocated_to;
	if (rsc->running_on) {
	current = pcmk__current_node(rsc);
	if (rsc->role == pcmk_role_stopped) {
	/* This can occur when resources are being recovered because
	* the current role can change in pcmk__primitive_create_actions()
	*/
	rsc->role = pcmk_role_started;
	}
	}

	if ((current == NULL) && pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	/* Don't log stopped orphans */
	return;
	}

	out->message(out, "rsc-action", rsc, current, next);
	}

	/*!
	* \internal
	* \brief Add a resource to a node's list of assigned resources
	*
	* \param[in,out] node Node to add resource to
	* \param[in] rsc Resource to add
	*/
	static inline void
	add_assigned_resource(pcmk_node_t node, pcmk_resource_t rsc)
	{
	node->details->allocated_rsc = g_list_prepend(node->details->allocated_rsc,
	rsc);
	}

	/*!
	* \internal
	* \brief Assign a specified resource (of any variant) to a node
	*
	* Assign a specified resource and its children (if any) to a specified node, if
	* the node can run the resource (or unconditionally, if \p force is true). Mark
	* the resources as no longer provisional.
	*
	* If a resource can't be assigned (or \p node is \c NULL), unassign any
	* previous assignment. If \p stop_if_fail is \c true, set next role to stopped
	* and update any existing actions scheduled for the resource.
	*
	* \param[in,out] rsc Resource to assign
	* \param[in,out] node Node to assign \p rsc to
	* \param[in] force If true, assign to \p node even if unavailable
	* \param[in] stop_if_fail If \c true and either \p rsc can't be assigned
	* or \p chosen is \c NULL, set next role to
	* stopped and update existing actions (if \p rsc
	* is not a primitive, this applies to its
	* primitive descendants instead)
	*
	* \return \c true if the assignment of \p rsc changed, or \c false otherwise
	*
	* \note Assigning a resource to the NULL node using this function is different
	* from calling pcmk__unassign_resource(), in that it may also update any
	* actions created for the resource.
	* \note The \c pcmk_assignment_methods_t:assign() method is preferred, unless
	* a resource should be assigned to the \c NULL node or every resource in
	* a tree should be assigned to the same node.
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	bool
	pcmk__assign_resource(pcmk_resource_t rsc, pcmk_node_t node, bool force,
	bool stop_if_fail)
	{
	bool changed = false;

	CRM_ASSERT(rsc != NULL);

	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t *child_rsc = iter->data;

	changed \|= pcmk__assign_resource(child_rsc, node, force,
	stop_if_fail);
	}
	return changed;
	}

	// Assigning a primitive

	if (!force && (node != NULL)
	&& ((node->weight < 0)
	// Allow graph to assume that guest node connections will come up
	\|\| (!pcmk__node_available(node, true, false)
	&& !pe__is_guest_node(node)))) {

	pcmk__rsc_debug(rsc,
	"All nodes for resource %s are unavailable, unclean or "
	"shutting down (%s can%s run resources, with score %s)",
	rsc->id, pcmk__node_name(node),
	(pcmk__node_available(node, true, false)? "" : "not"),
	pcmk_readable_score(node->weight));

	if (stop_if_fail) {
	pe__set_next_role(rsc, pcmk_role_stopped, "node availability");
	}
	node = NULL;
	}

	if (rsc->allocated_to != NULL) {
	- changed = !pe__same_node(rsc->allocated_to, node);
	+ changed = !pcmk__same_node(rsc->allocated_to, node);
	} else {
	changed = (node != NULL);
	}
	pcmk__unassign_resource(rsc);
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unassigned);

	if (node == NULL) {
	char *rc_stopped = NULL;

	pcmk__rsc_debug(rsc, "Could not assign %s to a node", rsc->id);

	if (!stop_if_fail) {
	return changed;
	}
	pe__set_next_role(rsc, pcmk_role_stopped, "unable to assign");

	for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t op = (pcmk_action_t ) iter->data;

	pcmk__rsc_debug(rsc, "Updating %s for %s assignment failure",
	op->uuid, rsc->id);

	if (pcmk__str_eq(op->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	pcmk__clear_action_flags(op, pcmk_action_optional);

	} else if (pcmk__str_eq(op->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	pcmk__clear_action_flags(op, pcmk_action_runnable);

	} else {
	// Cancel recurring actions, unless for stopped state
	const char *interval_ms_s = NULL;
	const char *target_rc_s = NULL;

	interval_ms_s = g_hash_table_lookup(op->meta,
	PCMK_META_INTERVAL);
	target_rc_s = g_hash_table_lookup(op->meta,
	PCMK__META_OP_TARGET_RC);
	if (rc_stopped == NULL) {
	rc_stopped = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
	}

	if (!pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)
	&& !pcmk__str_eq(rc_stopped, target_rc_s, pcmk__str_none)) {

	pcmk__clear_action_flags(op, pcmk_action_runnable);
	}
	}
	}
	free(rc_stopped);
	return changed;
	}

	pcmk__rsc_debug(rsc, "Assigning %s to %s", rsc->id, pcmk__node_name(node));
	rsc->allocated_to = pe__copy_node(node);

	add_assigned_resource(node, rsc);
	node->details->num_resources++;
	node->count++;
	pcmk__consume_node_capacity(node->details->utilization, rsc);

	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_show_utilization)) {
	pcmk__output_t *out = rsc->cluster->priv;

	out->message(out, "resource-util", rsc, node, __func__);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Remove any node assignment from a specified resource and its children
	*
	* If a specified resource has been assigned to a node, remove that assignment
	* and mark the resource as provisional again.
	*
	* \param[in,out] rsc Resource to unassign
	*
	* \note This function is called recursively on \p rsc and its children.
	*/
	void
	pcmk__unassign_resource(pcmk_resource_t *rsc)
	{
	pcmk_node_t *old = rsc->allocated_to;

	if (old == NULL) {
	crm_info("Unassigning %s", rsc->id);
	} else {
	crm_info("Unassigning %s from %s", rsc->id, pcmk__node_name(old));
	}

	pcmk__set_rsc_flags(rsc, pcmk_rsc_unassigned);

	if (rsc->children == NULL) {
	if (old == NULL) {
	return;
	}
	rsc->allocated_to = NULL;

	/* We're going to free the pcmk_node_t, but its details member is shared
	* and will remain, so update that appropriately first.
	*/
	old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc,
	rsc);
	old->details->num_resources--;
	pcmk__release_node_capacity(old->details->utilization, rsc);
	free(old);
	return;
	}

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk__unassign_resource((pcmk_resource_t *) iter->data);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has reached its migration threshold on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node Node to check
	* \param[out] failed If threshold has been reached, this will be set to
	* resource that failed (possibly a parent of \p rsc)
	*
	* \return true if the migration threshold has been reached, false otherwise
	*/
	bool
	pcmk__threshold_reached(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_resource_t **failed)
	{
	int fail_count, remaining_tries;
	pcmk_resource_t *rsc_to_ban = rsc;

	// Migration threshold of 0 means never force away
	if (rsc->migration_threshold == 0) {
	return false;
	}

	// If we're ignoring failures, also ignore the migration threshold
	if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	return false;
	}

	// If there are no failures, there's no need to force away
	fail_count = pe_get_failcount(node, rsc, NULL,
	pcmk__fc_effective\|pcmk__fc_fillers, NULL);
	if (fail_count <= 0) {
	return false;
	}

	// If failed resource is anonymous clone instance, we'll force clone away
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	rsc_to_ban = uber_parent(rsc);
	}

	// How many more times recovery will be tried on this node
	remaining_tries = rsc->migration_threshold - fail_count;

	if (remaining_tries <= 0) {
	pcmk__sched_warn("%s cannot run on %s due to reaching migration "
	"threshold (clean up resource to allow again)"
	CRM_XS " failures=%d "
	PCMK_META_MIGRATION_THRESHOLD "=%d",
	rsc_to_ban->id, pcmk__node_name(node), fail_count,
	rsc->migration_threshold);
	if (failed != NULL) {
	*failed = rsc_to_ban;
	}
	return true;
	}

	crm_info("%s can fail %d more time%s on "
	"%s before reaching migration threshold (%d)",
	rsc_to_ban->id, remaining_tries, pcmk__plural_s(remaining_tries),
	pcmk__node_name(node), rsc->migration_threshold);
	return false;
	}

	/*!
	* \internal
	* \brief Get a node's score
	*
	* \param[in] node Node with ID to check
	* \param[in] nodes List of nodes to look for \p node score in
	*
	* \return Node's score, or -INFINITY if not found
	*/
	static int
	get_node_score(const pcmk_node_t node, GHashTable nodes)
	{
	pcmk_node_t *found_node = NULL;

	if ((node != NULL) && (nodes != NULL)) {
	found_node = g_hash_table_lookup(nodes, node->details->id);
	}
	return (found_node == NULL)? -INFINITY : found_node->weight;
	}

	/*!
	* \internal
	* \brief Compare two resources according to which should be assigned first
	*
	* \param[in] a First resource to compare
	* \param[in] b Second resource to compare
	* \param[in] data Sorted list of all nodes in cluster
	*
	* \return -1 if \p a should be assigned before \b, 0 if they are equal,
	* or +1 if \p a should be assigned after \b
	*/
	static gint
	cmp_resources(gconstpointer a, gconstpointer b, gpointer data)
	{
	/* GLib insists that this function require gconstpointer arguments, but we
	* make a small, temporary change to each argument (setting the
	* pe_rsc_merging flag) during comparison
	*/
	pcmk_resource_t resource1 = (pcmk_resource_t ) a;
	pcmk_resource_t resource2 = (pcmk_resource_t ) b;
	const GList *nodes = data;

	int rc = 0;
	int r1_score = -INFINITY;
	int r2_score = -INFINITY;
	pcmk_node_t *r1_node = NULL;
	pcmk_node_t *r2_node = NULL;
	GHashTable *r1_nodes = NULL;
	GHashTable *r2_nodes = NULL;
	const char *reason = NULL;

	// Resources with highest priority should be assigned first
	reason = "priority";
	r1_score = resource1->priority;
	r2_score = resource2->priority;
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// We need nodes to make any other useful comparisons
	reason = "no node list";
	if (nodes == NULL) {
	goto done;
	}

	// Calculate and log node scores
	resource1->cmds->add_colocated_node_scores(resource1, NULL, resource1->id,
	&r1_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	resource2->cmds->add_colocated_node_scores(resource2, NULL, resource2->id,
	&r2_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	pe__show_node_scores(true, NULL, resource1->id, r1_nodes,
	resource1->cluster);
	pe__show_node_scores(true, NULL, resource2->id, r2_nodes,
	resource2->cluster);

	// The resource with highest score on its current node goes first
	reason = "current location";
	if (resource1->running_on != NULL) {
	r1_node = pcmk__current_node(resource1);
	}
	if (resource2->running_on != NULL) {
	r2_node = pcmk__current_node(resource2);
	}
	r1_score = get_node_score(r1_node, r1_nodes);
	r2_score = get_node_score(r2_node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// Otherwise a higher score on any node will do
	reason = "score";
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	const pcmk_node_t node = (const pcmk_node_t ) iter->data;

	r1_score = get_node_score(node, r1_nodes);
	r2_score = get_node_score(node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}
	}

	done:
	crm_trace("%s (%d)%s%s %c %s (%d)%s%s: %s",
	resource1->id, r1_score,
	((r1_node == NULL)? "" : " on "),
	((r1_node == NULL)? "" : r1_node->details->id),
	((rc < 0)? '>' : ((rc > 0)? '<' : '=')),
	resource2->id, r2_score,
	((r2_node == NULL)? "" : " on "),
	((r2_node == NULL)? "" : r2_node->details->id),
	reason);
	if (r1_nodes != NULL) {
	g_hash_table_destroy(r1_nodes);
	}
	if (r2_nodes != NULL) {
	g_hash_table_destroy(r2_nodes);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Sort resources in the order they should be assigned to nodes
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__sort_resources(pcmk_scheduler_t *scheduler)
	{
	GList *nodes = g_list_copy(scheduler->nodes);

	nodes = pcmk__sort_nodes(nodes, NULL);
	scheduler->resources = g_list_sort_with_data(scheduler->resources,
	cmp_resources, nodes);
	g_list_free(nodes);
	}
	diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
	index d89ddfdbeb..1e03ac0a72 100644
	--- a/lib/pengine/bundle.c
	+++ b/lib/pengine/bundle.c
	@@ -1,2233 +1,2233 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <ctype.h>
	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	enum pe__bundle_mount_flags {
	pe__bundle_mount_none = 0x00,

	// mount instance-specific subdirectory rather than source directly
	pe__bundle_mount_subdir = 0x01
	};

	typedef struct {
	char *source;
	char *target;
	char *options;
	uint32_t flags; // bitmask of pe__bundle_mount_flags
	} pe__bundle_mount_t;

	typedef struct {
	char *source;
	char *target;
	} pe__bundle_port_t;

	enum pe__container_agent {
	PE__CONTAINER_AGENT_UNKNOWN,
	PE__CONTAINER_AGENT_DOCKER,
	PE__CONTAINER_AGENT_RKT,
	PE__CONTAINER_AGENT_PODMAN,
	};

	#define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
	#define PE__CONTAINER_AGENT_DOCKER_S "docker"
	#define PE__CONTAINER_AGENT_RKT_S "rkt"
	#define PE__CONTAINER_AGENT_PODMAN_S "podman"

	typedef struct pe__bundle_variant_data_s {
	int promoted_max;
	int nreplicas;
	int nreplicas_per_host;
	char *prefix;
	char *image;
	const char *ip_last;
	char *host_network;
	char *host_netmask;
	char *control_port;
	char *container_network;
	char *ip_range_start;
	gboolean add_host;
	gchar *container_host_options;
	char *container_command;
	char *launcher_options;
	const char *attribute_target;

	pcmk_resource_t *child;

	GList replicas; // pcmk__bundle_replica_t
	GList ports; // pe__bundle_port_t
	GList mounts; // pe__bundle_mount_t

	enum pe__container_agent agent_type;
	} pe__bundle_variant_data_t;

	#define get_bundle_variant_data(data, rsc) \
	CRM_ASSERT(rsc != NULL); \
	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_bundle); \
	CRM_ASSERT(rsc->variant_opaque != NULL); \
	data = (pe__bundle_variant_data_t *) rsc->variant_opaque;

	/*!
	* \internal
	* \brief Get maximum number of bundle replicas allowed to run
	*
	* \param[in] rsc Bundle or bundled resource to check
	*
	* \return Maximum replicas for bundle corresponding to \p rsc
	*/
	int
	pe__bundle_max(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->nreplicas;
	}

	/*!
	* \internal
	* \brief Get the resource inside a bundle
	*
	* \param[in] bundle Bundle to check
	*
	* \return Resource inside \p bundle if any, otherwise NULL
	*/
	pcmk_resource_t *
	pe__bundled_resource(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->child;
	}

	/*!
	* \internal
	* \brief Get containerized resource corresponding to a given bundle container
	*
	* \param[in] instance Collective instance that might be a bundle container
	*
	* \return Bundled resource instance inside \p instance if it is a bundle
	* container instance, otherwise NULL
	*/
	const pcmk_resource_t *
	pe__get_rsc_in_container(const pcmk_resource_t *instance)
	{
	const pe__bundle_variant_data_t *data = NULL;
	const pcmk_resource_t *top = pe__const_top_resource(instance, true);

	if ((top == NULL) \|\| (top->variant != pcmk_rsc_variant_bundle)) {
	return NULL;
	}
	get_bundle_variant_data(data, top);

	for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	const pcmk__bundle_replica_t *replica = iter->data;

	if (instance == replica->container) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a given node is created by a bundle
	*
	* \param[in] bundle Bundle resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is an instance of \p bundle, otherwise false
	*/
	bool
	pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	- if (pe__same_node(node, replica->node)) {
	+ if (pcmk__same_node(node, replica->node)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Get the container of a bundle's first replica
	*
	* \param[in] bundle Bundle resource to get container for
	*
	* \return Container resource from first replica of \p bundle if any,
	* otherwise NULL
	*/
	pcmk_resource_t *
	pe__first_container(const pcmk_resource_t *bundle)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;
	const pcmk__bundle_replica_t *replica = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	if (bundle_data->replicas == NULL) {
	return NULL;
	}
	replica = bundle_data->replicas->data;
	return replica->container;
	}

	/*!
	* \internal
	* \brief Iterate over bundle replicas
	*
	* \param[in,out] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Iterate over const bundle replicas
	*
	* \param[in] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (const GList *iter = bundle_data->replicas; iter != NULL;
	iter = iter->next) {

	if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	static char *
	next_ip(const char *last_ip)
	{
	unsigned int oct1 = 0;
	unsigned int oct2 = 0;
	unsigned int oct3 = 0;
	unsigned int oct4 = 0;
	int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);

	if (rc != 4) {
	/@ TODO check for IPv6 /
	return NULL;

	} else if (oct3 > 253) {
	return NULL;

	} else if (oct4 > 253) {
	++oct3;
	oct4 = 1;

	} else {
	++oct4;
	}

	return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
	}

	static void
	allocate_ip(pe__bundle_variant_data_t data, pcmk__bundle_replica_t replica,
	GString *buffer)
	{
	if(data->ip_range_start == NULL) {
	return;

	} else if(data->ip_last) {
	replica->ipaddr = next_ip(data->ip_last);

	} else {
	replica->ipaddr = strdup(data->ip_range_start);
	}

	data->ip_last = replica->ipaddr;
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (data->add_host) {
	g_string_append_printf(buffer, " --add-host=%s-%d:%s",
	data->prefix, replica->offset,
	replica->ipaddr);
	} else {
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	}
	break;

	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	break;

	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	}

	static xmlNode *
	create_resource(const char name, const char provider, const char *kind)
	{
	xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	crm_xml_add(rsc, PCMK_XA_ID, name);
	crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
	crm_xml_add(rsc, PCMK_XA_TYPE, kind);

	return rsc;
	}

	/*!
	* \internal
	* \brief Check whether cluster can manage resource inside container
	*
	* \param[in,out] data Container variant data
	*
	* \return TRUE if networking configuration is acceptable, FALSE otherwise
	*
	* \note The resource is manageable if an IP range or control port has been
	* specified. If a control port is used without an IP range, replicas per
	* host must be 1.
	*/
	static bool
	valid_network(pe__bundle_variant_data_t *data)
	{
	if(data->ip_range_start) {
	return TRUE;
	}
	if(data->control_port) {
	if(data->nreplicas_per_host > 1) {
	pcmk__config_err("Specifying the 'control-port' for %s requires "
	"'replicas-per-host=1'", data->prefix);
	data->nreplicas_per_host = 1;
	// @TODO to be sure:
	// pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}
	return TRUE;
	}
	return FALSE;
	}

	static int
	create_ip_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if(data->ip_range_start) {
	char *id = NULL;
	xmlNode *xml_ip = NULL;
	xmlNode *xml_obj = NULL;

	id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
	crm_xml_sanitize_id(id);
	xml_ip = create_resource(id, "heartbeat", "IPaddr2");
	free(id);

	xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d",
	data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
	if(data->host_network) {
	crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
	}

	if(data->host_netmask) {
	crm_create_nvpair_xml(xml_obj, NULL,
	"cidr_netmask", data->host_netmask);

	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
	}

	xml_obj = create_xml_node(xml_ip, "operations");
	crm_create_op_xml(xml_obj, ID(xml_ip), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?

	if (pe__unpack_resource(xml_ip, &replica->ip, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	parent->children = g_list_append(parent->children, replica->ip);
	}
	return pcmk_rc_ok;
	}

	static const char*
	container_agent_str(enum pe__container_agent t)
	{
	switch (t) {
	case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
	case PE__CONTAINER_AGENT_RKT: return PE__CONTAINER_AGENT_RKT_S;
	case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	return PE__CONTAINER_AGENT_UNKNOWN_S;
	}

	static int
	create_container_resource(pcmk_resource_t *parent,
	const pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	char *id = NULL;
	xmlNode *xml_container = NULL;
	xmlNode *xml_obj = NULL;

	// Agent-specific
	const char *hostname_opt = NULL;
	const char *env_opt = NULL;
	const char *agent_str = NULL;
	int volid = 0; // rkt-only

	GString *buffer = NULL;
	GString *dbuffer = NULL;

	// Where syntax differences are drop-in replacements, set them now
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	hostname_opt = "-h ";
	env_opt = "-e ";
	break;
	case PE__CONTAINER_AGENT_RKT:
	hostname_opt = "--hostname=";
	env_opt = "--environment=";
	break;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	return pcmk_rc_unpack_error;
	}
	agent_str = container_agent_str(data->agent_type);

	buffer = g_string_sized_new(4096);

	id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
	replica->offset);
	crm_xml_sanitize_id(id);
	xml_container = create_resource(id, "heartbeat", agent_str);
	free(id);

	xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
	crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
	crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
	crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);

	if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
	g_string_append(buffer, " --restart=no");
	}

	/* Set a container hostname only if we have an IP to map it to. The user can
	* set -h or --uts=host themselves if they want a nicer name for logs, but
	* this makes applications happy who need their hostname to match the IP
	* they bind to.
	*/
	if (data->ip_range_start != NULL) {
	g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
	replica->offset);
	}
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);

	if (data->container_network != NULL) {
	pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
	}

	if (data->control_port != NULL) {
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
	data->control_port, NULL);
	} else {
	g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
	env_opt, DEFAULT_REMOTE_PORT);
	}

	for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
	pe__bundle_mount_t mount = (pe__bundle_mount_t ) iter->data;
	char *source = NULL;

	if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
	source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
	replica->offset);
	pcmk__add_separated_word(&dbuffer, 1024, source, ",");
	}

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	pcmk__g_strcat(buffer,
	" -v ", pcmk__s(source, mount->source),
	":", mount->target, NULL);

	if (mount->options != NULL) {
	pcmk__g_strcat(buffer, ":", mount->options, NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer,
	" --volume vol%d,kind=host,"
	"source=%s%s%s "
	"--mount volume=vol%d,target=%s",
	volid, pcmk__s(source, mount->source),
	(mount->options != NULL)? "," : "",
	pcmk__s(mount->options, ""),
	volid, mount->target);
	volid++;
	break;
	default:
	break;
	}
	free(source);
	}

	for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
	pe__bundle_port_t port = (pe__bundle_port_t ) iter->data;

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" -p ", replica->ipaddr, ":", port->source,
	":", port->target, NULL);

	} else if (!pcmk__str_eq(data->container_network, "host",
	pcmk__str_none)) {
	// No need to do port mapping if net == host
	pcmk__g_strcat(buffer,
	" -p ", port->source, ":", port->target,
	NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" --port=", port->target,
	":", replica->ipaddr, ":", port->source,
	NULL);
	} else {
	pcmk__g_strcat(buffer,
	" --port=", port->target, ":", port->source,
	NULL);
	}
	break;
	default:
	break;
	}
	}

	/* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
	* it would cause restarts during rolling upgrades.
	*
	* In a previous version of the container resource creation logic, if
	* data->launcher_options is not NULL, we append
	* (" %s", data->launcher_options) even if data->launcher_options is an
	* empty string. Likewise for data->container_host_options. Using
	*
	* pcmk__add_word(buffer, 0, data->launcher_options)
	*
	* removes that extra trailing space, causing a resource definition change.
	*/
	if (data->launcher_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
	}

	if (data->container_host_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
	}

	crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
	(const char *) buffer->str);
	g_string_free(buffer, TRUE);

	crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
	(dbuffer != NULL)? (const char *) dbuffer->str : "");
	if (dbuffer != NULL) {
	g_string_free(dbuffer, TRUE);
	}

	if (replica->child != NULL) {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	SBIN_DIR "/pacemaker-remoted");
	}

	/* TODO: Allow users to specify their own?
	*
	* We just want to know if the container is alive; we'll monitor the
	* child independently.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	#if 0
	/* @TODO Consider supporting the use case where we can start and stop
	* resources, but not proxy local commands (such as setting node
	* attributes), by running the local executor in stand-alone mode.
	* However, this would probably be better done via ACLs as with other
	* Pacemaker Remote nodes.
	*/
	} else if ((child != NULL) && data->untrusted) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	CRM_DAEMON_DIR "/pacemaker-execd");
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
	CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
	#endif
	} else {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	}

	/* TODO: Allow users to specify their own?
	*
	* We don't know what's in the container, so we just want to know if it
	* is alive.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	}

	xml_obj = create_xml_node(xml_container, "operations");
	crm_create_op_xml(xml_obj, ID(xml_container), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?
	if (pe__unpack_resource(xml_container, &replica->container, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}
	pcmk__set_rsc_flags(replica->container, pcmk_rsc_replica_container);
	parent->children = g_list_append(parent->children, replica->container);

	return pcmk_rc_ok;
	}

	/*!
	* \brief Ban a node from a resource's (and its children's) allowed nodes list
	*
	* \param[in,out] rsc Resource to modify
	* \param[in] uname Name of node to ban
	*/
	static void
	disallow_node(pcmk_resource_t rsc, const char uname)
	{
	gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);

	if (match) {
	((pcmk_node_t *) match)->weight = -INFINITY;
	((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
	}
	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
	}
	}

	static int
	create_remote_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if (replica->child && valid_network(data)) {
	GHashTableIter gIter;
	pcmk_node_t *node = NULL;
	xmlNode *xml_remote = NULL;
	char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
	char *port_s = NULL;
	const char *uname = NULL;
	const char *connect_name = NULL;

	if (pe_find_resource(parent->cluster->resources, id) != NULL) {
	free(id);
	// The biggest hammer we have
	id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
	replica->child->id, replica->offset);
	//@TODO return error instead of asserting?
	CRM_ASSERT(pe_find_resource(parent->cluster->resources,
	id) == NULL);
	}

	/* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
	* connection does not have its own IP is a magic string that we use to
	* support nested remotes (i.e. a bundle running on a remote node).
	*/
	connect_name = (replica->ipaddr? replica->ipaddr : "#uname");

	if (data->control_port == NULL) {
	port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}

	/* This sets replica->container as replica->remote's container, which is
	* similar to what happens with guest nodes. This is how the scheduler
	* knows that the bundle node is fenced by recovering the container, and
	* that remote should be ordered relative to the container.
	*/
	xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
	NULL, NULL, NULL,
	connect_name, (data->control_port?
	data->control_port : port_s));
	free(port_s);

	/* Abandon our created ID, and pull the copy from the XML, because we
	* need something that will get freed during scheduler data cleanup to
	* use as the node ID and uname.
	*/
	free(id);
	id = NULL;
	uname = ID(xml_remote);

	/* Ensure a node has been created for the guest (it may have already
	* been, if it has a permanent node attribute), and ensure its weight is
	* -INFINITY so no other resources can run on it.
	*/
	node = pe_find_node(parent->cluster->nodes, uname);
	if (node == NULL) {
	node = pe_create_node(uname, uname, "remote", "-INFINITY",
	parent->cluster);
	} else {
	node->weight = -INFINITY;
	}
	node->rsc_discover_mode = pcmk_probe_never;

	/* unpack_remote_nodes() ensures that each remote node and guest node
	* has a pcmk_node_t entry. Ideally, it would do the same for bundle
	* nodes. Unfortunately, a bundle has to be mostly unpacked before it's
	* obvious what nodes will be needed, so we do it just above.
	*
	* Worse, that means that the node may have been utilized while
	* unpacking other resources, without our weight correction. The most
	* likely place for this to happen is when pe__unpack_resource() calls
	* resource_location() to set a default score in symmetric clusters.
	* This adds a node copy to each resource's allowed nodes, and these
	* copies will have the wrong weight.
	*
	* As a hacky workaround, fix those copies here.
	*
	* @TODO Possible alternative: ensure bundles are unpacked before other
	* resources, so the weight is correct before any copies are made.
	*/
	g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
	(gpointer) uname);

	replica->node = pe__copy_node(node);
	replica->node->weight = 500;
	replica->node->rsc_discover_mode = pcmk_probe_exclusive;

	/* Ensure the node shows up as allowed and with the correct discovery set */
	if (replica->child->allowed_nodes != NULL) {
	g_hash_table_destroy(replica->child->allowed_nodes);
	}
	replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
	g_hash_table_insert(replica->child->allowed_nodes,
	(gpointer) replica->node->details->id,
	pe__copy_node(replica->node));

	{
	pcmk_node_t *copy = pe__copy_node(replica->node);
	copy->weight = -INFINITY;
	g_hash_table_insert(replica->child->parent->allowed_nodes,
	(gpointer) replica->node->details->id, copy);
	}
	if (pe__unpack_resource(xml_remote, &replica->remote, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
	if (pe__is_guest_or_remote_node(node)) {
	/* Remote resources can only run on 'normal' cluster node */
	node->weight = -INFINITY;
	}
	}

	replica->node->details->remote_rsc = replica->remote;

	// Ensure pe__is_guest_node() functions correctly immediately
	replica->remote->container = replica->container;

	/* A bundle's #kind is closer to "container" (guest node) than the
	* "remote" set by pe_create_node().
	*/
	g_hash_table_insert(replica->node->details->attrs,
	strdup(CRM_ATTR_KIND), strdup("container"));

	/* One effect of this is that setup_container() will add
	* replica->remote to replica->container's fillers, which will make
	* pe__resource_contains_guest_node() true for replica->container.
	*
	* replica->child does NOT get added to replica->container's fillers.
	* The only noticeable effect if it did would be for its fail count to
	* be taken into account when checking replica->container's migration
	* threshold.
	*/
	parent->children = g_list_append(parent->children, replica->remote);
	}
	return pcmk_rc_ok;
	}

	static int
	create_replica_resources(pcmk_resource_t *parent,
	pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	int rc = pcmk_rc_ok;

	rc = create_container_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_ip_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_remote_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
	add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
	}

	if (replica->remote != NULL) {
	/*
	* Allow the remote connection resource to be allocated to a
	* different node than the one on which the container is active.
	*
	* This makes it possible to have Pacemaker Remote nodes running
	* containers with pacemaker-remoted inside in order to start
	* services inside those containers.
	*/
	pcmk__set_rsc_flags(replica->remote, pcmk_rsc_remote_nesting_allowed);
	}
	return rc;
	}

	static void
	mount_add(pe__bundle_variant_data_t bundle_data, const char source,
	const char target, const char options, uint32_t flags)
	{
	pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));

	CRM_ASSERT(mount != NULL);
	mount->source = strdup(source);
	mount->target = strdup(target);
	pcmk__str_update(&mount->options, options);
	mount->flags = flags;
	bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
	}

	static void
	mount_free(pe__bundle_mount_t *mount)
	{
	free(mount->source);
	free(mount->target);
	free(mount->options);
	free(mount);
	}

	static void
	port_free(pe__bundle_port_t *port)
	{
	free(port->source);
	free(port->target);
	free(port);
	}

	static pcmk__bundle_replica_t *
	replica_for_remote(pcmk_resource_t *remote)
	{
	pcmk_resource_t *top = remote;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (top == NULL) {
	return NULL;
	}

	while (top->parent != NULL) {
	top = top->parent;
	}

	get_bundle_variant_data(bundle_data, top);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->remote == remote) {
	return replica;
	}
	}
	CRM_LOG_ASSERT(FALSE);
	return NULL;
	}

	bool
	pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
	{
	const char *value;
	GHashTable *params = NULL;

	if (rsc == NULL) {
	return false;
	}

	// Use NULL node since pcmk__bundle_expand() uses that to set value
	params = pe_rsc_params(rsc, NULL, rsc->cluster);
	value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);

	return pcmk__str_eq(value, "#uname", pcmk__str_casei)
	&& xml_contains_remote_node(rsc->xml);
	}

	const char *
	pe__add_bundle_remote_name(pcmk_resource_t rsc, pcmk_scheduler_t scheduler,
	xmlNode xml, const char field)
	{
	// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside

	pcmk_node_t *node = NULL;
	pcmk__bundle_replica_t *replica = NULL;

	if (!pe__bundle_needs_remote_name(rsc)) {
	return NULL;
	}

	replica = replica_for_remote(rsc);
	if (replica == NULL) {
	return NULL;
	}

	node = replica->container->allocated_to;
	if (node == NULL) {
	/* If it won't be running anywhere after the
	* transition, go with where it's running now.
	*/
	node = pcmk__current_node(replica->container);
	}

	if(node == NULL) {
	crm_trace("Cannot determine address for bundle connection %s", rsc->id);
	return NULL;
	}

	crm_trace("Setting address for bundle connection %s to bundle host %s",
	rsc->id, pcmk__node_name(node));
	if(xml != NULL && field != NULL) {
	crm_xml_add(xml, field, node->details->uname);
	}

	return node->details->uname;
	}

	#define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do { \
	flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Bundle mount", ID(mount_xml), flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	gboolean
	pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_resource = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	bool need_log_mount = TRUE;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
	rsc->variant_opaque = bundle_data;
	bundle_data->prefix = strdup(rsc->id);

	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
	} else {
	return FALSE;
	}
	}
	}

	// Use 0 for default, minimum, and invalid PCMK_META_PROMOTED_MAX
	value = crm_element_value(xml_obj, PCMK_META_PROMOTED_MAX);
	if (value == NULL) {
	// @COMPAT deprecated since 2.0.0
	value = crm_element_value(xml_obj, "masters");
	}
	pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);

	/* Default replicas to PCMK_META_PROMOTED_MAX if it was specified and 1
	* otherwise
	*/
	value = crm_element_value(xml_obj, "replicas");
	if ((value == NULL) && (bundle_data->promoted_max > 0)) {
	bundle_data->nreplicas = bundle_data->promoted_max;
	} else {
	pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
	}

	/*
	* Communication between containers on the same host via the
	* floating IPs only works if the container is started with:
	* --userland-proxy=false --ip-masq=false
	*/
	value = crm_element_value(xml_obj, "replicas-per-host");
	pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
	if (bundle_data->nreplicas_per_host == 1) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}

	bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
	bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
	bundle_data->image = crm_element_value_copy(xml_obj, "image");
	bundle_data->container_network = crm_element_value_copy(xml_obj, "network");

	xml_obj = first_named_child(rsc->xml, "network");
	if(xml_obj) {

	bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
	bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
	bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
	bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
	value = crm_element_value(xml_obj, "add-host");
	if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
	bundle_data->add_host = TRUE;
	}

	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
	port->source = crm_element_value_copy(xml_child, "port");

	if(port->source == NULL) {
	port->source = crm_element_value_copy(xml_child, "range");
	} else {
	port->target = crm_element_value_copy(xml_child, "internal-port");
	}

	if(port->source != NULL && strlen(port->source) > 0) {
	if(port->target == NULL) {
	port->target = strdup(port->source);
	}
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	} else {
	pcmk__config_err("Invalid port directive %s", ID(xml_child));
	port_free(port);
	}
	}
	}

	xml_obj = first_named_child(rsc->xml, "storage");
	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	const char *source = crm_element_value(xml_child, "source-dir");
	const char *target = crm_element_value(xml_child, "target-dir");
	const char *options = crm_element_value(xml_child, "options");
	int flags = pe__bundle_mount_none;

	if (source == NULL) {
	source = crm_element_value(xml_child, "source-dir-root");
	pe__set_bundle_mount_flags(xml_child, flags,
	pe__bundle_mount_subdir);
	}

	if (source && target) {
	mount_add(bundle_data, source, target, options, flags);
	if (strcmp(target, "/var/log") == 0) {
	need_log_mount = FALSE;
	}
	} else {
	pcmk__config_err("Invalid mount directive %s", ID(xml_child));
	}
	}

	xml_obj = first_named_child(rsc->xml, "primitive");
	if (xml_obj && valid_network(bundle_data)) {
	char *value = NULL;
	xmlNode *xml_set = NULL;

	xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);

	/* @COMPAT We no longer use the <master> tag, but we need to keep it as
	* part of the resource name, so that bundles don't restart in a rolling
	* upgrade. (It also avoids needing to change regression tests.)
	*/
	crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
	(bundle_data->promoted_max? "master"
	: (const char *)xml_resource->name));

	xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
	crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);

	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_ORDERED, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->nreplicas);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
	free(value);

	value = pcmk__itoa(bundle_data->nreplicas_per_host);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
	free(value);

	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
	pcmk__btoa(bundle_data->nreplicas_per_host > 1));

	if (bundle_data->promoted_max) {
	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->promoted_max);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
	free(value);
	}

	//crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
	add_node_copy(xml_resource, xml_obj);

	} else if(xml_obj) {
	pcmk__config_err("Cannot control %s inside %s without either "
	"ip-range-start or control-port",
	rsc->id, ID(xml_obj));
	return FALSE;
	}

	if(xml_resource) {
	int lpc = 0;
	GList *childIter = NULL;
	pe__bundle_port_t *port = NULL;
	GString *buffer = NULL;

	if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
	scheduler) != pcmk_rc_ok) {
	return FALSE;
	}

	/* Currently, we always map the default authentication key location
	* into the same location inside the container.
	*
	* Ideally, we would respect the host's PCMK_authkey_location, but:
	* - it may be different on different nodes;
	* - the actual connection will do extra checking to make sure the key
	* file exists and is readable, that we can't do here on the DC
	* - tools such as crm_resource and crm_simulate may not have the same
	* environment variables as the cluster, causing operation digests to
	* differ
	*
	* Always using the default location inside the container is fine,
	* because we control the pacemaker_remote environment, and it avoids
	* having to pass another environment variable to the container.
	*
	* @TODO A better solution may be to have only pacemaker_remote use the
	* environment variable, and have the cluster nodes use a new
	* cluster option for key location. This would introduce the limitation
	* of the location being the same on all cluster nodes, but that's
	* reasonable.
	*/
	mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
	DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);

	if (need_log_mount) {
	mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
	pe__bundle_mount_subdir);
	}

	port = calloc(1, sizeof(pe__bundle_port_t));
	if(bundle_data->control_port) {
	port->source = strdup(bundle_data->control_port);
	} else {
	/* If we wanted to respect PCMK_remote_port, we could use
	* crm_default_remote_port() here and elsewhere in this file instead
	* of DEFAULT_REMOTE_PORT.
	*
	* However, it gains nothing, since we control both the container
	* environment and the connection resource parameters, and the user
	* can use a different port if desired by setting control-port.
	*/
	port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}
	port->target = strdup(port->source);
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	buffer = g_string_sized_new(1024);
	for (childIter = bundle_data->child->children; childIter != NULL;
	childIter = childIter->next) {

	pcmk__bundle_replica_t *replica = NULL;

	replica = calloc(1, sizeof(pcmk__bundle_replica_t));
	replica->child = childIter->data;
	replica->child->exclusive_discover = TRUE;
	replica->offset = lpc++;

	// Ensure the child's notify gets set based on the underlying primitive's value
	if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
	pcmk__set_rsc_flags(bundle_data->child, pcmk_rsc_notify);
	}

	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	bundle_data->attribute_target =
	g_hash_table_lookup(replica->child->meta,
	PCMK_META_CONTAINER_ATTR_TARGET);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);

	if (bundle_data->attribute_target) {
	g_hash_table_replace(rsc->meta,
	strdup(PCMK_META_CONTAINER_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	g_hash_table_replace(bundle_data->child->meta,
	strdup(PCMK_META_CONTAINER_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	}

	} else {
	// Just a naked container, no pacemaker-remote
	GString *buffer = g_string_sized_new(1024);

	for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
	pcmk__bundle_replica_t *replica = NULL;

	replica = calloc(1, sizeof(pcmk__bundle_replica_t));
	replica->offset = lpc;
	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
	pcmk__config_err("Failed unpacking resource %s", rsc->id);
	rsc->fns->free(rsc);
	return FALSE;
	}

	/* Utilization needs special handling for bundles. It makes no sense for
	* the inner primitive to have utilization, because it is tied
	* one-to-one to the guest node created by the container resource -- and
	* there's no way to set capacities for that guest node anyway.
	*
	* What the user really wants is to configure utilization for the
	* container. However, the schema only allows utilization for
	* primitives, and the container resource is implicit anyway, so the
	* user can only configure utilization for the inner primitive. If
	* they do, move the primitive's utilization values to the container.
	*
	* @TODO This means that bundles without an inner primitive can't have
	* utilization. An alternative might be to allow utilization values in
	* the top-level bundle XML in the schema, and copy those to each
	* container.
	*/
	if (replica->child != NULL) {
	GHashTable *empty = replica->container->utilization;

	replica->container->utilization = replica->child->utilization;
	replica->child->utilization = empty;
	}
	}

	if (bundle_data->child) {
	rsc->children = g_list_append(rsc->children, bundle_data->child);
	}
	return TRUE;
	}

	static int
	replica_resource_active(pcmk_resource_t *rsc, gboolean all)
	{
	if (rsc) {
	gboolean child_active = rsc->fns->active(rsc, all);

	if (child_active && !all) {
	return TRUE;
	} else if (!child_active && all) {
	return FALSE;
	}
	}
	return -1;
	}

	gboolean
	pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	GList *iter = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;
	int rsc_active;

	rsc_active = replica_resource_active(replica->ip, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->child, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->container, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->remote, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}
	}

	/* If "all" is TRUE, we've already checked that no resources were inactive,
	* so return TRUE; if "all" is FALSE, we didn't find any active resources,
	* so return FALSE.
	*/
	return all;
	}

	/*!
	* \internal
	* \brief Find the bundle replica corresponding to a given node
	*
	* \param[in] bundle Top-level bundle resource
	* \param[in] node Node to search for
	*
	* \return Bundle replica if found, NULL otherwise
	*/
	pcmk_resource_t *
	pe__find_bundle_replica(const pcmk_resource_t bundle, const pcmk_node_t node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_ASSERT(bundle && node);

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica && replica->node);
	if (replica->node->details == node->details) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_rsc_in_list(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	if (rsc != NULL) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	rsc->fns->print(rsc, pre_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	bundle_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (pre_text == NULL) {
	pre_text = "";
	}
	child_text = crm_strdup_printf("%s ", pre_text);

	get_bundle_variant_data(bundle_data, rsc);

	status_print("%s<bundle ", pre_text);
	status_print(PCMK_XA_ID "=\"%s\" ", rsc->id);
	status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
	status_print("image=\"%s\" ", bundle_data->image);
	status_print("unique=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_unique));
	status_print("managed=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_managed));
	status_print("failed=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_failed));
	status_print(">\n");

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	status_print("%s <replica " PCMK_XA_ID "=\"%d\">\n",
	pre_text, replica->offset);
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	status_print("%s </replica>\n", pre_text);
	}
	status_print("%s</bundle>\n", pre_text);
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	const char *desc = NULL;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	char *id = NULL;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
	continue;
	}

	if (!printed_header) {
	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	rc = pe__name_and_nvpairs_xml(out, true, "bundle", 8,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_TYPE, container_agent_str(bundle_data->agent_type),
	"image", bundle_data->image,
	"unique", pcmk__flag_text(rsc->flags, pcmk_rsc_unique),
	"maintenance",
	pcmk__flag_text(rsc->flags, pcmk_rsc_maintenance),
	"managed", pcmk__flag_text(rsc->flags, pcmk_rsc_managed),
	"failed", pcmk__flag_text(rsc->flags, pcmk_rsc_failed),
	PCMK_XA_DESCRIPTION, desc);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	id = pcmk__itoa(replica->offset);
	rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, PCMK_XA_ID, id);
	free(id);
	CRM_ASSERT(rc == pcmk_rc_ok);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
	replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml), show_opts,
	replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml), show_opts,
	replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
	replica->remote, only_node, only_rsc);
	}

	pcmk__output_xml_pop_parent(out); // replica
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out); // bundle
	}

	return rc;
	}

	static void
	pe__bundle_replica_output_html(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_html(out, rsc, buffer, node, show_opts);
	}

	/*!
	* \internal
	* \brief Get a string describing a resource's unmanaged state or lack thereof
	*
	* \param[in] rsc Resource to describe
	*
	* \return A string indicating that a resource is in maintenance mode or
	* otherwise unmanaged, or an empty string otherwise
	*/
	static const char *
	get_unmanaged_str(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	return " (maintenance)";
	}
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return " (unmanaged)";
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
	}

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->end_list(out);
	}
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_html(out, replica,
	pcmk__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	pe__bundle_replica_output_text(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	const pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_text(out, rsc, buffer, node, show_opts);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	get_bundle_variant_data(bundle_data, rsc);

	CRM_ASSERT(rsc != NULL);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->list_item(out, NULL, "Replica[%d]", replica->offset);
	}

	out->begin_list(out, NULL, NULL, NULL);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	out->end_list(out);
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_text(out, replica,
	pcmk__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_bundle_replica(pcmk__bundle_replica_t replica, const char pre_text,
	long options, void *print_data)
	{
	pcmk_node_t *node = NULL;
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	node = pcmk__current_node(replica->container);
	common_print(rsc, pre_text, buffer, node, options, print_data);
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	pe__print_bundle(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (options & pe_print_xml) {
	bundle_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_bundle_variant_data(bundle_data, rsc);

	if (pre_text == NULL) {
	pre_text = " ";
	}

	status_print("%sContainer bundle%s: %s [%s]%s%s\n",
	pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}


	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (options & pe_print_html) {
	status_print("<li>");
	}

	if (pcmk_is_set(options, pe_print_implicit)) {
	child_text = crm_strdup_printf(" %s", pre_text);
	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	status_print(" %sReplica[%d]\n", pre_text, replica->offset);
	}
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	} else {
	child_text = crm_strdup_printf("%s ", pre_text);
	print_bundle_replica(replica, child_text, options, print_data);
	}
	free(child_text);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	}

	static void
	free_bundle_replica(pcmk__bundle_replica_t *replica)
	{
	if (replica == NULL) {
	return;
	}

	if (replica->node) {
	free(replica->node);
	replica->node = NULL;
	}

	if (replica->ip) {
	free_xml(replica->ip->xml);
	replica->ip->xml = NULL;
	replica->ip->fns->free(replica->ip);
	replica->ip = NULL;
	}
	if (replica->container) {
	free_xml(replica->container->xml);
	replica->container->xml = NULL;
	replica->container->fns->free(replica->container);
	replica->container = NULL;
	}
	if (replica->remote) {
	free_xml(replica->remote->xml);
	replica->remote->xml = NULL;
	replica->remote->fns->free(replica->remote);
	replica->remote = NULL;
	}
	free(replica->ipaddr);
	free(replica);
	}

	void
	pe__free_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);
	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	free(bundle_data->prefix);
	free(bundle_data->image);
	free(bundle_data->control_port);
	free(bundle_data->host_network);
	free(bundle_data->host_netmask);
	free(bundle_data->ip_range_start);
	free(bundle_data->container_network);
	free(bundle_data->launcher_options);
	free(bundle_data->container_command);
	g_free(bundle_data->container_host_options);

	g_list_free_full(bundle_data->replicas,
	(GDestroyNotify) free_bundle_replica);
	g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
	g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
	g_list_free(rsc->children);

	if(bundle_data->child) {
	free_xml(bundle_data->child->xml);
	bundle_data->child->xml = NULL;
	bundle_data->child->fns->free(bundle_data->child);
	}
	common_free(rsc);
	}

	enum rsc_role_e
	pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
	{
	enum rsc_role_e container_role = pcmk_role_unknown;
	return container_role;
	}

	/*!
	* \brief Get the number of configured replicas in a bundle
	*
	* \param[in] rsc Bundle resource
	*
	* \return Number of configured replicas, or 0 on error
	*/
	int
	pe_bundle_replicas(const pcmk_resource_t *rsc)
	{
	if ((rsc == NULL) \|\| (rsc->variant != pcmk_rsc_variant_bundle)) {
	return 0;
	} else {
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	return bundle_data->nreplicas;
	}
	}

	void
	pe__count_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
	pcmk__bundle_replica_t *replica = item->data;

	if (replica->ip) {
	replica->ip->fns->count(replica->ip);
	}
	if (replica->child) {
	replica->child->fns->count(replica->child);
	}
	if (replica->container) {
	replica->container->fns->count(replica->container);
	}
	if (replica->remote) {
	replica->remote->fns->count(replica->remote);
	}
	}
	}

	gboolean
	pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_bundle_variant_data(bundle_data, rsc);

	for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get a list of a bundle's containers
	*
	* \param[in] bundle Bundle resource
	*
	* \return Newly created list of \p bundle's containers
	* \note It is the caller's responsibility to free the result with
	* g_list_free().
	*/
	GList *
	pe__bundle_containers(const pcmk_resource_t *bundle)
	{
	GList *containers = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, bundle);
	for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	containers = g_list_append(containers, replica->container);
	}
	return containers;
	}

	// Bundle implementation of pcmk_rsc_methods_t:active_node()
	pcmk_node_t *
	pe__bundle_active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;
	pcmk_node_t *node = NULL;
	pcmk_resource_t *container = NULL;
	GList *containers = NULL;
	GList *iter = NULL;
	GHashTable *nodes = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}

	/* For the purposes of this method, we only care about where the bundle's
	* containers are active, so build a list of active containers.
	*/
	get_bundle_variant_data(data, rsc);
	for (iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (replica->container->running_on != NULL) {
	containers = g_list_append(containers, replica->container);
	}
	}
	if (containers == NULL) {
	return NULL;
	}

	/* If the bundle has only a single active container, just use that
	* container's method. If live migration is ever supported for bundle
	* containers, this will allow us to prefer the migration source when there
	* is only one container and it is migrating. For now, this just lets us
	* avoid creating the nodes table.
	*/
	if (pcmk__list_of_1(containers)) {
	container = containers->data;
	node = container->fns->active_node(container, count_all, count_clean);
	g_list_free(containers);
	return node;
	}

	// Add all containers' active nodes to a hash table (for uniqueness)
	nodes = g_hash_table_new(NULL, NULL);
	for (iter = containers; iter != NULL; iter = iter->next) {
	container = iter->data;

	for (GList *node_iter = container->running_on; node_iter != NULL;
	node_iter = node_iter->next) {
	node = node_iter->data;

	// If insert returns true, we haven't counted this node yet
	if (g_hash_table_insert(nodes, (gpointer) node->details,
	(gpointer) node)
	&& !pe__count_active_node(rsc, node, &active, count_all,
	count_clean)) {
	goto done;
	}
	}
	}

	done:
	g_list_free(containers);
	g_hash_table_destroy(nodes);
	return active;
	}

	/*!
	* \internal
	* \brief Get maximum bundle resource instances per node
	*
	* \param[in] rsc Bundle resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__bundle_max_per_node(const pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
	return (unsigned int) bundle_data->nreplicas_per_host;
	}
	diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
	index 8369601be9..e6890df821 100644
	--- a/lib/pengine/utils.c
	+++ b/lib/pengine/utils.c
	@@ -1,913 +1,914 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>

	#include "pe_status_private.h"

	extern bool pcmk__is_daemon;

	gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);

	/*!
	* \internal
	* \brief Check whether we can fence a particular node
	*
	* \param[in] scheduler Scheduler data
	* \param[in] node Name of node to check
	*
	* \return true if node can be fenced, false otherwise
	*/
	bool
	pe_can_fence(const pcmk_scheduler_t scheduler, const pcmk_node_t node)
	{
	if (pe__is_guest_node(node)) {
	/* Guest nodes are fenced by stopping their container resource. We can
	* do that if the container's host is either online or fenceable.
	*/
	pcmk_resource_t *rsc = node->details->remote_rsc->container;

	for (GList *n = rsc->running_on; n != NULL; n = n->next) {
	pcmk_node_t *container_node = n->data;

	if (!container_node->details->online
	&& !pe_can_fence(scheduler, container_node)) {
	return false;
	}
	}
	return true;

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	return false; /* Turned off */

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {
	return false; /* No devices */

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	return true;

	} else if (scheduler->no_quorum_policy == pcmk_no_quorum_ignore) {
	return true;

	} else if(node == NULL) {
	return false;

	} else if(node->details->online) {
	crm_notice("We can fence %s without quorum because they're in our membership",
	pcmk__node_name(node));
	return true;
	}

	crm_trace("Cannot fence %s", pcmk__node_name(node));
	return false;
	}

	/*!
	* \internal
	* \brief Copy a node object
	*
	* \param[in] this_node Node object to copy
	*
	* \return Newly allocated shallow copy of this_node
	* \note This function asserts on errors and is guaranteed to return non-NULL.
	*/
	pcmk_node_t *
	pe__copy_node(const pcmk_node_t *this_node)
	{
	pcmk_node_t *new_node = NULL;

	CRM_ASSERT(this_node != NULL);

	new_node = calloc(1, sizeof(pcmk_node_t));
	CRM_ASSERT(new_node != NULL);

	new_node->rsc_discover_mode = this_node->rsc_discover_mode;
	new_node->weight = this_node->weight;
	new_node->fixed = this_node->fixed; // @COMPAT deprecated and unused
	new_node->count = this_node->count;
	new_node->details = this_node->details;

	return new_node;
	}

	/*!
	* \internal
	* \brief Create a node hash table from a node list
	*
	* \param[in] list Node list
	*
	* \return Hash table equivalent of node list
	*/
	GHashTable *
	pe__node_list2table(const GList *list)
	{
	GHashTable *result = NULL;

	result = pcmk__strkey_table(NULL, free);
	for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *new_node = NULL;

	new_node = pe__copy_node((const pcmk_node_t *) gIter->data);
	g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Compare two nodes by name, with numeric portions sorted numerically
	*
	* Sort two node names case-insensitively like strcasecmp(), but with any
	* numeric portions of the name sorted numerically. For example, "node10" will
	* sort higher than "node9" but lower than "remotenode9".
	*
	* \param[in] a First node to compare (can be \c NULL)
	* \param[in] b Second node to compare (can be \c NULL)
	*
	* \retval -1 \c a comes before \c b (or \c a is \c NULL and \c b is not)
	* \retval 0 \c a and \c b are equal (or both are \c NULL)
	* \retval 1 \c a comes after \c b (or \c b is \c NULL and \c a is not)
	*/
	gint
	pe__cmp_node_name(gconstpointer a, gconstpointer b)
	{
	const pcmk_node_t node1 = (const pcmk_node_t ) a;
	const pcmk_node_t node2 = (const pcmk_node_t ) b;

	if ((node1 == NULL) && (node2 == NULL)) {
	return 0;
	}

	if (node1 == NULL) {
	return -1;
	}

	if (node2 == NULL) {
	return 1;
	}

	return pcmk__numeric_strcasecmp(node1->details->uname,
	node2->details->uname);
	}

	/*!
	* \internal
	* \brief Output node weights to stdout
	*
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	pe__output_node_weights(const pcmk_resource_t rsc, const char comment,
	GHashTable nodes, pcmk_scheduler_t scheduler)
	{
	pcmk__output_t *out = scheduler->priv;

	// Sort the nodes so the output is consistent for regression tests
	GList *list = g_list_sort(g_hash_table_get_values(nodes),
	pe__cmp_node_name);

	for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	const pcmk_node_t node = (const pcmk_node_t ) gIter->data;

	out->message(out, "node-weight", rsc, comment, node->details->uname,
	pcmk_readable_score(node->weight));
	}
	g_list_free(list);
	}

	/*!
	* \internal
	* \brief Log node weights at trace level
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] rsc If not NULL, include this resource's ID in logs
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes Nodes whose scores should be logged
	*/
	static void
	pe__log_node_weights(const char file, const char function, int line,
	const pcmk_resource_t rsc, const char comment,
	GHashTable *nodes)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	// Don't waste time if we're not tracing at this point
	pcmk__if_tracing({}, return);

	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (rsc) {
	qb_log_from_external_source(function, file,
	"%s: %s allocation score on %s: %s",
	LOG_TRACE, line, 0,
	comment, rsc->id,
	pcmk__node_name(node),
	pcmk_readable_score(node->weight));
	} else {
	qb_log_from_external_source(function, file, "%s: %s = %s",
	LOG_TRACE, line, 0,
	comment, pcmk__node_name(node),
	pcmk_readable_score(node->weight));
	}
	}
	}

	/*!
	* \internal
	* \brief Log or output node weights
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] to_log Log if true, otherwise output
	* \param[in] rsc If not NULL, use this resource's ID in logs,
	* and show scores recursively for any children
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes Nodes whose scores should be shown
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pe__show_node_scores_as(const char file, const char function, int line,
	bool to_log, const pcmk_resource_t *rsc,
	const char comment, GHashTable nodes,
	pcmk_scheduler_t *scheduler)
	{
	if ((rsc != NULL) && pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	// Don't show allocation scores for orphans
	return;
	}
	if (nodes == NULL) {
	// Nothing to show
	return;
	}

	if (to_log) {
	pe__log_node_weights(file, function, line, rsc, comment, nodes);
	} else {
	pe__output_node_weights(rsc, comment, nodes, scheduler);
	}

	// If this resource has children, repeat recursively for each
	if (rsc && rsc->children) {
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	pe__show_node_scores_as(file, function, line, to_log, child,
	comment, child->allowed_nodes, scheduler);
	}
	}
	}

	/*!
	* \internal
	* \brief Compare two resources by priority
	*
	* \param[in] a First resource to compare (can be \c NULL)
	* \param[in] b Second resource to compare (can be \c NULL)
	*
	* \retval -1 \c a->priority > \c b->priority (or \c b is \c NULL and \c a is
	* not)
	* \retval 0 \c a->priority == \c b->priority (or both \c a and \c b are
	* \c NULL)
	* \retval 1 \c a->priority < \c b->priority (or \c a is \c NULL and \c b is
	* not)
	*/
	gint
	pe__cmp_rsc_priority(gconstpointer a, gconstpointer b)
	{
	const pcmk_resource_t resource1 = (const pcmk_resource_t )a;
	const pcmk_resource_t resource2 = (const pcmk_resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->priority > resource2->priority) {
	return -1;
	}

	if (resource1->priority < resource2->priority) {
	return 1;
	}

	return 0;
	}

	static void
	resource_node_score(pcmk_resource_t rsc, const pcmk_node_t node, int score,
	const char *tag)
	{
	pcmk_node_t *match = NULL;

	if ((rsc->exclusive_discover
	\|\| (node->rsc_discover_mode == pcmk_probe_never))
	&& pcmk__str_eq(tag, "symmetric_default", pcmk__str_casei)) {
	/* This string comparision may be fragile, but exclusive resources and
	* exclusive nodes should not have the symmetric_default constraint
	* applied to them.
	*/
	return;

	} else if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	resource_node_score(child_rsc, node, score, tag);
	}
	}

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match == NULL) {
	match = pe__copy_node(node);
	g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
	}
	match->weight = pcmk__add_scores(match->weight, score);
	pcmk__rsc_trace(rsc,
	"Enabling %s preference (%s) for %s on %s (now %s)",
	tag, pcmk_readable_score(score), rsc->id,
	pcmk__node_name(node), pcmk_readable_score(match->weight));
	}

	void
	resource_location(pcmk_resource_t rsc, const pcmk_node_t node, int score,
	const char tag, pcmk_scheduler_t scheduler)
	{
	if (node != NULL) {
	resource_node_score(rsc, node, score, tag);

	} else if (scheduler != NULL) {
	GList *gIter = scheduler->nodes;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node_iter = (pcmk_node_t ) gIter->data;

	resource_node_score(rsc, node_iter, score, tag);
	}

	} else {
	GHashTableIter iter;
	pcmk_node_t *node_iter = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
	resource_node_score(rsc, node_iter, score, tag);
	}
	}

	if (node == NULL && score == -INFINITY) {
	if (rsc->allocated_to) {
	crm_info("Deallocating %s from %s",
	rsc->id, pcmk__node_name(rsc->allocated_to));
	free(rsc->allocated_to);
	rsc->allocated_to = NULL;
	}
	}
	}

	time_t
	get_effective_time(pcmk_scheduler_t *scheduler)
	{
	if(scheduler) {
	if (scheduler->now == NULL) {
	crm_trace("Recording a new 'now'");
	scheduler->now = crm_time_new(NULL);
	}
	return crm_time_get_seconds_since_epoch(scheduler->now);
	}

	crm_trace("Defaulting to 'now'");
	return time(NULL);
	}

	gboolean
	get_target_role(const pcmk_resource_t rsc, enum rsc_role_e role)
	{
	enum rsc_role_e local_role = pcmk_role_unknown;
	const char *value = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);

	CRM_CHECK(role != NULL, return FALSE);

	if (pcmk__str_eq(value, "started", pcmk__str_null_matches \| pcmk__str_casei)
	\|\| pcmk__str_eq("default", value, pcmk__str_casei)) {
	return FALSE;
	}

	local_role = text2role(value);
	if (local_role == pcmk_role_unknown) {
	pcmk__config_err("Ignoring '" PCMK_META_TARGET_ROLE "' for %s "
	"because '%s' is not valid", rsc->id, value);
	return FALSE;

	} else if (local_role > pcmk_role_started) {
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)) {
	if (local_role > pcmk_role_unpromoted) {
	/* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
	return FALSE;
	}

	} else {
	pcmk__config_err("Ignoring '" PCMK_META_TARGET_ROLE "' for %s "
	"because '%s' only makes sense for promotable "
	"clones", rsc->id, value);
	return FALSE;
	}
	}

	*role = local_role;
	return TRUE;
	}

	gboolean
	order_actions(pcmk_action_t lh_action, pcmk_action_t rh_action,
	uint32_t flags)
	{
	GList *gIter = NULL;
	pcmk__related_action_t *wrapper = NULL;
	GList *list = NULL;

	if (flags == pcmk__ar_none) {
	return FALSE;
	}

	if (lh_action == NULL \|\| rh_action == NULL) {
	return FALSE;
	}

	crm_trace("Creating action wrappers for ordering: %s then %s",
	lh_action->uuid, rh_action->uuid);

	/* Ensure we never create a dependency on ourselves... it's happened */
	CRM_ASSERT(lh_action != rh_action);

	/* Filter dups, otherwise update_action_states() has too much work to do */
	gIter = lh_action->actions_after;
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk__related_action_t *after = gIter->data;

	if (after->action == rh_action && (after->type & flags)) {
	return FALSE;
	}
	}

	wrapper = calloc(1, sizeof(pcmk__related_action_t));
	wrapper->action = rh_action;
	wrapper->type = flags;
	list = lh_action->actions_after;
	list = g_list_prepend(list, wrapper);
	lh_action->actions_after = list;

	wrapper = calloc(1, sizeof(pcmk__related_action_t));
	wrapper->action = lh_action;
	wrapper->type = flags;
	list = rh_action->actions_before;
	list = g_list_prepend(list, wrapper);
	rh_action->actions_before = list;
	return TRUE;
	}

	void
	destroy_ticket(gpointer data)
	{
	pcmk_ticket_t *ticket = data;

	if (ticket->state) {
	g_hash_table_destroy(ticket->state);
	}
	free(ticket->id);
	free(ticket);
	}

	pcmk_ticket_t *
	ticket_new(const char ticket_id, pcmk_scheduler_t scheduler)
	{
	pcmk_ticket_t *ticket = NULL;

	if (pcmk__str_empty(ticket_id)) {
	return NULL;
	}

	if (scheduler->tickets == NULL) {
	scheduler->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {

	ticket = calloc(1, sizeof(pcmk_ticket_t));
	if (ticket == NULL) {
	pcmk__sched_err("Cannot allocate ticket '%s'", ticket_id);
	return NULL;
	}

	crm_trace("Creaing ticket entry for %s", ticket_id);

	ticket->id = strdup(ticket_id);
	ticket->granted = FALSE;
	ticket->last_granted = -1;
	ticket->standby = FALSE;
	ticket->state = pcmk__strkey_table(free, free);

	g_hash_table_insert(scheduler->tickets, strdup(ticket->id), ticket);
	}

	return ticket;
	}

	const char *
	rsc_printable_id(const pcmk_resource_t *rsc)
	{
	return pcmk_is_set(rsc->flags, pcmk_rsc_unique)? rsc->id : ID(rsc->xml);
	}

	void
	pe__clear_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags)
	{
	pcmk__clear_rsc_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__clear_resource_flags_recursive((pcmk_resource_t *) gIter->data,
	flags);
	}
	}

	void
	pe__clear_resource_flags_on_all(pcmk_scheduler_t *scheduler, uint64_t flag)
	{
	for (GList *lpc = scheduler->resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t r = (pcmk_resource_t ) lpc->data;
	pe__clear_resource_flags_recursive(r, flag);
	}
	}

	void
	pe__set_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags)
	{
	pcmk__set_rsc_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__set_resource_flags_recursive((pcmk_resource_t *) gIter->data,
	flags);
	}
	}

	void
	trigger_unfencing(pcmk_resource_t rsc, pcmk_node_t node, const char *reason,
	pcmk_action_t dependency, pcmk_scheduler_t scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_enable_unfencing)) {
	/* No resources require it */
	return;

	} else if ((rsc != NULL)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	/* Wasn't a stonith device */
	return;

	} else if(node
	&& node->details->online
	&& node->details->unclean == FALSE
	&& node->details->shutdown == FALSE) {
	pcmk_action_t *unfence = pe_fence_op(node, PCMK_ACTION_ON, FALSE,
	reason, FALSE, scheduler);

	if(dependency) {
	order_actions(unfence, dependency, pcmk__ar_ordered);
	}

	} else if(rsc) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
	trigger_unfencing(rsc, node, reason, dependency, scheduler);
	}
	}
	}
	}

	gboolean
	add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref)
	{
	pcmk_tag_t *tag = NULL;
	GList *gIter = NULL;
	gboolean is_existing = FALSE;

	CRM_CHECK(tags && tag_name && obj_ref, return FALSE);

	tag = g_hash_table_lookup(tags, tag_name);
	if (tag == NULL) {
	tag = calloc(1, sizeof(pcmk_tag_t));
	if (tag == NULL) {
	pcmk__sched_err("Could not allocate memory for tag %s", tag_name);
	return FALSE;
	}
	tag->id = strdup(tag_name);
	tag->refs = NULL;
	g_hash_table_insert(tags, strdup(tag_name), tag);
	}

	for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
	const char existing_ref = (const char ) gIter->data;

	if (pcmk__str_eq(existing_ref, obj_ref, pcmk__str_none)){
	is_existing = TRUE;
	break;
	}
	}

	if (is_existing == FALSE) {
	tag->refs = g_list_append(tag->refs, strdup(obj_ref));
	crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Check whether shutdown has been requested for a node
	*
	* \param[in] node Node to check
	*
	* \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
	* \note This differs from simply using node->details->shutdown in that it can
	* be used before that has been determined (and in fact to determine it),
	* and it can also be used to distinguish requested shutdown from implicit
	* shutdown of remote nodes by virtue of their connection stopping.
	*/
	bool
	pe__shutdown_requested(const pcmk_node_t *node)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);

	return !pcmk__str_eq(shutdown, "0", pcmk__str_null_matches);
	}

	/*!
	* \internal
	* \brief Update "recheck by" time in scheduler data
	*
	* \param[in] recheck Epoch time when recheck should happen
	* \param[in,out] scheduler Scheduler data
	* \param[in] reason What time is being updated for (for logs)
	*/
	void
	pe__update_recheck_time(time_t recheck, pcmk_scheduler_t *scheduler,
	const char *reason)
	{
	if ((recheck > get_effective_time(scheduler))
	&& ((scheduler->recheck_by == 0)
	\|\| (scheduler->recheck_by > recheck))) {
	scheduler->recheck_by = recheck;
	crm_debug("Updated next scheduler recheck to %s for %s",
	pcmk__trim(ctime(&recheck)), reason);
	}
	}

	/*!
	* \internal
	* \brief Extract nvpair blocks contained by a CIB XML element into a hash table
	*
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name If not NULL, only use blocks of this element
	* \param[in] rule_data Matching parameters to use when unpacking
	* \param[out] hash Where to store extracted name/value pairs
	* \param[in] always_first If not NULL, process block with this ID first
	* \param[in] overwrite Whether to replace existing values with same name
	* \param[in,out] scheduler Scheduler data containing \p xml_obj
	*/
	void
	pe__unpack_dataset_nvpairs(const xmlNode xml_obj, const char set_name,
	const pe_rule_eval_data_t *rule_data,
	GHashTable hash, const char always_first,
	gboolean overwrite, pcmk_scheduler_t *scheduler)
	{
	crm_time_t *next_change = crm_time_new_undefined();

	pe_eval_nvpairs(scheduler->input, xml_obj, set_name, rule_data, hash,
	always_first, overwrite, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, scheduler, "rule evaluation");
	}
	crm_time_free(next_change);
	}

	bool
	pe__resource_is_disabled(const pcmk_resource_t *rsc)
	{
	const char *target_role = NULL;

	CRM_CHECK(rsc != NULL, return false);
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	if (target_role) {
	// If invalid, we've already logged an error when unpacking
	enum rsc_role_e target_role_e = text2role(target_role);

	if ((target_role_e == pcmk_role_stopped)
	\|\| ((target_role_e == pcmk_role_unpromoted)
	&& pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable))) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a resource is running only on given node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p rsc is running only on \p node, otherwise false
	*/
	bool
	pe__rsc_running_on_only(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	return (rsc != NULL) && pcmk__list_of_1(rsc->running_on)
	- && pe__same_node((const pcmk_node_t *) rsc->running_on->data, node);
	+ && pcmk__same_node((const pcmk_node_t *) rsc->running_on->data,
	+ node);
	}

	bool
	pe__rsc_running_on_any(pcmk_resource_t rsc, GList node_list)
	{
	for (GList *ele = rsc->running_on; ele; ele = ele->next) {
	pcmk_node_t node = (pcmk_node_t ) ele->data;
	if (pcmk__str_in_list(node->details->uname, node_list,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	return true;
	}
	}

	return false;
	}

	bool
	pcmk__rsc_filtered_by_node(pcmk_resource_t rsc, GList only_node)
	{
	return (rsc->fns->active(rsc, FALSE) && !pe__rsc_running_on_any(rsc, only_node));
	}

	GList *
	pe__filter_rsc_list(GList rscs, GList filter)
	{
	GList *retval = NULL;

	for (GList *gIter = rscs; gIter; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	/* I think the second condition is safe here for all callers of this
	* function. If not, it needs to move into pe__node_text.
	*/
	if (pcmk__str_in_list(rsc_printable_id(rsc), filter, pcmk__str_star_matches) \|\|
	(rsc->parent && pcmk__str_in_list(rsc_printable_id(rsc->parent), filter, pcmk__str_star_matches))) {
	retval = g_list_prepend(retval, rsc);
	}
	}

	return retval;
	}

	GList *
	pe__build_node_name_list(pcmk_scheduler_t scheduler, const char s)
	{
	GList *nodes = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	/* Nothing was given so return a list of all node names. Or, '*' was
	* given. This would normally fall into the pe__unames_with_tag branch
	* where it will return an empty list. Catch it here instead.
	*/
	nodes = g_list_prepend(nodes, strdup("*"));
	} else {
	pcmk_node_t *node = pe_find_node(scheduler->nodes, s);

	if (node) {
	/* The given string was a valid uname for a node. Return a
	* singleton list containing just that uname.
	*/
	nodes = g_list_prepend(nodes, strdup(s));
	} else {
	/* The given string was not a valid uname. It's either a tag or
	* it's a typo or something. In the first case, we'll return a
	* list of all the unames of the nodes with the given tag. In the
	* second case, we'll return a NULL pointer and nothing will
	* get displayed.
	*/
	nodes = pe__unames_with_tag(scheduler, s);
	}
	}

	return nodes;
	}

	GList *
	pe__build_rsc_list(pcmk_scheduler_t scheduler, const char s)
	{
	GList *resources = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	resources = g_list_prepend(resources, strdup("*"));
	} else {
	const uint32_t flags = pcmk_rsc_match_history\|pcmk_rsc_match_basename;
	pcmk_resource_t *rsc = pe_find_resource_with_flags(scheduler->resources,
	s, flags);

	if (rsc) {
	/* A colon in the name we were given means we're being asked to filter
	* on a specific instance of a cloned resource. Put that exact string
	* into the filter list. Otherwise, use the printable ID of whatever
	* resource was found that matches what was asked for.
	*/
	if (strstr(s, ":") != NULL) {
	resources = g_list_prepend(resources, strdup(rsc->id));
	} else {
	resources = g_list_prepend(resources, strdup(rsc_printable_id(rsc)));
	}
	} else {
	/* The given string was not a valid resource name. It's a tag or a
	* typo or something. See pe__build_node_name_list() for more
	* detail.
	*/
	resources = pe__rscs_with_tag(scheduler, s);
	}
	}

	return resources;
	}

	xmlNode *
	pe__failed_probe_for_rsc(const pcmk_resource_t rsc, const char name)
	{
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
	const char *rsc_id = rsc->id;

	if (parent->variant == pcmk_rsc_variant_clone) {
	rsc_id = pe__clone_child_id(parent);
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(rsc->cluster->failed); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	const char *value = NULL;
	char *op_id = NULL;

	/* This resource operation is not a failed probe. */
	if (!pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	/* This resource operation was not run on the given node. Note that if name is
	* NULL, this will always succeed.
	*/
	value = crm_element_value(xml_op, PCMK__META_ON_NODE);
	if (value == NULL \|\| !pcmk__str_eq(value, name, pcmk__str_casei\|pcmk__str_null_matches)) {
	continue;
	}

	if (!parse_op_key(pe__xe_history_key(xml_op), &op_id, NULL, NULL)) {
	continue; // This history entry is missing an operation key
	}

	/* This resource operation's ID does not match the rsc_id we are looking for. */
	if (!pcmk__str_eq(op_id, rsc_id, pcmk__str_none)) {
	free(op_id);
	continue;
	}

	free(op_id);
	return xml_op;
	}

	return NULL;
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 4:31 PM (1 d, 20 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1652363
Default Alt Text: (764 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions