No OneTemporary
Actions

Size

328 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
	index 01b54ddb84..c25ad299c0 100644
	--- a/include/crm/pengine/internal.h
	+++ b/include/crm/pengine/internal.h
	@@ -1,684 +1,692 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PE_INTERNAL__H
	# define PE_INTERNAL__H

	# include <stdint.h>
	# include <string.h>
	# include <crm/pengine/status.h>
	# include <crm/pengine/remote_internal.h>
	# include <crm/common/internal.h>
	# include <crm/common/options_internal.h>
	# include <crm/common/output_internal.h>

	enum pe__clone_flags {
	// Whether instances should be started sequentially
	pe__clone_ordered = (1 << 0),

	// Whether promotion scores have been added
	pe__clone_promotion_added = (1 << 1),

	// Whether promotion constraints have been added
	pe__clone_promotion_constrained = (1 << 2),
	};

	bool pe__clone_is_ordered(pe_resource_t *clone);
	int pe__set_clone_flag(pe_resource_t *clone, enum pe__clone_flags flag);


	enum pe__group_flags {
	pe__group_ordered = (1 << 0), // Members start sequentially
	pe__group_colocated = (1 << 1), // Members must be on same node
	};

	bool pe__group_flag_is_set(const pe_resource_t *group, uint32_t flags);
	pe_resource_t pe__last_group_member(const pe_resource_t group);


	# define pe_rsc_info(rsc, fmt, args...) crm_log_tag(LOG_INFO, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_debug(rsc, fmt, args...) crm_log_tag(LOG_DEBUG, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_trace(rsc, fmt, args...) crm_log_tag(LOG_TRACE, rsc ? rsc->id : "<NULL>", fmt, ##args)

	# define pe_err(fmt...) do { \
	was_processing_error = TRUE; \
	pcmk__config_err(fmt); \
	} while (0)

	# define pe_warn(fmt...) do { \
	was_processing_warning = TRUE; \
	pcmk__config_warn(fmt); \
	} while (0)

	# define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
	# define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }

	#define pe__set_working_set_flags(working_set, flags_to_set) do { \
	(working_set)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_working_set_flags(working_set, flags_to_clear) do { \
	(working_set)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_resource_flags(resource, flags_to_set) do { \
	(resource)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_resource_flags(resource, flags_to_clear) do { \
	(resource)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_action_flags(action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags(action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_raw_action_flags(action_flags, action_name, flags_to_set) do { \
	action_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action", action_name, \
	(action_flags), \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_raw_action_flags(action_flags, action_name, flags_to_clear) do { \
	action_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", action_name, \
	(action_flags), \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_action_flags_as(function, line, action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags_as(function, line, action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_order_flags(order_flags, flags_to_set) do { \
	order_flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_order_flags(order_flags, flags_to_clear) do { \
	order_flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	// Some warnings we don't want to print every transition

	enum pe_warn_once_e {
	pe_wo_blind = (1 << 0),
	pe_wo_restart_type = (1 << 1),
	pe_wo_role_after = (1 << 2),
	pe_wo_poweroff = (1 << 3),
	pe_wo_require_all = (1 << 4),
	pe_wo_order_score = (1 << 5),
	pe_wo_neg_threshold = (1 << 6),
	pe_wo_remove_after = (1 << 7),
	pe_wo_ping_node = (1 << 8),
	pe_wo_order_inst = (1 << 9),
	pe_wo_coloc_inst = (1 << 10),
	pe_wo_group_order = (1 << 11),
	pe_wo_group_coloc = (1 << 12),
	};

	extern uint32_t pe_wo;

	#define pe_warn_once(pe_wo_bit, fmt...) do { \
	if (!pcmk_is_set(pe_wo, pe_wo_bit)) { \
	if (pe_wo_bit == pe_wo_blind) { \
	crm_warn(fmt); \
	} else { \
	pe_warn(fmt); \
	} \
	pe_wo = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Warn-once", "logging", pe_wo, \
	(pe_wo_bit), #pe_wo_bit); \
	} \
	} while (0);


	typedef struct pe__location_constraint_s {
	char *id; // Constraint XML ID
	pe_resource_t *rsc_lh; // Resource being located
	enum rsc_role_e role_filter; // Role to locate
	enum pe_discover_e discover_mode; // Resource discovery
	GList node_list_rh; // List of pe_node_t
	} pe__location_t;

	typedef struct pe__order_constraint_s {
	int id;
	uint32_t flags; // Group of enum pe_ordering flags

	void *lh_opaque;
	pe_resource_t *lh_rsc;
	pe_action_t *lh_action;
	char *lh_action_task;

	void *rh_opaque;
	pe_resource_t *rh_rsc;
	pe_action_t *rh_action;
	char *rh_action_task;
	} pe__ordering_t;

	typedef struct notify_data_s {
	GSList *keys; // Environment variable name/value pairs

	const char *action;

	pe_action_t *pre;
	pe_action_t *post;
	pe_action_t *pre_done;
	pe_action_t *post_done;

	GList active; / notify_entry_t* */
	GList inactive; / notify_entry_t* */
	GList start; / notify_entry_t* */
	GList stop; / notify_entry_t* */
	GList demote; / notify_entry_t* */
	GList promote; / notify_entry_t* */
	GList promoted; / notify_entry_t* */
	GList unpromoted; / notify_entry_t* */
	GHashTable *allowed_nodes;

	} notify_data_t;

	int pe__clone_promoted_max(pe_resource_t *clone);
	int pe__clone_promoted_node_max(pe_resource_t *clone);

	pe_action_t pe__new_rsc_pseudo_action(pe_resource_t rsc, const char *task,
	bool optional, bool runnable);

	void pe__create_promotable_pseudo_ops(pe_resource_t *clone, bool any_promoting,
	bool any_demoting);

	-bool pe_can_fence(pe_working_set_t data_set, pe_node_t node);
	+bool pe_can_fence(const pe_working_set_t data_set, const pe_node_t node);

	void add_hash_param(GHashTable * hash, const char name, const char value);

	char native_parameter(pe_resource_t rsc, pe_node_t * node, gboolean create, const char *name,
	pe_working_set_t * data_set);
	pe_node_t native_location(const pe_resource_t rsc, GList **list, int current);

	void pe_metadata(pcmk__output_t *out);
	void verify_pe_options(GHashTable * options);

	void common_update_score(pe_resource_t * rsc, const char *id, int score);
	void native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set, gboolean failed);

	gboolean native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean group_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean pe__unpack_bundle(pe_resource_t rsc, pe_working_set_t data_set);

	pe_resource_t native_find_rsc(pe_resource_t rsc, const char id, const pe_node_t node,
	int flags);

	gboolean native_active(pe_resource_t * rsc, gboolean all);
	gboolean group_active(pe_resource_t * rsc, gboolean all);
	gboolean clone_active(pe_resource_t * rsc, gboolean all);
	gboolean pe__bundle_active(pe_resource_t *rsc, gboolean all);

	//! \deprecated This function will be removed in a future release
	void native_print(pe_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void group_print(pe_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void clone_print(pe_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void pe__print_bundle(pe_resource_t rsc, const char pre_text, long options,
	void *print_data);

	gchar * pcmk__native_output_string(pe_resource_t rsc, const char name, pe_node_t *node,
	uint32_t show_opts, const char *target_role, bool show_nodes);

	int pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...);
	char pe__node_display_name(pe_node_t node, bool print_detail);


	// Clone notifications (pe_notif.c)
	void pe__create_notifications(pe_resource_t rsc, notify_data_t n_data);
	notify_data_t pe__clone_notif_pseudo_ops(pe_resource_t rsc, const char *task,
	pe_action_t *action,
	pe_action_t *complete);
	void pe__free_notification_data(notify_data_t *n_data);
	void pe__order_notifs_after_fencing(pe_action_t action, pe_resource_t rsc,
	pe_action_t *stonith_op);


	static inline const char *
	pe__rsc_bool_str(const pe_resource_t *rsc, uint64_t rsc_flag)
	{
	return pcmk__btoa(pcmk_is_set(rsc->flags, rsc_flag));
	}

	int pe__clone_xml(pcmk__output_t *out, va_list args);
	int pe__clone_default(pcmk__output_t *out, va_list args);
	int pe__group_xml(pcmk__output_t *out, va_list args);
	int pe__group_default(pcmk__output_t *out, va_list args);
	int pe__bundle_xml(pcmk__output_t *out, va_list args);
	int pe__bundle_html(pcmk__output_t *out, va_list args);
	int pe__bundle_text(pcmk__output_t *out, va_list args);
	int pe__node_html(pcmk__output_t *out, va_list args);
	int pe__node_text(pcmk__output_t *out, va_list args);
	int pe__node_xml(pcmk__output_t *out, va_list args);
	int pe__resource_xml(pcmk__output_t *out, va_list args);
	int pe__resource_html(pcmk__output_t *out, va_list args);
	int pe__resource_text(pcmk__output_t *out, va_list args);

	void native_free(pe_resource_t * rsc);
	void group_free(pe_resource_t * rsc);
	void clone_free(pe_resource_t * rsc);
	void pe__free_bundle(pe_resource_t *rsc);

	enum rsc_role_e native_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e group_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e clone_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e pe__bundle_resource_state(const pe_resource_t *rsc,
	gboolean current);

	void pe__count_common(pe_resource_t *rsc);
	void pe__count_bundle(pe_resource_t *rsc);

	void common_free(pe_resource_t * rsc);

	pe_node_t pe__copy_node(const pe_node_t this_node);
	extern time_t get_effective_time(pe_working_set_t * data_set);

	/* Failure handling utilities (from failcounts.c) */

	// bit flags for fail count handling options
	enum pe_fc_flags_e {
	pe_fc_default = (1 << 0),
	pe_fc_effective = (1 << 1), // don't count expired failures
	pe_fc_fillers = (1 << 2), // if container, include filler failures in count
	};

	int pe_get_failcount(const pe_node_t node, pe_resource_t rsc,
	time_t *last_failure, uint32_t flags,
	const xmlNode *xml_op);

	-pe_action_t pe__clear_failcount(pe_resource_t rsc, pe_node_t *node,
	+pe_action_t pe__clear_failcount(pe_resource_t rsc, const pe_node_t *node,
	const char *reason,
	pe_working_set_t *data_set);

	/* Functions for finding/counting a resource's active nodes */

	pe_node_t pe__find_active_on(const pe_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);
	pe_node_t pe__find_active_requires(const pe_resource_t rsc,
	unsigned int *count);

	static inline pe_node_t *
	pe__current_node(const pe_resource_t *rsc)
	{
	return pe__find_active_on(rsc, NULL, NULL);
	}


	/* Binary like operators for lists of nodes */
	extern void node_list_exclude(GHashTable * list, GList *list2, gboolean merge_scores);

	GHashTable pe__node_list2table(GList list);

	static inline gpointer
	pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
	{
	if (hash) {
	return g_hash_table_lookup(hash, key);
	}
	return NULL;
	}

	extern pe_action_t get_pseudo_op(const char name, pe_working_set_t * data_set);
	extern gboolean order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order);

	void pe__show_node_weights_as(const char file, const char function,
	int line, bool to_log, pe_resource_t *rsc,
	const char comment, GHashTable nodes,
	pe_working_set_t *data_set);

	#define pe__show_node_weights(level, rsc, text, nodes, data_set) \
	pe__show_node_weights_as(__FILE__, __func__, __LINE__, \
	(level), (rsc), (text), (nodes), (data_set))

	xmlNode find_rsc_op_entry(const pe_resource_t rsc, const char *key);

	pe_action_t custom_action(pe_resource_t rsc, char key, const char task,
	const pe_node_t *on_node, gboolean optional,
	gboolean foo, pe_working_set_t *data_set);

	# define delete_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DELETE, 0)
	# define delete_action(rsc, node, optional) custom_action( \
	rsc, delete_key(rsc), CRMD_ACTION_DELETE, node, \
	optional, TRUE, rsc->cluster);

	# define stopped_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
	# define stopped_action(rsc, node, optional) custom_action( \
	rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node, \
	optional, TRUE, rsc->cluster);

	# define stop_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOP, 0)
	# define stop_action(rsc, node, optional) custom_action( \
	rsc, stop_key(rsc), CRMD_ACTION_STOP, node, \
	optional, TRUE, rsc->cluster);

	# define reload_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_RELOAD_AGENT, 0)
	# define start_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_START, 0)
	# define start_action(rsc, node, optional) custom_action( \
	rsc, start_key(rsc), CRMD_ACTION_START, node, \
	optional, TRUE, rsc->cluster)

	# define started_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STARTED, 0)
	# define started_action(rsc, node, optional) custom_action( \
	rsc, started_key(rsc), CRMD_ACTION_STARTED, node, \
	optional, TRUE, rsc->cluster)

	# define promote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
	# define promote_action(rsc, node, optional) custom_action( \
	rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node, \
	optional, TRUE, rsc->cluster)

	# define promoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
	# define promoted_action(rsc, node, optional) custom_action( \
	rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node, \
	optional, TRUE, rsc->cluster)

	# define demote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
	# define demote_action(rsc, node, optional) custom_action( \
	rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node, \
	optional, TRUE, rsc->cluster)

	# define demoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
	# define demoted_action(rsc, node, optional) custom_action( \
	rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node, \
	optional, TRUE, rsc->cluster)

	extern int pe_get_configured_timeout(pe_resource_t rsc, const char action,
	pe_working_set_t *data_set);

	pe_action_t find_first_action(const GList input, const char *uuid,
	const char task, const pe_node_t on_node);
	extern enum action_tasks get_complex_task(pe_resource_t * rsc, const char *name,
	gboolean allow_non_atomic);

	extern GList find_actions(GList input, const char key, const pe_node_t on_node);
	GList find_actions_exact(GList input, const char *key,
	const pe_node_t *on_node);
	GList pe__resource_actions(const pe_resource_t rsc, const pe_node_t *node,
	const char *task, bool require_node);

	extern void pe_free_action(pe_action_t * action);

	extern void resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
	pe_working_set_t * data_set);

	extern int pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b,
	bool same_node_default);
	extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
	extern gboolean get_target_role(pe_resource_t * rsc, enum rsc_role_e *role);
	void pe__set_next_role(pe_resource_t *rsc, enum rsc_role_e role,
	const char *why);

	extern pe_resource_t find_clone_instance(pe_resource_t rsc, const char *sub_id,
	pe_working_set_t * data_set);

	extern void destroy_ticket(gpointer data);
	extern pe_ticket_t ticket_new(const char ticket_id, pe_working_set_t * data_set);

	// Resources for manipulating resource names
	const char pe_base_name_end(const char id);
	char clone_strip(const char last_rsc_id);
	char clone_zero(const char last_rsc_id);

	static inline bool
	pe_base_name_eq(pe_resource_t rsc, const char id)
	{
	if (id && rsc && rsc->id) {
	// Number of characters in rsc->id before any clone suffix
	size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;

	return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
	}
	return false;
	}

	int pe__target_rc_from_xml(const xmlNode *xml_op);

	gint pe__cmp_node_name(gconstpointer a, gconstpointer b);
	bool is_set_recursive(const pe_resource_t *rsc, long long flag, bool any);

	enum rsc_digest_cmp_val {
	/! Digests are the same /
	RSC_DIGEST_MATCH = 0,
	/! Params that require a restart changed /
	RSC_DIGEST_RESTART,
	/! Some parameter changed. /
	RSC_DIGEST_ALL,
	/*! rsc op didn't have a digest associated with it, so
	* it is unknown if parameters changed or not. */
	RSC_DIGEST_UNKNOWN,
	};

	typedef struct op_digest_cache_s {
	enum rsc_digest_cmp_val rc;
	xmlNode *params_all;
	xmlNode *params_secure;
	xmlNode *params_restart;
	char *digest_all_calc;
	char *digest_secure_calc;
	char *digest_restart_calc;
	} op_digest_cache_t;

	op_digest_cache_t pe__calculate_digests(pe_resource_t rsc, const char *task,
	- guint interval_ms, pe_node_t node,
	- xmlNode xml_op, GHashTable overrides,
	+ guint *interval_ms,
	+ const pe_node_t *node,
	+ const xmlNode *xml_op,
	+ GHashTable *overrides,
	bool calc_secure,
	pe_working_set_t *data_set);

	void pe__free_digests(gpointer ptr);

	-op_digest_cache_t rsc_action_digest_cmp(pe_resource_t rsc, xmlNode * xml_op, pe_node_t * node,
	- pe_working_set_t * data_set);
	+op_digest_cache_t rsc_action_digest_cmp(pe_resource_t rsc,
	+ const xmlNode *xml_op,
	+ const pe_node_t *node,
	+ pe_working_set_t *data_set);

	-pe_action_t pe_fence_op(pe_node_t node, const char op, bool optional, const char reason, bool priority_delay, pe_working_set_t * data_set);
	-void trigger_unfencing(
	- pe_resource_t * rsc, pe_node_t node, const char reason, pe_action_t dependency, pe_working_set_t data_set);
	+pe_action_t pe_fence_op(const pe_node_t node, const char *op, bool optional,
	+ const char *reason, bool priority_delay,
	+ pe_working_set_t *data_set);
	+void trigger_unfencing(pe_resource_t rsc, const pe_node_t node,
	+ const char reason, pe_action_t dependency,
	+ pe_working_set_t *data_set);

	char pe__action2reason(pe_action_t action, enum pe_action_flags flag);
	void pe_action_set_reason(pe_action_t action, const char reason, bool overwrite);
	void pe__add_action_expected_result(pe_action_t *action, int expected_result);

	void pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_on_all(pe_working_set_t *data_set, uint64_t flag);

	gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref);

	//! \deprecated This function will be removed in a future release
	void print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void * print_data, gboolean print_all);
	int pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, unsigned int options);
	void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay);

	pe_node_t pe_create_node(const char id, const char uname, const char type,
	const char score, pe_working_set_t data_set);

	//! \deprecated This function will be removed in a future release
	void common_print(pe_resource_t rsc, const char pre_text, const char *name,
	pe_node_t node, long options, void print_data);
	int pe__common_output_text(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, unsigned int options);
	int pe__common_output_html(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, unsigned int options);
	pe_resource_t pe__find_bundle_replica(const pe_resource_t bundle,
	const pe_node_t *node);
	bool pe__bundle_needs_remote_name(pe_resource_t *rsc);
	const char pe__add_bundle_remote_name(pe_resource_t rsc,
	pe_working_set_t *data_set,
	xmlNode xml, const char field);
	const char pe_node_attribute_calculated(const pe_node_t node,
	const char *name,
	const pe_resource_t *rsc);
	const char pe_node_attribute_raw(const pe_node_t node, const char *name);
	bool pe__is_universal_clone(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	-void pe__add_param_check(xmlNode rsc_op, pe_resource_t rsc, pe_node_t *node,
	- enum pe_check_parameters, pe_working_set_t *data_set);
	+void pe__add_param_check(const xmlNode rsc_op, pe_resource_t rsc,
	+ const pe_node_t *node, enum pe_check_parameters,
	+ pe_working_set_t *data_set);
	void pe__foreach_param_check(pe_working_set_t *data_set,
	- void (cb)(pe_resource_t, pe_node_t, xmlNode,
	- enum pe_check_parameters,
	- pe_working_set_t*));
	+ void (cb)(pe_resource_t, const pe_node_t*,
	+ const xmlNode*,
	+ enum pe_check_parameters));
	void pe__free_param_checks(pe_working_set_t *data_set);

	bool pe__shutdown_requested(pe_node_t *node);
	void pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set);

	/*!
	* \internal
	* \brief Register xml formatting message functions.
	*
	* \param[in,out] out Output object to register messages with
	*/
	void pe__register_messages(pcmk__output_t *out);

	void pe__unpack_dataset_nvpairs(const xmlNode xml_obj, const char set_name,
	pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	pe_working_set_t *data_set);

	bool pe__resource_is_disabled(pe_resource_t *rsc);
	pe_action_t pe__clear_resource_history(pe_resource_t rsc, pe_node_t *node,
	pe_working_set_t *data_set);

	GList pe__rscs_with_tag(pe_working_set_t data_set, const char *tag_name);
	GList pe__unames_with_tag(pe_working_set_t data_set, const char *tag_name);
	bool pe__rsc_has_tag(pe_working_set_t data_set, const char rsc, const char *tag);
	bool pe__uname_has_tag(pe_working_set_t data_set, const char node, const char *tag);

	bool pe__rsc_running_on_any(pe_resource_t rsc, GList node_list);
	GList pe__filter_rsc_list(GList rscs, GList *filter);
	GList * pe__build_node_name_list(pe_working_set_t data_set, const char s);
	GList * pe__build_rsc_list(pe_working_set_t data_set, const char s);

	bool pcmk__rsc_filtered_by_node(pe_resource_t rsc, GList only_node);

	gboolean pe__bundle_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__clone_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__group_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__native_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);

	xmlNode pe__failed_probe_for_rsc(pe_resource_t rsc, const char *name);

	const char pe__clone_child_id(pe_resource_t rsc);

	int pe__sum_node_health_scores(const pe_node_t *node, int base_health);
	int pe__node_health(pe_node_t *node);

	static inline enum pcmk__health_strategy
	pe__health_strategy(pe_working_set_t *data_set)
	{
	return pcmk__parse_health_strategy(pe_pref(data_set->config_hash,
	PCMK__OPT_NODE_HEALTH_STRATEGY));
	}

	static inline int
	pe__health_score(const char option, pe_working_set_t data_set)
	{
	return char2score(pe_pref(data_set->config_hash, option));
	}

	/*!
	* \internal
	* \brief Return a string suitable for logging as a node name
	*
	* \param[in] node Node to return a node name string for
	*
	* \return Node name if available, otherwise node ID if available,
	* otherwise "unspecified node" if node is NULL or "unidentified node"
	* if node has neither a name nor ID.
	*/
	static inline const char *
	pe__node_name(const pe_node_t *node)
	{
	if (node == NULL) {
	return "unspecified node";

	} else if (node->details->uname != NULL) {
	return node->details->uname;

	} else if (node->details->id != NULL) {
	return node->details->id;

	} else {
	return "unidentified node";
	}
	}

	/*!
	* \internal
	* \brief Check whether two node objects refer to the same node
	*
	* \param[in] node1 First node object to compare
	* \param[in] node2 Second node object to compare
	*
	* \return true if \p node1 and \p node2 refer to the same node
	*/
	static inline bool
	pe__same_node(const pe_node_t node1, const pe_node_t node2)
	{
	return (node1 != NULL) && (node2 != NULL)
	&& (node1->details == node2->details);
	}
	#endif
	diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h
	index 6079fdb3a5..269993e057 100644
	--- a/lib/pacemaker/libpacemaker_private.h
	+++ b/lib/pacemaker/libpacemaker_private.h
	@@ -1,823 +1,823 @@
	/*
	* Copyright 2021-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__LIBPACEMAKER_PRIVATE__H
	# define PCMK__LIBPACEMAKER_PRIVATE__H

	/* This header is for the sole use of libpacemaker, so that functions can be
	* declared with G_GNUC_INTERNAL for efficiency.
	*/

	#include <crm/pengine/pe_types.h> // pe_action_t, pe_node_t, pe_working_set_t

	// Flags to modify the behavior of pcmk__add_colocated_node_scores()
	enum pcmk__coloc_select {
	// With no other flags, apply all "with this" colocations
	pcmk__coloc_select_default = 0,

	// Apply "this with" colocations instead of "with this" colocations
	pcmk__coloc_select_this_with = (1 << 0),

	// Apply only colocations with non-negative scores
	pcmk__coloc_select_nonnegative = (1 << 1),

	// Apply only colocations with at least one matching node
	pcmk__coloc_select_active = (1 << 2),
	};

	// Flags the update_ordered_actions() method can return
	enum pcmk__updated {
	pcmk__updated_none = 0, // Nothing changed
	pcmk__updated_first = (1 << 0), // First action was updated
	pcmk__updated_then = (1 << 1), // Then action was updated
	};

	#define pcmk__set_updated_flags(au_flags, action, flags_to_set) do { \
	au_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action update", \
	(action)->uuid, au_flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pcmk__clear_updated_flags(au_flags, action, flags_to_clear) do { \
	au_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action update", \
	(action)->uuid, au_flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	// Resource allocation methods
	struct resource_alloc_functions_s {
	/*!
	* \internal
	* \brief Assign a resource to a node
	*
	* \param[in,out] rsc Resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*/
	pe_node_t (assign)(pe_resource_t rsc, const pe_node_t prefer);

	/*!
	* \internal
	* \brief Create all actions needed for a given resource
	*
	* \param[in,out] rsc Resource to create actions for
	*/
	void (create_actions)(pe_resource_t rsc);

	/*!
	* \internal
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in] rsc Resource to create probe for
	* \param[in] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool (create_probe)(pe_resource_t rsc, pe_node_t *node);

	/*!
	* \internal
	* \brief Create implicit constraints needed for a resource
	*
	* \param[in,out] rsc Resource to create implicit constraints for
	*/
	void (internal_constraints)(pe_resource_t rsc);

	/*!
	* \internal
	* \brief Apply a colocation's score to node weights or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node weights (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void (apply_coloc_score) (pe_resource_t dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	/*!
	* \internal
	* \brief Create list of all resources in colocations with a given resource
	*
	* Given a resource, create a list of all resources involved in mandatory
	* colocations with it, whether directly or indirectly via chained colocations.
	*
	* \param[in] rsc Resource to add to colocated list
	* \param[in] orig_rsc Resource originally requested
	* \param[in] colocated_rscs Existing list
	*
	* \return List of given resource and all resources involved in colocations
	*
	* \note This function is recursive; top-level callers should pass NULL as
	* \p colocated_rscs and \p orig_rsc, and the desired resource as
	* \p rsc. The recursive calls will use other values.
	*/
	GList (colocated_resources)(pe_resource_t rsc, pe_resource_t orig_rsc,
	GList *colocated_rscs);

	/*!
	* \internal
	* \brief Apply a location constraint to a resource's allowed node scores
	*
	* \param[in,out] rsc Resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*/
	void (apply_location)(pe_resource_t rsc, pe__location_t *location);

	/*!
	* \internal
	* \brief Return action flags for a given resource action
	*
	* \param[in,out] action Action to get flags for
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	* \note For primitives, this will be the same as action->flags regardless
	* of node. For collective resources, the flags can differ due to
	* multiple instances possibly being involved.
	*/
	enum pe_action_flags (action_flags)(pe_action_t action,
	const pe_node_t *node);

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions'
	* flags (and runnable_before members if appropriate) as appropriate for the
	* ordering. In some cases, the ordering could be disabled as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this
	* node (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates
	* (may include pe_action_optional to affect only
	* mandatory actions, and pe_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pe_ordering flags to apply
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t (update_ordered_actions)(pe_action_t first, pe_action_t *then,
	const pe_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pe_working_set_t *data_set);

	void (output_actions)(pe_resource_t rsc);

	/*!
	* \internal
	* \brief Add a resource's actions to the transition graph
	*
	* \param[in] rsc Resource whose actions should be added
	*/
	void (add_actions_to_graph)(pe_resource_t rsc);

	/*!
	* \internal
	* \brief Add meta-attributes relevant to transition graph actions to XML
	*
	* If a given resource supports variant-specific meta-attributes that are
	* needed for transition graph actions, add them to a given XML element.
	*
	* \param[in] rsc Resource whose meta-attributes should be added
	* \param[in,out] xml Transition graph action attributes XML to add to
	*/
	void (add_graph_meta)(pe_resource_t rsc, xmlNode *xml);

	/*!
	* \internal
	* \brief Add a resource's utilization to a table of utilization values
	*
	* This function is used when summing the utilization of a resource and all
	* resources colocated with it, to determine whether a node has sufficient
	* capacity. Given a resource and a table of utilization values, it will add
	* the resource's utilization to the existing values, if the resource has
	* not yet been allocated to a node.
	*
	* \param[in] rsc Resource with utilization to add
	* \param[in] orig_rsc Resource being allocated (for logging only)
	* \param[in] all_rscs List of all resources that will be summed
	* \param[in,out] utilization Table of utilization values to add to
	*/
	void (add_utilization)(const pe_resource_t rsc,
	const pe_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization);

	/*!
	* \internal
	* \brief Apply a shutdown lock for a resource, if appropriate
	*
	* \param[in] rsc Resource to check for shutdown lock
	*/
	void (shutdown_lock)(pe_resource_t rsc);
	};

	// Actions (pcmk_sched_actions.c)

	G_GNUC_INTERNAL
	void pcmk__update_action_for_orderings(pe_action_t *action,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	uint32_t pcmk__update_ordered_actions(pe_action_t first, pe_action_t then,
	const pe_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__log_action(const char pre_text, pe_action_t action, bool details);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_cancel_action(pe_resource_t rsc, const char *name,
	guint interval_ms, const pe_node_t *node);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_shutdown_action(pe_node_t node);

	G_GNUC_INTERNAL
	bool pcmk__action_locks_rsc_to_node(const pe_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__deduplicate_action_inputs(pe_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__output_actions(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	-bool pcmk__check_action_config(pe_resource_t rsc, pe_node_t node,
	- xmlNode *xml_op);
	+bool pcmk__check_action_config(pe_resource_t rsc, const pe_node_t node,
	+ const xmlNode *xml_op);

	G_GNUC_INTERNAL
	void pcmk__handle_rsc_config_changes(pe_working_set_t *data_set);


	// Recurring actions (pcmk_sched_recurring.c)

	G_GNUC_INTERNAL
	void pcmk__create_recurring_actions(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__schedule_cancel(pe_resource_t rsc, const char call_id,
	const char *task, guint interval_ms,
	const pe_node_t node, const char reason);

	G_GNUC_INTERNAL
	void pcmk__reschedule_recurring(pe_resource_t rsc, const char task,
	- guint interval_ms, pe_node_t *node);
	+ guint interval_ms, const pe_node_t *node);

	G_GNUC_INTERNAL
	bool pcmk__action_is_recurring(const pe_action_t *action);


	// Producing transition graphs (pcmk_graph_producer.c)

	G_GNUC_INTERNAL
	bool pcmk__graph_has_loop(pe_action_t init_action, pe_action_t action,
	pe_action_wrapper_t *input);

	G_GNUC_INTERNAL
	void pcmk__add_rsc_actions_to_graph(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__create_graph(pe_working_set_t *data_set);


	// Fencing (pcmk_sched_fencing.c)

	G_GNUC_INTERNAL
	void pcmk__order_vs_fence(pe_action_t stonith_op, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__order_vs_unfence(pe_resource_t rsc, pe_node_t node,
	pe_action_t *action, enum pe_ordering order);

	G_GNUC_INTERNAL
	void pcmk__fence_guest(pe_node_t *node);

	G_GNUC_INTERNAL
	bool pcmk__node_unfenced(pe_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__order_restart_vs_unfence(gpointer data, gpointer user_data);


	// Injected scheduler inputs (pcmk_sched_injections.c)

	void pcmk__inject_scheduler_input(pe_working_set_t data_set, cib_t cib,
	pcmk_injections_t *injections);


	// Constraints of any type (pcmk_sched_constraints.c)

	G_GNUC_INTERNAL
	pe_resource_t pcmk__find_constraint_resource(GList rsc_list, const char *id);

	G_GNUC_INTERNAL
	xmlNode pcmk__expand_tags_in_sets(xmlNode xml_obj,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__valid_resource_or_tag(pe_working_set_t data_set, const char id,
	pe_resource_t rsc, pe_tag_t tag);

	G_GNUC_INTERNAL
	bool pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	bool convert_rsc, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__create_internal_constraints(pe_working_set_t *data_set);


	// Location constraints

	G_GNUC_INTERNAL
	void pcmk__unpack_location(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	pe__location_t pcmk__new_location(const char id, pe_resource_t *rsc,
	int node_weight, const char *discover_mode,
	pe_node_t *foo_node,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_locations(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_location(pe_resource_t rsc, pe__location_t constraint);


	// Colocation constraints (pcmk_sched_colocation.c)

	enum pcmk__coloc_affects {
	pcmk__coloc_affects_nothing = 0,
	pcmk__coloc_affects_location,
	pcmk__coloc_affects_role,
	};

	G_GNUC_INTERNAL
	enum pcmk__coloc_affects pcmk__colocation_affects(const pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool preview);

	G_GNUC_INTERNAL
	void pcmk__apply_coloc_to_weights(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation);

	G_GNUC_INTERNAL
	void pcmk__apply_coloc_to_priority(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation);

	G_GNUC_INTERNAL
	void pcmk__add_colocated_node_scores(pe_resource_t rsc, const char log_id,
	GHashTable *nodes, const char attr,
	float factor, uint32_t flags);

	G_GNUC_INTERNAL
	void pcmk__unpack_colocation(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__add_this_with(pe_resource_t rsc, pcmk__colocation_t colocation);

	G_GNUC_INTERNAL
	void pcmk__add_with_this(pe_resource_t rsc, pcmk__colocation_t colocation);

	G_GNUC_INTERNAL
	void pcmk__new_colocation(const char id, const char node_attr, int score,
	pe_resource_t dependent, pe_resource_t primary,
	const char dependent_role, const char primary_role,
	bool influence, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__block_colocation_dependents(pe_action_t *action,
	pe_working_set_t *data_set);

	/*!
	* \internal
	* \brief Check whether colocation's dependent preferences should be considered
	*
	* \param[in] colocation Colocation constraint
	* \param[in] rsc Primary instance (normally this will be
	* colocation->primary, which NULL will be treated as,
	* but for clones or bundles with multiple instances
	* this can be a particular instance)
	*
	* \return true if colocation influence should be effective, otherwise false
	*/
	static inline bool
	pcmk__colocation_has_influence(const pcmk__colocation_t *colocation,
	const pe_resource_t *rsc)
	{
	if (rsc == NULL) {
	rsc = colocation->primary;
	}

	/* A bundle replica colocates its remote connection with its container,
	* using a finite score so that the container can run on Pacemaker Remote
	* nodes.
	*
	* Moving a connection is lightweight and does not interrupt the service,
	* while moving a container is heavyweight and does interrupt the service,
	* so don't move a clean, active container based solely on the preferences
	* of its connection.
	*
	* This also avoids problematic scenarios where two containers want to
	* perpetually swap places.
	*/
	if (pcmk_is_set(colocation->dependent->flags, pe_rsc_allow_remote_remotes)
	&& !pcmk_is_set(rsc->flags, pe_rsc_failed)
	&& pcmk__list_of_1(rsc->running_on)) {
	return false;
	}

	/* The dependent in a colocation influences the primary's location
	* if the influence option is true or the primary is not yet active.
	*/
	return colocation->influence \|\| (rsc->running_on == NULL);
	}


	// Ordering constraints (pcmk_sched_ordering.c)

	G_GNUC_INTERNAL
	void pcmk__new_ordering(pe_resource_t first_rsc, char first_task,
	pe_action_t first_action, pe_resource_t then_rsc,
	char then_task, pe_action_t then_action,
	uint32_t flags, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__unpack_ordering(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__disable_invalid_orderings(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__order_stops_before_shutdown(pe_node_t *node,
	pe_action_t *shutdown_op);

	G_GNUC_INTERNAL
	void pcmk__apply_orderings(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__order_after_each(pe_action_t after, GList list);


	/*!
	* \internal
	* \brief Create a new ordering between two resource actions
	*
	* \param[in] first_rsc Resource for 'first' action
	* \param[in] then_rsc Resource for 'then' action
	* \param[in] first_task Action key for 'first' action
	* \param[in] then_task Action key for 'then' action
	* \param[in] flags Bitmask of enum pe_ordering flags
	* \param[in] data_set Cluster working set to add ordering to
	*/
	#define pcmk__order_resource_actions(first_rsc, first_task, \
	then_rsc, then_task, flags) \
	pcmk__new_ordering((first_rsc), \
	pcmk__op_key((first_rsc)->id, (first_task), 0), \
	NULL, \
	(then_rsc), \
	pcmk__op_key((then_rsc)->id, (then_task), 0), \
	NULL, (flags), (first_rsc)->cluster)

	#define pcmk__order_starts(rsc1, rsc2, flags) \
	pcmk__order_resource_actions((rsc1), CRMD_ACTION_START, \
	(rsc2), CRMD_ACTION_START, (flags))

	#define pcmk__order_stops(rsc1, rsc2, flags) \
	pcmk__order_resource_actions((rsc1), CRMD_ACTION_STOP, \
	(rsc2), CRMD_ACTION_STOP, (flags))


	// Ticket constraints (pcmk_sched_tickets.c)

	G_GNUC_INTERNAL
	void pcmk__unpack_rsc_ticket(xmlNode xml_obj, pe_working_set_t data_set);


	// Promotable clone resources (pcmk_sched_promotable.c)

	G_GNUC_INTERNAL
	void pcmk__add_promotion_scores(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__require_promotion_tickets(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__set_instance_roles(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__create_promotable_actions(pe_resource_t *clone);

	G_GNUC_INTERNAL
	void pcmk__promotable_restart_ordering(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__order_promotable_instances(pe_resource_t *clone);

	G_GNUC_INTERNAL
	void pcmk__update_dependent_with_promotable(const pe_resource_t *primary,
	pe_resource_t *dependent,
	const pcmk__colocation_t *colocation);

	G_GNUC_INTERNAL
	void pcmk__update_promotable_dependent_priority(const pe_resource_t *primary,
	pe_resource_t *dependent,
	const pcmk__colocation_t *colocation);


	// Pacemaker Remote nodes (pcmk_sched_remote.c)

	G_GNUC_INTERNAL
	bool pcmk__is_failed_remote_node(pe_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__order_remote_connection_actions(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__rsc_corresponds_to_guest(pe_resource_t rsc, pe_node_t node);

	G_GNUC_INTERNAL
	pe_node_t pcmk__connection_host_for_action(pe_action_t action);

	G_GNUC_INTERNAL
	void pcmk__substitute_remote_addr(pe_resource_t rsc, GHashTable params);

	G_GNUC_INTERNAL
	void pcmk__add_bundle_meta_to_xml(xmlNode args_xml, pe_action_t action);


	// Primitives (pcmk_sched_primitive.c)

	G_GNUC_INTERNAL
	pe_node_t pcmk__primitive_assign(pe_resource_t rsc, const pe_node_t *prefer);

	G_GNUC_INTERNAL
	void pcmk__primitive_create_actions(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__primitive_internal_constraints(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	enum pe_action_flags pcmk__primitive_action_flags(pe_action_t *action,
	const pe_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__primitive_apply_coloc_score(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	G_GNUC_INTERNAL
	void pcmk__schedule_cleanup(pe_resource_t rsc, const pe_node_t node,
	bool optional);

	G_GNUC_INTERNAL
	void pcmk__primitive_add_graph_meta(pe_resource_t rsc, xmlNode xml);

	G_GNUC_INTERNAL
	void pcmk__primitive_add_utilization(const pe_resource_t *rsc,
	const pe_resource_t *orig_rsc,
	GList all_rscs, GHashTable utilization);

	G_GNUC_INTERNAL
	void pcmk__primitive_shutdown_lock(pe_resource_t *rsc);


	// Groups (pcmk_sched_group.c)

	G_GNUC_INTERNAL
	pe_node_t pcmk__group_assign(pe_resource_t rsc, const pe_node_t *prefer);

	G_GNUC_INTERNAL
	void pcmk__group_create_actions(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__group_internal_constraints(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__group_apply_coloc_score(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	G_GNUC_INTERNAL
	void pcmk__group_apply_location(pe_resource_t rsc, pe__location_t location);

	G_GNUC_INTERNAL
	enum pe_action_flags pcmk__group_action_flags(pe_action_t *action,
	const pe_node_t *node);

	G_GNUC_INTERNAL
	uint32_t pcmk__group_update_ordered_actions(pe_action_t *first,
	pe_action_t *then,
	const pe_node_t *node,
	uint32_t flags, uint32_t filter,
	uint32_t type,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	GList pcmk__group_colocated_resources(pe_resource_t rsc,
	pe_resource_t *orig_rsc,
	GList *colocated_rscs);

	G_GNUC_INTERNAL
	void pcmk__group_add_utilization(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization);

	G_GNUC_INTERNAL
	void pcmk__group_shutdown_lock(pe_resource_t *rsc);


	// Clones (pcmk_sched_clone.c)

	G_GNUC_INTERNAL
	void pcmk__clone_apply_coloc_score(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	// Bundles (pcmk_sched_bundle.c)

	G_GNUC_INTERNAL
	void pcmk__bundle_apply_coloc_score(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	G_GNUC_INTERNAL
	void pcmk__output_bundle_actions(pe_resource_t *rsc);


	// Injections (pcmk_injections.c)

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_node(cib_t cib_conn, const char node, const char uuid);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_node_state_change(cib_t cib_conn, const char *node,
	bool up);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_resource_history(pcmk__output_t out, xmlNode *cib_node,
	const char *resource,
	const char *lrm_name,
	const char *rclass,
	const char *rtype,
	const char *rprovider);

	G_GNUC_INTERNAL
	void pcmk__inject_failcount(pcmk__output_t out, xmlNode cib_node,
	const char resource, const char task,
	guint interval_ms, int rc);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_action_result(xmlNode cib_resource,
	lrmd_event_data_t *op, int target_rc);


	// Nodes (pcmk_sched_nodes.c)

	G_GNUC_INTERNAL
	bool pcmk__node_available(const pe_node_t *node, bool consider_score,
	bool consider_guest);

	G_GNUC_INTERNAL
	bool pcmk__any_node_available(GHashTable *nodes);

	G_GNUC_INTERNAL
	GHashTable pcmk__copy_node_table(GHashTable nodes);

	G_GNUC_INTERNAL
	GList pcmk__sort_nodes(GList nodes, pe_node_t *active_node);

	G_GNUC_INTERNAL
	void pcmk__apply_node_health(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	pe_node_t pcmk__top_allowed_node(const pe_resource_t rsc,
	const pe_node_t *node);


	// Functions applying to more than one variant (pcmk_sched_resource.c)

	G_GNUC_INTERNAL
	void pcmk__set_allocation_methods(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__rsc_agent_changed(pe_resource_t rsc, pe_node_t node,
	const xmlNode *rsc_entry, bool active_on_node);

	G_GNUC_INTERNAL
	GList pcmk__rscs_matching_id(const char id, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	GList pcmk__colocated_resources(pe_resource_t rsc, pe_resource_t *orig_rsc,
	GList *colocated_rscs);

	G_GNUC_INTERNAL
	void pcmk__noop_add_graph_meta(pe_resource_t rsc, xmlNode xml);

	G_GNUC_INTERNAL
	void pcmk__output_resource_actions(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__finalize_assignment(pe_resource_t rsc, pe_node_t chosen,
	bool force);

	G_GNUC_INTERNAL
	bool pcmk__assign_resource(pe_resource_t rsc, pe_node_t node, bool force);

	G_GNUC_INTERNAL
	void pcmk__unassign_resource(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__threshold_reached(pe_resource_t rsc, pe_node_t node,
	pe_resource_t **failed);

	G_GNUC_INTERNAL
	void pcmk__sort_resources(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	gint pcmk__cmp_instance(gconstpointer a, gconstpointer b);

	G_GNUC_INTERNAL
	gint pcmk__cmp_instance_number(gconstpointer a, gconstpointer b);


	// Functions related to probes (pcmk_sched_probes.c)

	G_GNUC_INTERNAL
	bool pcmk__probe_rsc_on_node(pe_resource_t rsc, pe_node_t node);

	G_GNUC_INTERNAL
	void pcmk__order_probes(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__probe_resource_list(GList rscs, pe_node_t node);

	G_GNUC_INTERNAL
	void pcmk__schedule_probes(pe_working_set_t *data_set);


	// Functions related to live migration (pcmk_sched_migration.c)

	void pcmk__create_migration_actions(pe_resource_t *rsc,
	const pe_node_t *current);

	void pcmk__abort_dangling_migration(void data, void user_data);

	bool pcmk__rsc_can_migrate(const pe_resource_t rsc, const pe_node_t current);

	void pcmk__order_migration_equivalents(pe__ordering_t *order);


	// Functions related to node utilization (pcmk_sched_utilization.c)

	G_GNUC_INTERNAL
	int pcmk__compare_node_capacities(const pe_node_t *node1,
	const pe_node_t *node2);

	G_GNUC_INTERNAL
	void pcmk__consume_node_capacity(GHashTable *current_utilization,
	pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__release_node_capacity(GHashTable *current_utilization,
	const pe_resource_t *rsc);

	G_GNUC_INTERNAL
	const pe_node_t pcmk__ban_insufficient_capacity(pe_resource_t rsc);

	G_GNUC_INTERNAL
	void pcmk__create_utilization_constraints(pe_resource_t *rsc,
	GList *allowed_nodes);

	G_GNUC_INTERNAL
	void pcmk__show_node_capacities(const char desc, pe_working_set_t data_set);

	#endif // PCMK__LIBPACEMAKER_PRIVATE__H
	diff --git a/lib/pacemaker/pcmk_sched_actions.c b/lib/pacemaker/pcmk_sched_actions.c
	index 05760ced90..1f9aab1543 100644
	--- a/lib/pacemaker/pcmk_sched_actions.c
	+++ b/lib/pacemaker/pcmk_sched_actions.c
	@@ -1,1928 +1,1930 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/param.h>
	#include <glib.h>

	#include <crm/lrmd_internal.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Get the action flags relevant to ordering constraints
	*
	* \param[in] action Action to check
	* \param[in] node Node that other action in the ordering is on
	* (used only for clone resource actions)
	*
	* \return Action flags that should be used for orderings
	*/
	static enum pe_action_flags
	action_flags_for_ordering(pe_action_t action, pe_node_t node)
	{
	bool runnable = false;
	enum pe_action_flags flags;

	// For non-resource actions, return the action flags
	if (action->rsc == NULL) {
	return action->flags;
	}

	/* For non-clone resources, or a clone action not assigned to a node,
	* return the flags as determined by the resource method without a node
	* specified.
	*/
	flags = action->rsc->cmds->action_flags(action, NULL);
	if ((node == NULL) \|\| !pe_rsc_is_clone(action->rsc)) {
	return flags;
	}

	/* Otherwise (i.e., for clone resource actions on a specific node), first
	* remember whether the non-node-specific action is runnable.
	*/
	runnable = pcmk_is_set(flags, pe_action_runnable);

	// Then recheck the resource method with the node
	flags = action->rsc->cmds->action_flags(action, node);

	/* For clones in ordering constraints, the node-specific "runnable" doesn't
	* matter, just the non-node-specific setting (i.e., is the action runnable
	* anywhere).
	*
	* This applies only to runnable, and only for ordering constraints. This
	* function shouldn't be used for other types of constraints without
	* changes. Not very satisfying, but it's logical and appears to work well.
	*/
	if (runnable && !pcmk_is_set(flags, pe_action_runnable)) {
	pe__set_raw_action_flags(flags, action->rsc->id,
	pe_action_runnable);
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Get action UUID that should be used with a resource ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the UUID and resource of the first action in an
	* ordering, this returns the UUID of the action that should actually be used
	* for ordering (for example, "CLONE_started_0" instead of "CLONE_start_0").
	*
	* \param[in] first_uuid UUID of first action in ordering
	* \param[in] first_rsc Resource of first action in ordering
	*
	* \return Newly allocated copy of UUID to use with ordering
	* \note It is the caller's responsibility to free the return value.
	*/
	static char *
	action_uuid_for_ordering(const char first_uuid, pe_resource_t first_rsc)
	{
	guint interval_ms = 0;
	char *uuid = NULL;
	char *rid = NULL;
	char *first_task_str = NULL;
	enum action_tasks first_task = no_action;
	enum action_tasks remapped_task = no_action;

	// Only non-notify actions for collective resources need remapping
	if ((strstr(first_uuid, "notify") != NULL)
	\|\| (first_rsc->variant < pe_group)) {
	goto done;
	}

	// Only non-recurring actions need remapping
	CRM_ASSERT(parse_op_key(first_uuid, &rid, &first_task_str, &interval_ms));
	if (interval_ms > 0) {
	goto done;
	}

	first_task = text2task(first_task_str);
	switch (first_task) {
	case stop_rsc:
	case start_rsc:
	case action_notify:
	case action_promote:
	case action_demote:
	remapped_task = first_task + 1;
	break;
	case stopped_rsc:
	case started_rsc:
	case action_notified:
	case action_promoted:
	case action_demoted:
	remapped_task = first_task;
	break;
	case monitor_rsc:
	case shutdown_crm:
	case stonith_node:
	break;
	default:
	crm_err("Unknown action '%s' in ordering", first_task_str);
	break;
	}

	if (remapped_task != no_action) {
	/* If a (clone) resource has notifications enabled, we want to order
	* relative to when all notifications have been sent for the remapped
	* task. Only outermost resources or those in bundles have
	* notifications.
	*/
	if (pcmk_is_set(first_rsc->flags, pe_rsc_notify)
	&& ((first_rsc->parent == NULL)
	\|\| (pe_rsc_is_clone(first_rsc)
	&& (first_rsc->parent->variant == pe_container)))) {
	uuid = pcmk__notify_key(rid, "confirmed-post",
	task2text(remapped_task));
	} else {
	uuid = pcmk__op_key(rid, task2text(remapped_task), 0);
	}
	pe_rsc_trace(first_rsc,
	"Remapped action UUID %s to %s for ordering purposes",
	first_uuid, uuid);
	}

	done:
	if (uuid == NULL) {
	uuid = strdup(first_uuid);
	CRM_ASSERT(uuid != NULL);
	}
	free(first_task_str);
	free(rid);
	return uuid;
	}

	/*!
	* \internal
	* \brief Get actual action that should be used with an ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the first action in an ordering, this returns the
	* the action that should actually be used for ordering (for example, the
	* started action instead of the start action).
	*
	* \param[in] action First action in an ordering
	*
	* \return Actual action that should be used for the ordering
	*/
	static pe_action_t *
	action_for_ordering(pe_action_t *action)
	{
	pe_action_t *result = action;
	pe_resource_t *rsc = action->rsc;

	if ((rsc != NULL) && (rsc->variant >= pe_group) && (action->uuid != NULL)) {
	char *uuid = action_uuid_for_ordering(action->uuid, rsc);

	result = find_first_action(rsc->actions, uuid, NULL, NULL);
	if (result == NULL) {
	crm_warn("Not remapping %s to %s because %s does not have "
	"remapped action", action->uuid, uuid, rsc->id);
	result = action;
	}
	free(uuid);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Update flags for ordering's actions appropriately for ordering's flags
	*
	* \param[in] first First action in an ordering
	* \param[in] then Then action in an ordering
	* \param[in] first_flags Action flags for \p first for ordering purposes
	* \param[in] then_flags Action flags for \p then for ordering purposes
	* \param[in] order Action wrapper for \p first in ordering
	* \param[in] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags
	*/
	static uint32_t
	update_action_for_ordering_flags(pe_action_t first, pe_action_t then,
	enum pe_action_flags first_flags,
	enum pe_action_flags then_flags,
	pe_action_wrapper_t *order,
	pe_working_set_t *data_set)
	{
	uint32_t changed = pcmk__updated_none;

	/* The node will only be used for clones. If interleaved, node will be NULL,
	* otherwise the ordering scope will be limited to the node. Normally, the
	* whole 'then' clone should restart if 'first' is restarted, so then->node
	* is needed.
	*/
	pe_node_t *node = then->node;

	if (pcmk_is_set(order->type, pe_order_implies_then_on_node)) {
	/* For unfencing, only instances of 'then' on the same node as 'first'
	* (the unfencing operation) should restart, so reset node to
	* first->node, at which point this case is handled like a normal
	* pe_order_implies_then.
	*/
	pe__clear_order_flags(order->type, pe_order_implies_then_on_node);
	pe__set_order_flags(order->type, pe_order_implies_then);
	node = first->node;
	pe_rsc_trace(then->rsc,
	"%s then %s: mapped pe_order_implies_then_on_node to "
	"pe_order_implies_then on %s",
	first->uuid, then->uuid, pe__node_name(node));
	}

	if (pcmk_is_set(order->type, pe_order_implies_then)) {
	if (then->rsc != NULL) {
	changed \|= then->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags & pe_action_optional,
	pe_action_optional,
	pe_order_implies_then,
	data_set);
	} else if (!pcmk_is_set(first_flags, pe_action_optional)
	&& pcmk_is_set(then->flags, pe_action_optional)) {
	pe__clear_action_flags(then, pe_action_optional);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_implies_then",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_restart) && (then->rsc != NULL)) {
	enum pe_action_flags restart = pe_action_optional\|pe_action_runnable;

	changed \|= then->rsc->cmds->update_ordered_actions(first, then, node,
	first_flags, restart,
	pe_order_restart,
	data_set);
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_restart",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_implies_first)) {
	if (first->rsc != NULL) {
	changed \|= first->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags,
	pe_action_optional,
	pe_order_implies_first,
	data_set);
	} else if (!pcmk_is_set(first_flags, pe_action_optional)
	&& pcmk_is_set(first->flags, pe_action_runnable)) {
	pe__clear_action_flags(first, pe_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_promoted_implies_first)) {
	if (then->rsc != NULL) {
	changed \|= then->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags & pe_action_optional,
	pe_action_optional,
	pe_order_promoted_implies_first,
	data_set);
	}
	pe_rsc_trace(then->rsc,
	"%s then %s: %s after pe_order_promoted_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_one_or_more)) {
	if (then->rsc != NULL) {
	changed \|= then->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags,
	pe_action_runnable,
	pe_order_one_or_more,
	data_set);

	} else if (pcmk_is_set(first_flags, pe_action_runnable)) {
	// We have another runnable instance of "first"
	then->runnable_before++;

	/* Mark "then" as runnable if it requires a certain number of
	* "before" instances to be runnable, and they now are.
	*/
	if ((then->runnable_before >= then->required_runnable_before)
	&& !pcmk_is_set(then->flags, pe_action_runnable)) {

	pe__set_action_flags(then, pe_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_one_or_more",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_probe) && (then->rsc != NULL)) {
	if (!pcmk_is_set(first_flags, pe_action_runnable)
	&& (first->rsc->running_on != NULL)) {

	pe_rsc_trace(then->rsc,
	"%s then %s: ignoring because first is stopping",
	first->uuid, then->uuid);
	order->type = pe_order_none;
	} else {
	changed \|= then->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags,
	pe_action_runnable,
	pe_order_runnable_left,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_probe",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_runnable_left)) {
	if (then->rsc != NULL) {
	changed \|= then->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags,
	pe_action_runnable,
	pe_order_runnable_left,
	data_set);

	} else if (!pcmk_is_set(first_flags, pe_action_runnable)
	&& pcmk_is_set(then->flags, pe_action_runnable)) {

	pe__clear_action_flags(then, pe_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_runnable_left",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_implies_first_migratable)) {
	if (then->rsc != NULL) {
	changed \|= then->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags,
	pe_action_optional,
	pe_order_implies_first_migratable,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after "
	"pe_order_implies_first_migratable",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_pseudo_left)) {
	if (then->rsc != NULL) {
	changed \|= then->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags,
	pe_action_optional,
	pe_order_pseudo_left,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_pseudo_left",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_optional)) {
	if (then->rsc != NULL) {
	changed \|= then->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags,
	pe_action_runnable,
	pe_order_optional,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_optional",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_asymmetrical)) {
	if (then->rsc != NULL) {
	changed \|= then->rsc->cmds->update_ordered_actions(first, then,
	node,
	first_flags,
	pe_action_runnable,
	pe_order_asymmetrical,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_asymmetrical",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(first->flags, pe_action_runnable)
	&& pcmk_is_set(order->type, pe_order_implies_then_printed)
	&& !pcmk_is_set(first_flags, pe_action_optional)) {

	pe_rsc_trace(then->rsc, "%s will be in graph because %s is required",
	then->uuid, first->uuid);
	pe__set_action_flags(then, pe_action_print_always);
	// Don't bother marking 'then' as changed just for this
	}

	if (pcmk_is_set(order->type, pe_order_implies_first_printed)
	&& !pcmk_is_set(then_flags, pe_action_optional)) {

	pe_rsc_trace(then->rsc, "%s will be in graph because %s is required",
	first->uuid, then->uuid);
	pe__set_action_flags(first, pe_action_print_always);
	// Don't bother marking 'first' as changed just for this
	}

	if (pcmk_any_flags_set(order->type, pe_order_implies_then
	\|pe_order_implies_first
	\|pe_order_restart)
	&& (first->rsc != NULL)
	&& !pcmk_is_set(first->rsc->flags, pe_rsc_managed)
	&& pcmk_is_set(first->rsc->flags, pe_rsc_block)
	&& !pcmk_is_set(first->flags, pe_action_runnable)
	&& pcmk__str_eq(first->task, RSC_STOP, pcmk__str_casei)) {

	if (pcmk_is_set(then->flags, pe_action_runnable)) {
	pe__clear_action_flags(then, pe_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after checking whether first "
	"is blocked, unmanaged, unrunnable stop",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	return changed;
	}

	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pe_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pe_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pe_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	/*!
	* \internal
	* \brief Update an action's flags for all orderings where it is "then"
	*
	* \param[in] then Action to update
	* \param[in] data_set Cluster working set
	*/
	void
	pcmk__update_action_for_orderings(pe_action_t then, pe_working_set_t data_set)
	{
	GList *lpc = NULL;
	uint32_t changed = pcmk__updated_none;
	int last_flags = then->flags;

	pe_rsc_trace(then->rsc, "Updating %s %s (%s %s) on %s",
	action_type_str(then->flags), then->uuid,
	action_optional_str(then->flags),
	action_runnable_str(then->flags), action_node_str(then));

	if (pcmk_is_set(then->flags, pe_action_requires_any)) {
	/* Initialize current known "runnable before" actions. As
	* update_action_for_ordering_flags() is called for each of then's
	* before actions, this number will increment as runnable 'first'
	* actions are encountered.
	*/
	then->runnable_before = 0;

	if (then->required_runnable_before == 0) {
	/* @COMPAT This ordering constraint uses the deprecated
	* "require-all=false" attribute. Treat it like "clone-min=1".
	*/
	then->required_runnable_before = 1;
	}

	/* The pe_order_one_or_more clause of update_action_for_ordering_flags()
	* (called below) will reset runnable if appropriate.
	*/
	pe__clear_action_flags(then, pe_action_runnable);
	}

	for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;
	pe_action_t *first = other->action;

	pe_node_t *then_node = then->node;
	pe_node_t *first_node = first->node;

	if ((first->rsc != NULL)
	&& (first->rsc->variant == pe_group)
	&& pcmk__str_eq(first->task, RSC_START, pcmk__str_casei)) {

	first_node = first->rsc->fns->location(first->rsc, NULL, FALSE);
	if (first_node != NULL) {
	pe_rsc_trace(first->rsc, "Found %s for 'first' %s",
	pe__node_name(first_node), first->uuid);
	}
	}

	if ((then->rsc != NULL)
	&& (then->rsc->variant == pe_group)
	&& pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)) {

	then_node = then->rsc->fns->location(then->rsc, NULL, FALSE);
	if (then_node != NULL) {
	pe_rsc_trace(then->rsc, "Found %s for 'then' %s",
	pe__node_name(then_node), then->uuid);
	}
	}

	// Disable constraint if it only applies when on same node, but isn't
	if (pcmk_is_set(other->type, pe_order_same_node)
	&& (first_node != NULL) && (then_node != NULL)
	&& (first_node->details != then_node->details)) {

	pe_rsc_trace(then->rsc,
	"Disabled ordering %s on %s then %s on %s: not same node",
	other->action->uuid, pe__node_name(first_node),
	then->uuid, pe__node_name(then_node));
	other->type = pe_order_none;
	continue;
	}

	pcmk__clear_updated_flags(changed, then, pcmk__updated_first);

	if ((first->rsc != NULL)
	&& pcmk_is_set(other->type, pe_order_then_cancels_first)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {

	/* 'then' is required, so we must abandon 'first'
	* (e.g. a required stop cancels any agent reload).
	*/
	pe__set_action_flags(other->action, pe_action_optional);
	if (!strcmp(first->task, CRMD_ACTION_RELOAD_AGENT)) {
	pe__clear_resource_flags(first->rsc, pe_rsc_reload);
	}
	}

	if ((first->rsc != NULL) && (then->rsc != NULL)
	&& (first->rsc != then->rsc) && !is_parent(then->rsc, first->rsc)) {
	first = action_for_ordering(first);
	}
	if (first != other->action) {
	pe_rsc_trace(then->rsc, "Ordering %s after %s instead of %s",
	then->uuid, first->uuid, other->action->uuid);
	}

	pe_rsc_trace(then->rsc,
	"%s (%#.6x) then %s (%#.6x): type=%#.6x node=%s",
	first->uuid, first->flags, then->uuid, then->flags,
	other->type, action_node_str(first));

	if (first == other->action) {
	/* 'first' was not remapped (e.g. from 'start' to 'running'), which
	* could mean it is a non-resource action, a primitive resource
	* action, or already expanded.
	*/
	enum pe_action_flags first_flags, then_flags;

	first_flags = action_flags_for_ordering(first, then_node);
	then_flags = action_flags_for_ordering(then, first_node);

	changed \|= update_action_for_ordering_flags(first, then,
	first_flags, then_flags,
	other, data_set);

	/* 'first' was for a complex resource (clone, group, etc),
	* create a new dependency if necessary
	*/
	} else if (order_actions(first, then, other->type)) {
	/* This was the first time 'first' and 'then' were associated,
	* start again to get the new actions_before list
	*/
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	pe_rsc_trace(then->rsc,
	"Disabled ordering %s then %s in favor of %s then %s",
	other->action->uuid, then->uuid, first->uuid,
	then->uuid);
	other->type = pe_order_none;
	}


	if (pcmk_is_set(changed, pcmk__updated_first)) {
	crm_trace("Re-processing %s and its 'after' actions "
	"because it changed", first->uuid);
	for (GList *lpc2 = first->actions_after; lpc2 != NULL;
	lpc2 = lpc2->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc2->data;

	pcmk__update_action_for_orderings(other->action, data_set);
	}
	pcmk__update_action_for_orderings(first, data_set);
	}
	}

	if (pcmk_is_set(then->flags, pe_action_requires_any)) {
	if (last_flags == then->flags) {
	pcmk__clear_updated_flags(changed, then, pcmk__updated_then);
	} else {
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	}
	}

	if (pcmk_is_set(changed, pcmk__updated_then)) {
	crm_trace("Re-processing %s and its 'after' actions because it changed",
	then->uuid);
	if (pcmk_is_set(last_flags, pe_action_runnable)
	&& !pcmk_is_set(then->flags, pe_action_runnable)) {
	pcmk__block_colocation_dependents(then, data_set);
	}
	pcmk__update_action_for_orderings(then, data_set);
	for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;

	pcmk__update_action_for_orderings(other->action, data_set);
	}
	}
	}

	static inline bool
	is_primitive_action(pe_action_t *action)
	{
	return action && action->rsc && (action->rsc->variant == pe_native);
	}

	/*!
	* \internal
	* \brief Clear a single action flag and set reason text
	*
	* \param[in] action Action whose flag should be cleared
	* \param[in] flag Action flag that should be cleared
	* \param[in] reason Action that is the reason why flag is being cleared
	*/
	#define clear_action_flag_because(action, flag, reason) do { \
	if (pcmk_is_set((action)->flags, (flag))) { \
	pe__clear_action_flags(action, flag); \
	if ((action)->rsc != (reason)->rsc) { \
	char *reason_text = pe__action2reason((reason), (flag)); \
	pe_action_set_reason((action), reason_text, \
	((flag) == pe_action_migrate_runnable)); \
	free(reason_text); \
	} \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Update actions in an asymmetric ordering
	*
	* \param[in] first 'First' action in an asymmetric ordering
	* \param[in] then 'Then' action in an asymmetric ordering
	*/
	static void
	handle_asymmetric_ordering(pe_action_t first, pe_action_t then)
	{
	enum rsc_role_e then_rsc_role = RSC_ROLE_UNKNOWN;
	GList *then_on = NULL;

	if (then->rsc == NULL) {
	// Asymmetric orderings only matter if there's a resource involved
	return;
	}

	then_rsc_role = then->rsc->fns->state(then->rsc, TRUE);
	then_on = then->rsc->running_on;

	if ((then_rsc_role == RSC_ROLE_STOPPED)
	&& pcmk__str_eq(then->task, RSC_STOP, pcmk__str_none)) {
	/* Nothing needs to be done for asymmetric ordering if 'then' is
	* supposed to be stopped after 'first' but is already stopped.
	*/
	return;
	}

	if ((then_rsc_role >= RSC_ROLE_STARTED)
	&& pcmk_is_set(then->flags, pe_action_optional)
	&& (then->node != NULL)
	&& pcmk__list_of_1(then_on)
	&& (then->node->details == ((pe_node_t *) then_on->data)->details)
	&& pcmk__str_eq(then->task, RSC_START, pcmk__str_none)) {
	/* Nothing needs to be done for asymmetric ordering if 'then' is
	* supposed to be started after 'first' but is already started --
	* unless the start is mandatory, which indicates the resource is
	* restarting and the ordering is still needed.
	*/
	return;
	}

	if (!pcmk_is_set(first->flags, pe_action_runnable)) {
	// 'First' can't run, so 'then' can't either
	clear_action_flag_because(then, pe_action_optional, first);
	clear_action_flag_because(then, pe_action_runnable, first);
	}
	}

	/*!
	* \internal
	* \brief Set action bits appropriately when pe_restart_order is used
	*
	* \param[in] first 'First' action in an ordering with pe_restart_order
	* \param[in] then 'Then' action in an ordering with pe_restart_order
	* \param[in] filter What action flags to care about
	*
	* \note pe_restart_order is set for "stop resource before starting it" and
	* "stop later group member before stopping earlier group member"
	*/
	static void
	handle_restart_ordering(pe_action_t first, pe_action_t then, uint32_t filter)
	{
	const char *reason = NULL;

	CRM_ASSERT(is_primitive_action(first));
	CRM_ASSERT(is_primitive_action(then));

	// We need to update the action in two cases:

	// ... if 'then' is required
	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {
	reason = "restart";
	}

	/* ... if 'then' is unrunnable action on same resource (if a resource
	* should restart but can't start, we still want to stop)
	*/
	if (pcmk_is_set(filter, pe_action_runnable)
	&& !pcmk_is_set(then->flags, pe_action_runnable)
	&& pcmk_is_set(then->rsc->flags, pe_rsc_managed)
	&& (first->rsc == then->rsc)) {
	reason = "stop";
	}

	if (reason == NULL) {
	return;
	}

	pe_rsc_trace(first->rsc, "Handling %s -> %s for %s",
	first->uuid, then->uuid, reason);

	// Make 'first' required if it is runnable
	if (pcmk_is_set(first->flags, pe_action_runnable)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}

	// Make 'first' required if 'then' is required
	if (!pcmk_is_set(then->flags, pe_action_optional)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}

	// Make 'first' unmigratable if 'then' is unmigratable
	if (!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	clear_action_flag_because(first, pe_action_migrate_runnable, then);
	}

	// Make 'then' unrunnable if 'first' is required but unrunnable
	if (!pcmk_is_set(first->flags, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_runnable)) {
	clear_action_flag_because(then, pe_action_runnable, first);
	}
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions'
	* flags (and runnable_before members if appropriate) as appropriate for the
	* ordering. In some cases, the ordering could be disabled as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (ignored)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pe_action_optional to affect only mandatory
	* actions, and pe_action_runnable to affect only
	* runnable actions)
	* \param[in] type Group of enum pe_ordering flags to apply
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__update_ordered_actions(pe_action_t first, pe_action_t then,
	const pe_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pe_working_set_t *data_set)
	{
	uint32_t changed = pcmk__updated_none;
	uint32_t then_flags = then->flags;
	uint32_t first_flags = first->flags;

	if (pcmk_is_set(type, pe_order_asymmetrical)) {
	handle_asymmetric_ordering(first, then);
	}

	if (pcmk_is_set(type, pe_order_implies_first)
	&& !pcmk_is_set(then_flags, pe_action_optional)) {
	// Then is required, and implies first should be, too

	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(flags, pe_action_optional)
	&& pcmk_is_set(first_flags, pe_action_optional)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}

	if (pcmk_is_set(flags, pe_action_migrate_runnable)
	&& !pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	clear_action_flag_because(first, pe_action_migrate_runnable, then);
	}
	}

	if (pcmk_is_set(type, pe_order_promoted_implies_first)
	&& (then->rsc != NULL) && (then->rsc->role == RSC_ROLE_PROMOTED)
	&& pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {

	clear_action_flag_because(first, pe_action_optional, then);

	if (pcmk_is_set(first->flags, pe_action_migrate_runnable)
	&& !pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	clear_action_flag_because(first, pe_action_migrate_runnable,
	then);
	}
	}

	if (pcmk_is_set(type, pe_order_implies_first_migratable)
	&& pcmk_is_set(filter, pe_action_optional)) {

	if (!pcmk_all_flags_set(then->flags,
	pe_action_migrate_runnable\|pe_action_runnable)) {
	clear_action_flag_because(first, pe_action_runnable, then);
	}

	if (!pcmk_is_set(then->flags, pe_action_optional)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}
	}

	if (pcmk_is_set(type, pe_order_pseudo_left)
	&& pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_runnable)) {

	clear_action_flag_because(then, pe_action_migrate_runnable, first);
	pe__clear_action_flags(then, pe_action_pseudo);
	}

	if (pcmk_is_set(type, pe_order_runnable_left)
	&& pcmk_is_set(filter, pe_action_runnable)
	&& pcmk_is_set(then->flags, pe_action_runnable)
	&& !pcmk_is_set(flags, pe_action_runnable)) {

	clear_action_flag_because(then, pe_action_runnable, first);
	clear_action_flag_because(then, pe_action_migrate_runnable, first);
	}

	if (pcmk_is_set(type, pe_order_implies_then)
	&& pcmk_is_set(filter, pe_action_optional)
	&& pcmk_is_set(then->flags, pe_action_optional)
	&& !pcmk_is_set(flags, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_migrate_runnable)) {

	clear_action_flag_because(then, pe_action_optional, first);
	}

	if (pcmk_is_set(type, pe_order_restart)) {
	handle_restart_ordering(first, then, filter);
	}

	if (then_flags != then->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	pe_rsc_trace(then->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'first' %s (%#.6x)",
	then->uuid, pe__node_name(then->node),
	then->flags, then_flags, first->uuid, first->flags);

	if ((then->rsc != NULL) && (then->rsc->parent != NULL)) {
	// Required to handle "X_stop then X_start" for cloned groups
	pcmk__update_action_for_orderings(then, data_set);
	}
	}

	if (first_flags != first->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	pe_rsc_trace(first->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'then' %s (%#.6x)",
	first->uuid, pe__node_name(first->node),
	first->flags, first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	/*!
	* \internal
	* \brief Trace-log an action (optionally with its dependent actions)
	*
	* \param[in] pre_text If not NULL, prefix the log with this plus ": "
	* \param[in] action Action to log
	* \param[in] details If true, recursively log dependent actions
	*/
	void
	pcmk__log_action(const char pre_text, pe_action_t action, bool details)
	{
	const char *node_uname = NULL;
	const char *node_uuid = NULL;
	const char *desc = NULL;

	CRM_CHECK(action != NULL, return);

	if (!pcmk_is_set(action->flags, pe_action_pseudo)) {
	if (action->node != NULL) {
	node_uname = action->node->details->uname;
	node_uuid = action->node->details->id;
	} else {
	node_uname = "<none>";
	}
	}

	switch (text2task(action->task)) {
	case stonith_node:
	case shutdown_crm:
	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	desc = "Pseudo ";
	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	desc = "Optional ";
	} else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else if (pcmk_is_set(action->flags, pe_action_processed)) {
	desc = "";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	default:
	if (pcmk_is_set(action->flags, pe_action_optional)) {
	desc = "Optional ";
	} else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	desc = "Pseudo ";
	} else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else if (pcmk_is_set(action->flags, pe_action_processed)) {
	desc = "";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(action->rsc? action->rsc->id : "<none>"),
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	}

	if (details) {
	GList *iter = NULL;

	crm_trace("\t\t====== Preceding Actions");
	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) iter->data;

	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== Subsequent Actions");
	for (iter = action->actions_after; iter != NULL; iter = iter->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) iter->data;

	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== End");

	} else {
	crm_trace("\t\t(before=%d, after=%d)",
	g_list_length(action->actions_before),
	g_list_length(action->actions_after));
	}
	}

	/*!
	* \internal
	* \brief Create a new shutdown action for a node
	*
	* \param[in] node Node being shut down
	*
	* \return Newly created shutdown action for \p node
	*/
	pe_action_t *
	pcmk__new_shutdown_action(pe_node_t *node)
	{
	char *shutdown_id = NULL;
	pe_action_t *shutdown_op = NULL;

	CRM_ASSERT(node != NULL);

	shutdown_id = crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN,
	node->details->uname);

	shutdown_op = custom_action(NULL, shutdown_id, CRM_OP_SHUTDOWN, node, FALSE,
	TRUE, node->details->data_set);

	pcmk__order_stops_before_shutdown(node, shutdown_op);
	add_hash_param(shutdown_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	return shutdown_op;
	}

	/*!
	* \internal
	* \brief Calculate and add an operation digest to XML
	*
	* Calculate an operation digest, which enables us to later determine when a
	* restart is needed due to the resource's parameters being changed, and add it
	* to given XML.
	*
	* \param[in] op Operation result from executor
	* \param[in] update XML to add digest to
	*/
	static void
	add_op_digest_to_xml(lrmd_event_data_t op, xmlNode update)
	{
	char *digest = NULL;
	xmlNode *args_xml = NULL;

	if (op->params == NULL) {
	return;
	}
	args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
	g_hash_table_foreach(op->params, hash2field, args_xml);
	pcmk__filter_op_for_digest(args_xml);
	digest = calculate_operation_digest(args_xml, NULL);
	crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);
	free_xml(args_xml);
	free(digest);
	}

	#define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"

	/*!
	* \internal
	* \brief Create XML for resource operation history update
	*
	* \param[in,out] parent Parent XML node to add to
	* \param[in,out] op Operation event data
	* \param[in] caller_version DC feature set
	* \param[in] target_rc Expected result of operation
	* \param[in] node Name of node on which operation was performed
	* \param[in] origin Arbitrary description of update source
	*
	* \return Newly created XML node for history update
	*/
	xmlNode *
	pcmk__create_history_xml(xmlNode parent, lrmd_event_data_t op,
	const char *caller_version, int target_rc,
	const char node, const char origin)
	{
	char *key = NULL;
	char *magic = NULL;
	char *op_id = NULL;
	char *op_id_additional = NULL;
	char *local_user_data = NULL;
	const char *exit_reason = NULL;

	xmlNode *xml_op = NULL;
	const char *task = NULL;

	CRM_CHECK(op != NULL, return NULL);
	crm_trace("Creating history XML for %s-interval %s action for %s on %s "
	"(DC version: %s, origin: %s)",
	pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
	((node == NULL)? "no node" : node), caller_version, origin);

	task = op->op_type;

	/* Record a successful agent reload as a start, and a failed one as a
	* monitor, to make life easier for the scheduler when determining the
	* current state.
	*
	* @COMPAT We should check "reload" here only if the operation was for a
	* pre-OCF-1.1 resource agent, but we don't know that here, and we should
	* only ever get results for actions scheduled by us, so we can reasonably
	* assume any "reload" is actually a pre-1.1 agent reload.
	*/
	if (pcmk__str_any_of(task, CRMD_ACTION_RELOAD, CRMD_ACTION_RELOAD_AGENT,
	NULL)) {
	if (op->op_status == PCMK_EXEC_DONE) {
	task = CRMD_ACTION_START;
	} else {
	task = CRMD_ACTION_STATUS;
	}
	}

	key = pcmk__op_key(op->rsc_id, task, op->interval_ms);
	if (pcmk__str_eq(task, CRMD_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = crm_meta_value(op->params, "notify_type");
	const char *n_task = crm_meta_value(op->params, "notify_operation");

	CRM_LOG_ASSERT(n_type != NULL);
	CRM_LOG_ASSERT(n_task != NULL);
	op_id = pcmk__notify_key(op->rsc_id, n_type, n_task);

	if (op->op_status != PCMK_EXEC_PENDING) {
	/* Ignore notify errors.
	*
	* @TODO It might be better to keep the correct result here, and
	* ignore it in process_graph_event().
	*/
	lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	}

	/* Migration history is preserved separately, which usually matters for
	* multiple nodes and is important for future cluster transitions.
	*/
	} else if (pcmk__str_any_of(op->op_type, CRMD_ACTION_MIGRATE,
	CRMD_ACTION_MIGRATED, NULL)) {
	op_id = strdup(key);

	} else if (did_rsc_op_fail(op, target_rc)) {
	op_id = pcmk__op_key(op->rsc_id, "last_failure", 0);
	if (op->interval_ms == 0) {
	// Ensure 'last' gets updated, in case record-pending is true
	op_id_additional = pcmk__op_key(op->rsc_id, "last", 0);

	} else {
	// Ensure any pending recurring monitor gets updated if it fails
	op_id_additional = strdup(key);
	}
	exit_reason = op->exit_reason;

	} else if (op->interval_ms > 0) {
	op_id = strdup(key);

	} else {
	op_id = pcmk__op_key(op->rsc_id, "last", 0);
	}

	again:
	xml_op = pcmk__xe_match(parent, XML_LRM_TAG_RSC_OP, XML_ATTR_ID, op_id);
	if (xml_op == NULL) {
	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
	}

	if (op->user_data == NULL) {
	crm_debug("Generating fake transition key for: " PCMK__OP_FMT
	" %d from %s", op->rsc_id, op->op_type, op->interval_ms,
	op->call_id, origin);
	local_user_data = pcmk__transition_key(-1, op->call_id, target_rc,
	FAKE_TE_ID);
	op->user_data = local_user_data;
	}

	if (magic == NULL) {
	magic = crm_strdup_printf("%d:%d;%s", op->op_status, op->rc,
	(const char *) op->user_data);
	}

	crm_xml_add(xml_op, XML_ATTR_ID, op_id);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
	crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
	crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
	crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, exit_reason == NULL ? "" : exit_reason);
	crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); /* For context during triage */

	crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
	crm_xml_add_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, op->interval_ms);

	if (compare_version("2.1", caller_version) <= 0) {
	if (op->t_run \|\| op->t_rcchange \|\| op->exec_time \|\| op->queue_time) {
	crm_trace("Timing data (" PCMK__OP_FMT
	"): last=%u change=%u exec=%u queue=%u",
	op->rsc_id, op->op_type, op->interval_ms,
	op->t_run, op->t_rcchange, op->exec_time, op->queue_time);

	if ((op->interval_ms != 0) && (op->t_rcchange != 0)) {
	// Recurring ops may have changed rc after initial run
	crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
	(long long) op->t_rcchange);
	} else {
	crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
	(long long) op->t_run);
	}

	crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
	crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
	}
	}

	if (pcmk__str_any_of(op->op_type, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
	/*
	* Record migrate_source and migrate_target always for migrate ops.
	*/
	const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;

	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));

	name = XML_LRM_ATTR_MIGRATE_TARGET;
	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
	}

	add_op_digest_to_xml(op, xml_op);

	if (op_id_additional) {
	free(op_id);
	op_id = op_id_additional;
	op_id_additional = NULL;
	goto again;
	}

	if (local_user_data) {
	free(local_user_data);
	op->user_data = NULL;
	}
	free(magic);
	free(op_id);
	free(key);
	return xml_op;
	}

	/*!
	* \internal
	* \brief Check whether an action shutdown-locks a resource to a node
	*
	* If the shutdown-lock cluster property is set, resources will not be recovered
	* on a different node if cleanly stopped, and may start only on that same node.
	* This function checks whether that applies to a given action, so that the
	* transition graph can be marked appropriately.
	*
	* \param[in] action Action to check
	*
	* \return true if \p action locks its resource to the action's node,
	* otherwise false
	*/
	bool
	pcmk__action_locks_rsc_to_node(const pe_action_t *action)
	{
	// Only resource actions taking place on resource's lock node are locked
	if ((action == NULL) \|\| (action->rsc == NULL)
	\|\| (action->rsc->lock_node == NULL) \|\| (action->node == NULL)
	\|\| (action->node->details != action->rsc->lock_node->details)) {
	return false;
	}

	/* During shutdown, only stops are locked (otherwise, another action such as
	* a demote would cause the controller to clear the lock)
	*/
	if (action->node->details->shutdown && (action->task != NULL)
	&& (strcmp(action->task, RSC_STOP) != 0)) {
	return false;
	}

	return true;
	}

	/* lowest to highest */
	static gint
	sort_action_id(gconstpointer a, gconstpointer b)
	{
	const pe_action_wrapper_t action_wrapper2 = (const pe_action_wrapper_t )a;
	const pe_action_wrapper_t action_wrapper1 = (const pe_action_wrapper_t )b;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}
	if (action_wrapper1->action->id < action_wrapper2->action->id) {
	return 1;
	}
	if (action_wrapper1->action->id > action_wrapper2->action->id) {
	return -1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Remove any duplicate action inputs, merging action flags
	*
	* \param[in] action Action whose inputs should be checked
	*/
	void
	pcmk__deduplicate_action_inputs(pe_action_t *action)
	{
	GList *item = NULL;
	GList *next = NULL;
	pe_action_wrapper_t *last_input = NULL;

	action->actions_before = g_list_sort(action->actions_before,
	sort_action_id);
	for (item = action->actions_before; item != NULL; item = next) {
	pe_action_wrapper_t input = (pe_action_wrapper_t ) item->data;

	next = item->next;
	if ((last_input != NULL)
	&& (input->action->id == last_input->action->id)) {
	crm_trace("Input %s (%d) duplicate skipped for action %s (%d)",
	input->action->uuid, input->action->id,
	action->uuid, action->id);

	/* For the purposes of scheduling, the ordering flags no longer
	* matter, but crm_simulate looks at certain ones when creating a
	* dot graph. Combining the flags is sufficient for that purpose.
	*/
	last_input->type \|= input->type;
	if (input->state == pe_link_dumped) {
	last_input->state = pe_link_dumped;
	}

	free(item->data);
	action->actions_before = g_list_delete_link(action->actions_before,
	item);
	} else {
	last_input = input;
	input->state = pe_link_not_dumped;
	}
	}
	}

	/*!
	* \internal
	* \brief Output all scheduled actions
	*
	* \param[in] data_set Cluster working set
	*/
	void
	pcmk__output_actions(pe_working_set_t *data_set)
	{
	pcmk__output_t *out = data_set->priv;

	// Output node (non-resource) actions
	for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) {
	char *node_name = NULL;
	char *task = NULL;
	pe_action_t action = (pe_action_t ) iter->data;

	if (action->rsc != NULL) {
	continue; // Resource actions will be output later

	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	continue; // This action was not scheduled
	}

	if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	task = strdup("Shutdown");

	} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
	const char *op = g_hash_table_lookup(action->meta, "stonith_action");

	task = crm_strdup_printf("Fence (%s)", op);

	} else {
	continue; // Don't display other node action types
	}

	if (pe__is_guest_node(action->node)) {
	node_name = crm_strdup_printf("%s (resource: %s)",
	pe__node_name(action->node),
	action->node->details->remote_rsc->container->id);
	} else if (action->node != NULL) {
	node_name = crm_strdup_printf("%s", pe__node_name(action->node));
	}

	out->message(out, "node-action", task, node_name, action->reason);

	free(node_name);
	free(task);
	}

	// Output resource actions
	for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	rsc->cmds->output_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether action from resource history is still in configuration
	*
	* \param[in] rsc Resource that action is for
	* \param[in] task Action's name
	* \param[in] interval_ms Action's interval (in milliseconds)
	*
	* \return true if action is still in resource configuration, otherwise false
	*/
	static bool
	action_in_config(pe_resource_t rsc, const char task, guint interval_ms)
	{
	char *key = pcmk__op_key(rsc->id, task, interval_ms);
	bool config = (find_rsc_op_entry(rsc, key) != NULL);

	free(key);
	return config;
	}

	/*!
	* \internal
	* \brief Get action name needed to compare digest for configuration changes
	*
	* \param[in] task Action name from history
	* \param[in] interval_ms Action interval (in milliseconds)
	*
	* \return Action name whose digest should be compared
	*/
	static const char *
	task_for_digest(const char *task, guint interval_ms)
	{
	/* Certain actions need to be compared against the parameters used to start
	* the resource.
	*/
	if ((interval_ms == 0)
	&& pcmk__str_any_of(task, RSC_STATUS, RSC_MIGRATED, RSC_PROMOTE, NULL)) {
	task = RSC_START;
	}
	return task;
	}

	/*!
	* \internal
	* \brief Check whether only sanitized parameters to an action changed
	*
	* When collecting CIB files for troubleshooting, crm_report will mask
	* sensitive resource parameters. If simulations were run using that, affected
	* resources would appear to need a restart, which would complicate
	* troubleshooting. To avoid that, we save a "secure digest" of non-sensitive
	* parameters. This function used that digest to check whether only masked
	* parameters are different.
	*
	* \param[in] xml_op Resource history entry with secure digest
	* \param[in] digest_data Operation digest information being compared
	* \param[in] data_set Cluster working set
	*
	* \return true if only sanitized parameters changed, otherwise false
	*/
	static bool
	-only_sanitized_changed(xmlNode xml_op, const op_digest_cache_t digest_data,
	- pe_working_set_t *data_set)
	+only_sanitized_changed(const xmlNode *xml_op,
	+ const op_digest_cache_t *digest_data,
	+ const pe_working_set_t *data_set)
	{
	const char *digest_secure = NULL;

	if (!pcmk_is_set(data_set->flags, pe_flag_sanitized)) {
	// The scheduler is not being run as a simulation
	return false;
	}

	digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);

	return (digest_data->rc != RSC_DIGEST_MATCH) && (digest_secure != NULL)
	&& (digest_data->digest_secure_calc != NULL)
	&& (strcmp(digest_data->digest_secure_calc, digest_secure) == 0);
	}

	/*!
	* \internal
	* \brief Force a restart due to a configuration change
	*
	* \param[in] rsc Resource that action is for
	* \param[in] task Name of action whose configuration changed
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] node Node where resource should be restarted
	*/
	static void
	force_restart(pe_resource_t rsc, const char task, guint interval_ms,
	- pe_node_t *node)
	+ const pe_node_t *node)
	{
	char *key = pcmk__op_key(rsc->id, task, interval_ms);
	pe_action_t *required = custom_action(rsc, key, task, NULL, FALSE, TRUE,
	rsc->cluster);

	pe_action_set_reason(required, "resource definition change", true);
	trigger_unfencing(rsc, node, "Device parameters changed", NULL,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Schedule a reload of a resource on a node
	*
	* \param[in] rsc Resource to reload
	* \param[in] node Where resource should be reloaded
	*/
	static void
	-schedule_reload(pe_resource_t rsc, pe_node_t node)
	+schedule_reload(pe_resource_t rsc, const pe_node_t node)
	{
	pe_action_t *reload = NULL;

	// For collective resources, just call recursively for children
	if (rsc->variant > pe_native) {
	- g_list_foreach(rsc->children, (GFunc) schedule_reload, node);
	+ g_list_foreach(rsc->children, (GFunc) schedule_reload, (gpointer) node);
	return;
	}

	// Skip the reload in certain situations
	if ((node == NULL)
	\|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)
	\|\| pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_trace(rsc, "Skip reload of %s:%s%s %s",
	rsc->id,
	pcmk_is_set(rsc->flags, pe_rsc_managed)? "" : " unmanaged",
	pcmk_is_set(rsc->flags, pe_rsc_failed)? " failed" : "",
	(node == NULL)? "inactive" : node->details->uname);
	return;
	}

	/* If a resource's configuration changed while a start was pending,
	* force a full restart instead of a reload.
	*/
	if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
	pe_rsc_trace(rsc, "%s: preventing agent reload because start pending",
	rsc->id);
	custom_action(rsc, stop_key(rsc), CRMD_ACTION_STOP, node, FALSE, TRUE,
	rsc->cluster);
	return;
	}

	// Schedule the reload
	pe__set_resource_flags(rsc, pe_rsc_reload);
	reload = custom_action(rsc, reload_key(rsc), CRMD_ACTION_RELOAD_AGENT, node,
	FALSE, TRUE, rsc->cluster);
	pe_action_set_reason(reload, "resource definition change", FALSE);

	// Set orderings so that a required stop or demote cancels the reload
	pcmk__new_ordering(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
	pe_order_optional\|pe_order_then_cancels_first,
	rsc->cluster);
	pcmk__new_ordering(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
	pe_order_optional\|pe_order_then_cancels_first,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Handle any configuration change for an action
	*
	* Given an action from resource history, if the resource's configuration
	* changed since the action was done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, etc.).
	*
	- * \param[in] rsc Resource that action is for
	- * \param[in] node Node that action was on
	- * \param[in] xml_op Action XML from resource history
	+ * \param[in,out] rsc Resource that action is for
	+ * \param[in] node Node that action was on
	+ * \param[in] xml_op Action XML from resource history
	*
	* \return true if action configuration changed, otherwise false
	*/
	bool
	-pcmk__check_action_config(pe_resource_t rsc, pe_node_t node, xmlNode *xml_op)
	+pcmk__check_action_config(pe_resource_t rsc, const pe_node_t node,
	+ const xmlNode *xml_op)
	{
	guint interval_ms = 0;
	const char *task = NULL;
	const op_digest_cache_t *digest_data = NULL;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (xml_op != NULL),
	return false);

	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	CRM_CHECK(task != NULL, return false);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	// If this is a recurring action, check whether it has been orphaned
	if (interval_ms > 0) {
	if (action_in_config(rsc, task, interval_ms)) {
	pe_rsc_trace(rsc, "%s-interval %s for %s on %s is in configuration",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node));
	} else if (pcmk_is_set(rsc->cluster->flags,
	pe_flag_stop_action_orphans)) {
	pcmk__schedule_cancel(rsc,
	crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	task, interval_ms, node, "orphan");
	return true;
	} else {
	pe_rsc_debug(rsc, "%s-interval %s for %s on %s is orphaned",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node));
	return true;
	}
	}

	crm_trace("Checking %s-interval %s for %s on %s for configuration changes",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node));
	task = task_for_digest(task, interval_ms);
	digest_data = rsc_action_digest_cmp(rsc, xml_op, node, rsc->cluster);

	if (only_sanitized_changed(xml_op, digest_data, rsc->cluster)) {
	if (!pcmk__is_daemon && (rsc->cluster->priv != NULL)) {
	pcmk__output_t *out = rsc->cluster->priv;

	out->info(out,
	"Only 'private' parameters to %s-interval %s for %s "
	"on %s changed: %s",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node),
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	}
	return false;
	}

	switch (digest_data->rc) {
	case RSC_DIGEST_RESTART:
	crm_log_xml_debug(digest_data->params_restart, "params:restart");
	force_restart(rsc, task, interval_ms, node);
	return true;

	case RSC_DIGEST_ALL:
	case RSC_DIGEST_UNKNOWN:
	// Changes that can potentially be handled by an agent reload

	if (interval_ms > 0) {
	/* Recurring actions aren't reloaded per se, they are just
	* re-scheduled so the next run uses the new parameters.
	* The old instance will be cancelled automatically.
	*/
	crm_log_xml_debug(digest_data->params_all, "params:reschedule");
	pcmk__reschedule_recurring(rsc, task, interval_ms, node);

	} else if (crm_element_value(xml_op,
	XML_LRM_ATTR_RESTART_DIGEST) != NULL) {
	// Agent supports reload, so use it
	trigger_unfencing(rsc, node,
	"Device parameters changed (reload)", NULL,
	rsc->cluster);
	crm_log_xml_debug(digest_data->params_all, "params:reload");
	schedule_reload(rsc, node);

	} else {
	pe_rsc_trace(rsc,
	"Restarting %s because agent doesn't support reload",
	rsc->id);
	crm_log_xml_debug(digest_data->params_restart,
	"params:restart");
	force_restart(rsc, task, interval_ms, node);
	}
	return true;

	default:
	break;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Create a list of resource's action history entries, sorted by call ID
	*
	* \param[in] rsc Resource whose history should be checked
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[out] start_index Where to store index of start-like action, if any
	* \param[out] stop_index Where to store index of stop action, if any
	*/
	static GList *
	rsc_history_as_list(pe_resource_t rsc, xmlNode rsc_entry,
	int start_index, int stop_index)
	{
	GList *ops = NULL;

	for (xmlNode *rsc_op = first_named_child(rsc_entry, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {
	ops = g_list_prepend(ops, rsc_op);
	}
	ops = g_list_sort(ops, sort_op_by_callid);
	calculate_active_ops(ops, start_index, stop_index);
	return ops;
	}

	/*!
	* \internal
	* \brief Process a resource's action history from the CIB status
	*
	* Given a resource's action history, if the resource's configuration
	* changed since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[in] rsc Resource whose history is being processed
	* \param[in] node Node whose history is being processed
	*/
	static void
	process_rsc_history(xmlNode rsc_entry, pe_resource_t rsc, pe_node_t *node)
	{
	int offset = -1;
	int stop_index = 0;
	int start_index = 0;
	GList *sorted_op_list = NULL;

	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	if (pe_rsc_is_anon_clone(uber_parent(rsc))) {
	pe_rsc_trace(rsc,
	"Skipping configuration check "
	"for orphaned clone instance %s",
	rsc->id);
	} else {
	pe_rsc_trace(rsc,
	"Skipping configuration check and scheduling clean-up "
	"for orphaned resource %s", rsc->id);
	pcmk__schedule_cleanup(rsc, node, false);
	}
	return;
	}

	if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, false)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}
	pe_rsc_trace(rsc,
	"Skipping configuration check for %s "
	"because no longer active on %s",
	rsc->id, pe__node_name(node));
	return;
	}

	pe_rsc_trace(rsc, "Checking for configuration changes for %s on %s",
	rsc->id, pe__node_name(node));

	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, true)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}

	sorted_op_list = rsc_history_as_list(rsc, rsc_entry, &start_index,
	&stop_index);
	if (start_index < stop_index) {
	return; // Resource is stopped
	}

	for (GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	xmlNode rsc_op = (xmlNode ) iter->data;
	const char *task = NULL;
	guint interval_ms = 0;

	if (++offset < start_index) {
	// Skip actions that happened before a start
	continue;
	}

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	if ((interval_ms > 0)
	&& (pcmk_is_set(rsc->flags, pe_rsc_maintenance)
	\|\| node->details->maintenance)) {
	// Maintenance mode cancels recurring operations
	pcmk__schedule_cancel(rsc,
	crm_element_value(rsc_op, XML_LRM_ATTR_CALLID),
	task, interval_ms, node, "maintenance mode");

	} else if ((interval_ms > 0)
	\|\| pcmk__strcase_any_of(task, RSC_STATUS, RSC_START,
	RSC_PROMOTE, RSC_MIGRATED, NULL)) {
	/* If a resource operation failed, and the operation's definition
	* has changed, clear any fail count so they can be retried fresh.
	*/

	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't allocated resources to nodes yet, so if the
	* REMOTE_CONTAINER_HACK is used, we may calculate the digest
	* based on the literal "#uname" value rather than the properly
	* substituted value. That would mistakenly make the action
	* definition appear to have been changed. Defer the check until
	* later in this case.
	*/
	pe__add_param_check(rsc_op, rsc, node, pe_check_active,
	rsc->cluster);

	} else if (pcmk__check_action_config(rsc, node, rsc_op)
	&& (pe_get_failcount(node, rsc, NULL, pe_fc_effective,
	NULL) != 0)) {
	pe__clear_failcount(rsc, node, "action definition changed",
	rsc->cluster);
	}
	}
	}
	g_list_free(sorted_op_list);
	}

	/*!
	* \internal
	* \brief Process a node's action history from the CIB status
	*
	* Given a node's resource history, if the resource's configuration changed
	* since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in] node Node whose history is being processed
	* \param[in] lrm_rscs Node's <lrm_resources> from CIB status XML
	* \param[in] data_set Cluster working set
	*/
	static void
	process_node_history(pe_node_t node, xmlNode lrm_rscs, pe_working_set_t *data_set)
	{
	crm_trace("Processing node history for %s", pe__node_name(node));
	for (xmlNode *rsc_entry = first_named_child(lrm_rscs, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	if (xml_has_children(rsc_entry)) {
	GList *result = pcmk__rscs_matching_id(ID(rsc_entry), data_set);

	for (GList *iter = result; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	if (rsc->variant == pe_native) {
	process_rsc_history(rsc_entry, rsc, node);
	}
	}
	g_list_free(result);
	}
	}
	}

	// XPath to find a node's resource history
	#define XPATH_NODE_HISTORY "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
	"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES

	/*!
	* \internal
	* \brief Process any resource configuration changes in the CIB status
	*
	* Go through all nodes' resource history, and if a resource's configuration
	* changed since its actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in] data_set Cluster working set
	*/
	void
	pcmk__handle_rsc_config_changes(pe_working_set_t *data_set)
	{
	crm_trace("Check resource and action configuration for changes");

	/* Rather than iterate through the status section, iterate through the nodes
	* and search for the appropriate status subsection for each. This skips
	* orphaned nodes and lets us eliminate some cases before searching the XML.
	*/
	for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;

	/* Don't bother checking actions for a node that can't run actions ...
	* unless it's in maintenance mode, in which case we still need to
	* cancel any existing recurring monitors.
	*/
	if (node->details->maintenance
	\|\| pcmk__node_available(node, false, false)) {

	char *xpath = NULL;
	xmlNode *history = NULL;

	xpath = crm_strdup_printf(XPATH_NODE_HISTORY, node->details->uname);
	history = get_xpath_object(xpath, data_set->input, LOG_NEVER);
	free(xpath);

	process_node_history(node, history, data_set);
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c
	index 634251a793..ad53ef92f3 100644
	--- a/lib/pacemaker/pcmk_sched_allocate.c
	+++ b/lib/pacemaker/pcmk_sched_allocate.c
	@@ -1,804 +1,804 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	CRM_TRACE_INIT_DATA(pacemaker);

	/*!
	* \internal
	* \brief Do deferred action checks after allocation
	*
	* When unpacking the resource history, the scheduler checks for resource
	* configurations that have changed since an action was run. However, at that
	* time, bundles using the REMOTE_CONTAINER_HACK don't have their final
	* parameter information, so instead they add a deferred check to a list. This
	* function processes one entry in that list.
	*
	- * \param[in] rsc Resource that action history is for
	- * \param[in] node Node that action history is for
	- * \param[in] rsc_op Action history entry
	- * \param[in] check Type of deferred check to do
	- * \param[in] data_set Working set for cluster
	+ * \param[in,out] rsc Resource that action history is for
	+ * \param[in] node Node that action history is for
	+ * \param[in] rsc_op Action history entry
	+ * \param[in] check Type of deferred check to do
	*/
	static void
	-check_params(pe_resource_t rsc, pe_node_t node, xmlNode *rsc_op,
	- enum pe_check_parameters check, pe_working_set_t *data_set)
	+check_params(pe_resource_t rsc, const pe_node_t node, const xmlNode *rsc_op,
	+ enum pe_check_parameters check)
	{
	const char *reason = NULL;
	op_digest_cache_t *digest_data = NULL;

	switch (check) {
	case pe_check_active:
	if (pcmk__check_action_config(rsc, node, rsc_op)
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL)) {
	reason = "action definition changed";
	}
	break;

	case pe_check_last_failure:
	- digest_data = rsc_action_digest_cmp(rsc, rsc_op, node, data_set);
	+ digest_data = rsc_action_digest_cmp(rsc, rsc_op, node,
	+ rsc->cluster);
	switch (digest_data->rc) {
	case RSC_DIGEST_UNKNOWN:
	crm_trace("Resource %s history entry %s on %s has "
	"no digest to compare",
	rsc->id, ID(rsc_op), node->details->id);
	break;
	case RSC_DIGEST_MATCH:
	break;
	default:
	reason = "resource parameters have changed";
	break;
	}
	break;
	}
	if (reason != NULL) {
	- pe__clear_failcount(rsc, node, reason, data_set);
	+ pe__clear_failcount(rsc, node, reason, rsc->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has failcount clearing scheduled on a node
	*
	* \param[in] node Node to check
	* \param[in] rsc Resource to check
	*
	* \return true if \p rsc has failcount clearing scheduled on \p node,
	* otherwise false
	*/
	static bool
	failcount_clear_action_exists(pe_node_t node, pe_resource_t rsc)
	{
	GList *list = pe__resource_actions(rsc, node, CRM_OP_CLEAR_FAILCOUNT, TRUE);

	if (list != NULL) {
	g_list_free(list);
	return true;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it reached its failure threshold there
	*
	* \param[in] rsc Resource to check failure threshold for
	* \param[in] node Node to check \p rsc on
	*/
	static void
	check_failure_threshold(pe_resource_t rsc, pe_node_t node)
	{
	// If this is a collective resource, apply recursively to children instead
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, (GFunc) check_failure_threshold,
	node);
	return;

	} else if (failcount_clear_action_exists(node, rsc)) {
	/* Don't force the resource away from this node due to a failcount
	* that's going to be cleared.
	*
	* @TODO Failcount clearing can be scheduled in
	* pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
	* schedule_resource_actions() via check_params(). This runs well before
	* then, so it cannot detect those, meaning we might check the migration
	* threshold when we shouldn't. Worst case, we stop or move the
	* resource, then move it back in the next transition.
	*/
	return;

	} else {
	pe_resource_t *failed = NULL;

	if (pcmk__threshold_reached(rsc, node, &failed)) {
	resource_location(failed, node, -INFINITY, "__fail_limit__",
	rsc->cluster);
	}
	}
	}

	/*!
	* \internal
	* \brief If resource has exclusive discovery, ban node if not allowed
	*
	* Location constraints have a resource-discovery option that allows users to
	* specify where probes are done for the affected resource. If this is set to
	* exclusive, probes will only be done on nodes listed in exclusive constraints.
	* This function bans the resource from the node if the node is not listed.
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check \p rsc on
	*/
	static void
	apply_exclusive_discovery(pe_resource_t rsc, pe_node_t node)
	{
	if (rsc->exclusive_discover \|\| uber_parent(rsc)->exclusive_discover) {
	pe_node_t *match = NULL;

	// If this is a collective resource, apply recursively to children
	g_list_foreach(rsc->children, (GFunc) apply_exclusive_discovery, node);

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if ((match != NULL)
	&& (match->rsc_discover_mode != pe_discover_exclusive)) {
	match->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Apply stickiness to a resource if appropriate
	*
	* \param[in] rsc Resource to check for stickiness
	* \param[in] data_set Cluster working set
	*/
	static void
	apply_stickiness(pe_resource_t rsc, pe_working_set_t data_set)
	{
	pe_node_t *node = NULL;

	// If this is a collective resource, apply recursively to children instead
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, (GFunc) apply_stickiness, data_set);
	return;
	}

	/* A resource is sticky if it is managed, has stickiness configured, and is
	* active on a single node.
	*/
	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)
	\|\| (rsc->stickiness < 1) \|\| !pcmk__list_of_1(rsc->running_on)) {
	return;
	}

	node = rsc->running_on->data;

	/* In a symmetric cluster, stickiness can always be used. In an
	* asymmetric cluster, we have to check whether the resource is still
	* allowed on the node, so we don't keep the resource somewhere it is no
	* longer explicitly enabled.
	*/
	if (!pcmk_is_set(rsc->cluster->flags, pe_flag_symmetric_cluster)
	&& (pe_hash_table_lookup(rsc->allowed_nodes,
	node->details->id) == NULL)) {
	pe_rsc_debug(rsc,
	"Ignoring %s stickiness because the cluster is "
	"asymmetric and %s is not explicitly allowed",
	rsc->id, pe__node_name(node));
	return;
	}

	pe_rsc_debug(rsc, "Resource %s has %d stickiness on %s",
	rsc->id, rsc->stickiness, pe__node_name(node));
	resource_location(rsc, node, rsc->stickiness, "stickiness",
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Apply shutdown locks for all resources as appropriate
	*
	* \param[in] data_set Cluster working set
	*/
	static void
	apply_shutdown_locks(pe_working_set_t *data_set)
	{
	if (!pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
	return;
	}
	for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	rsc->cmds->shutdown_lock(rsc);
	}
	}

	/*!
	* \internal
	* \brief Calculate the number of available nodes in the cluster
	*
	* \param[in] data_set Cluster working set
	*/
	static void
	count_available_nodes(pe_working_set_t *data_set)
	{
	if (pcmk_is_set(data_set->flags, pe_flag_no_compat)) {
	return;
	}

	// @COMPAT for API backward compatibility only (cluster does not use value)
	for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;

	if ((node != NULL) && (node->weight >= 0) && node->details->online
	&& (node->details->type != node_ping)) {
	data_set->max_valid_nodes++;
	}
	}
	crm_trace("Online node count: %d", data_set->max_valid_nodes);
	}

	/*
	* \internal
	* \brief Apply node-specific scheduling criteria
	*
	* After the CIB has been unpacked, process node-specific scheduling criteria
	* including shutdown locks, location constraints, resource stickiness,
	* migration thresholds, and exclusive resource discovery.
	*/
	static void
	apply_node_criteria(pe_working_set_t *data_set)
	{
	crm_trace("Applying node-specific scheduling criteria");
	apply_shutdown_locks(data_set);
	count_available_nodes(data_set);
	pcmk__apply_locations(data_set);
	g_list_foreach(data_set->resources, (GFunc) apply_stickiness, data_set);

	for (GList *node_iter = data_set->nodes; node_iter != NULL;
	node_iter = node_iter->next) {
	for (GList *rsc_iter = data_set->resources; rsc_iter != NULL;
	rsc_iter = rsc_iter->next) {
	pe_node_t node = (pe_node_t ) node_iter->data;
	pe_resource_t rsc = (pe_resource_t ) rsc_iter->data;

	check_failure_threshold(rsc, node);
	apply_exclusive_discovery(rsc, node);
	}
	}
	}

	/*!
	* \internal
	* \brief Allocate resources to nodes
	*
	* \param[in] data_set Cluster working set
	*/
	static void
	allocate_resources(pe_working_set_t *data_set)
	{
	GList *iter = NULL;

	crm_trace("Allocating resources to nodes");

	if (!pcmk__str_eq(data_set->placement_strategy, "default", pcmk__str_casei)) {
	pcmk__sort_resources(data_set);
	}
	pcmk__show_node_capacities("Original", data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	/* Allocate remote connection resources first (which will also allocate
	* any colocation dependencies). If the connection is migrating, always
	* prefer the partial migration target.
	*/
	for (iter = data_set->resources; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	if (rsc->is_remote_node) {
	pe_rsc_trace(rsc, "Allocating remote connection resource '%s'",
	rsc->id);
	rsc->cmds->assign(rsc, rsc->partial_migration_target);
	}
	}
	}

	/* now do the rest of the resources */
	for (iter = data_set->resources; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	if (!rsc->is_remote_node) {
	pe_rsc_trace(rsc, "Allocating %s resource '%s'",
	crm_element_name(rsc->xml), rsc->id);
	rsc->cmds->assign(rsc, NULL);
	}
	}

	pcmk__show_node_capacities("Remaining", data_set);
	}

	/*!
	* \internal
	* \brief Schedule fail count clearing on online nodes if resource is orphaned
	*
	* \param[in] rsc Resource to check
	* \param[in] data_set Cluster working set
	*/
	static void
	clear_failcounts_if_orphaned(pe_resource_t rsc, pe_working_set_t data_set)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	return;
	}
	crm_trace("Clear fail counts for orphaned resource %s", rsc->id);

	/* There's no need to recurse into rsc->children because those
	* should just be unallocated clone instances.
	*/

	for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;
	pe_action_t *clear_op = NULL;

	if (!node->details->online) {
	continue;
	}
	if (pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL) == 0) {
	continue;
	}

	clear_op = pe__clear_failcount(rsc, node, "it is orphaned", data_set);

	/* We can't use order_action_then_stop() here because its
	* pe_order_preserve breaks things
	*/
	pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc),
	NULL, pe_order_optional, data_set);
	}
	}

	/*!
	* \internal
	* \brief Schedule any resource actions needed
	*
	* \param[in] data_set Cluster working set
	*/
	static void
	schedule_resource_actions(pe_working_set_t *data_set)
	{
	// Process deferred action checks
	pe__foreach_param_check(data_set, check_params);
	pe__free_param_checks(data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
	crm_trace("Scheduling probes");
	pcmk__schedule_probes(data_set);
	}

	if (pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
	g_list_foreach(data_set->resources,
	(GFunc) clear_failcounts_if_orphaned, data_set);
	}

	crm_trace("Scheduling resource actions");
	for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	rsc->cmds->create_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource or any of its descendants are managed
	*
	* \param[in] rsc Resource to check
	*
	* \return true if resource or any descendent is managed, otherwise false
	*/
	static bool
	is_managed(const pe_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	return true;
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	if (is_managed((pe_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether any resources in the cluster are managed
	*
	* \param[in] data_set Cluster working set
	*
	* \return true if any resource is managed, otherwise false
	*/
	static bool
	any_managed_resources(pe_working_set_t *data_set)
	{
	for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
	if (is_managed((pe_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node requires fencing
	*
	* \param[in] node Node to check
	* \param[in] have_managed Whether any resource in cluster is managed
	* \param[in] data_set Cluster working set
	*
	* \return true if \p node should be fenced, otherwise false
	*/
	static bool
	needs_fencing(pe_node_t node, bool have_managed, pe_working_set_t data_set)
	{
	return have_managed && node->details->unclean
	&& pe_can_fence(data_set, node);
	}

	/*!
	* \internal
	* \brief Check whether a node requires shutdown
	*
	* \param[in] node Node to check
	*
	* \return true if \p node should be shut down, otherwise false
	*/
	static bool
	needs_shutdown(pe_node_t *node)
	{
	if (pe__is_guest_or_remote_node(node)) {
	/* Do not send shutdown actions for Pacemaker Remote nodes.
	* @TODO We might come up with a good use for this in the future.
	*/
	return false;
	}
	return node->details->online && node->details->shutdown;
	}

	/*!
	* \internal
	* \brief Track and order non-DC fencing
	*
	* \param[in] list List of existing non-DC fencing actions
	* \param[in] action Fencing action to prepend to \p list
	*
	* \return (Possibly new) head of \p list
	*/
	static GList *
	add_nondc_fencing(GList list, pe_action_t action, pe_working_set_t *data_set)
	{
	if (!pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)
	&& (list != NULL)) {
	/* Concurrent fencing is disabled, so order each non-DC
	* fencing in a chain. If there is any DC fencing or
	* shutdown, it will be ordered after the last action in the
	* chain later.
	*/
	order_actions((pe_action_t *) list->data, action, pe_order_optional);
	}
	return g_list_prepend(list, action);
	}

	/*!
	* \internal
	* \brief Schedule a node for fencing
	*
	* \param[in] node Node that requires fencing
	* \param[in] data_set Cluster working set
	*/
	static pe_action_t *
	schedule_fencing(pe_node_t node, pe_working_set_t data_set)
	{
	pe_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean",
	FALSE, data_set);

	pe_warn("Scheduling node %s for fencing", pe__node_name(node));
	pcmk__order_vs_fence(fencing, data_set);
	return fencing;
	}

	/*!
	* \internal
	* \brief Create and order node fencing and shutdown actions
	*
	* \param[in] data_set Cluster working set
	*/
	static void
	schedule_fencing_and_shutdowns(pe_working_set_t *data_set)
	{
	pe_action_t *dc_down = NULL;
	bool integrity_lost = false;
	bool have_managed = any_managed_resources(data_set);
	GList *fencing_ops = NULL;
	GList *shutdown_ops = NULL;

	crm_trace("Scheduling fencing and shutdowns as needed");
	if (!have_managed) {
	crm_notice("No fencing will be done until there are resources to manage");
	}

	// Check each node for whether it needs fencing or shutdown
	for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;
	pe_action_t *fencing = NULL;

	/* Guest nodes are "fenced" by recovering their container resource,
	* so handle them separately.
	*/
	if (pe__is_guest_node(node)) {
	if (node->details->remote_requires_reset && have_managed
	&& pe_can_fence(data_set, node)) {
	pcmk__fence_guest(node);
	}
	continue;
	}

	if (needs_fencing(node, have_managed, data_set)) {
	fencing = schedule_fencing(node, data_set);

	// Track DC and non-DC fence actions separately
	if (node->details->is_dc) {
	dc_down = fencing;
	} else {
	fencing_ops = add_nondc_fencing(fencing_ops, fencing, data_set);
	}

	} else if (needs_shutdown(node)) {
	pe_action_t *down_op = pcmk__new_shutdown_action(node);

	// Track DC and non-DC shutdown actions separately
	if (node->details->is_dc) {
	dc_down = down_op;
	} else {
	shutdown_ops = g_list_prepend(shutdown_ops, down_op);
	}
	}

	if ((fencing == NULL) && node->details->unclean) {
	integrity_lost = true;
	pe_warn("Node %s is unclean but cannot be fenced",
	pe__node_name(node));
	}
	}

	if (integrity_lost) {
	if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pe_warn("Resource functionality and data integrity cannot be "
	"guaranteed (configure, enable, and test fencing to "
	"correct this)");

	} else if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	crm_notice("Unclean nodes will not be fenced until quorum is "
	"attained or no-quorum-policy is set to ignore");
	}
	}

	if (dc_down != NULL) {
	/* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
	* DC elections. However, we don't want to order non-DC shutdowns before
	* a DC fencing, because even though we don't want a node that's
	* shutting down to become DC, the DC fencing could be ordered before a
	* clone stop that's also ordered before the shutdowns, thus leading to
	* a graph loop.
	*/
	if (pcmk__str_eq(dc_down->task, CRM_OP_SHUTDOWN, pcmk__str_none)) {
	pcmk__order_after_each(dc_down, shutdown_ops);
	}

	// Order any non-DC fencing before any DC fencing or shutdown

	if (pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)) {
	/* With concurrent fencing, order each non-DC fencing action
	* separately before any DC fencing or shutdown.
	*/
	pcmk__order_after_each(dc_down, fencing_ops);
	} else if (fencing_ops != NULL) {
	/* Without concurrent fencing, the non-DC fencing actions are
	* already ordered relative to each other, so we just need to order
	* the DC fencing after the last action in the chain (which is the
	* first item in the list).
	*/
	order_actions((pe_action_t *) fencing_ops->data, dc_down,
	pe_order_optional);
	}
	}
	g_list_free(fencing_ops);
	g_list_free(shutdown_ops);
	}

	static void
	log_resource_details(pe_working_set_t *data_set)
	{
	pcmk__output_t *out = data_set->priv;
	GList *all = NULL;

	/* We need a list of nodes that we are allowed to output information for.
	* This is necessary because out->message for all the resource-related
	* messages expects such a list, due to the `crm_mon --node=` feature. Here,
	* we just make it a list of all the nodes.
	*/
	all = g_list_prepend(all, (gpointer) "*");

	for (GList *item = data_set->resources; item != NULL; item = item->next) {
	pe_resource_t rsc = (pe_resource_t ) item->data;

	// Log all resources except inactive orphans
	if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)
	\|\| (rsc->role != RSC_ROLE_STOPPED)) {
	out->message(out, crm_map_element_name(rsc->xml), 0, rsc, all, all);
	}
	}

	g_list_free(all);
	}

	static void
	log_all_actions(pe_working_set_t *data_set)
	{
	/* This only ever outputs to the log, so ignore whatever output object was
	* previously set and just log instead.
	*/
	pcmk__output_t *prev_out = data_set->priv;
	pcmk__output_t *out = NULL;

	if (pcmk__log_output_new(&out) != pcmk_rc_ok) {
	return;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);
	pcmk__output_set_log_level(out, LOG_NOTICE);
	data_set->priv = out;

	out->begin_list(out, NULL, NULL, "Actions");
	pcmk__output_actions(data_set);
	out->end_list(out);
	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);

	data_set->priv = prev_out;
	}

	/*!
	* \internal
	* \brief Log all required but unrunnable actions at trace level
	*
	* \param[in] data_set Cluster working set
	*/
	static void
	log_unrunnable_actions(pe_working_set_t *data_set)
	{
	const uint64_t flags = pe_action_optional\|pe_action_runnable\|pe_action_pseudo;

	crm_trace("Required but unrunnable actions:");
	for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) {
	pe_action_t action = (pe_action_t ) iter->data;

	if (!pcmk_any_flags_set(action->flags, flags)) {
	pcmk__log_action("\t", action, true);
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack the CIB for scheduling
	*
	* \param[in] cib CIB XML to unpack (may be NULL if previously unpacked)
	* \param[in] flags Working set flags to set in addition to defaults
	* \param[in] data_set Cluster working set
	*/
	static void
	unpack_cib(xmlNode cib, unsigned long long flags, pe_working_set_t data_set)
	{
	const char* localhost_save = NULL;

	if (pcmk_is_set(data_set->flags, pe_flag_have_status)) {
	crm_trace("Reusing previously calculated cluster status");
	pe__set_working_set_flags(data_set, flags);
	return;
	}

	if (data_set->localhost) {
	localhost_save = data_set->localhost;
	}

	CRM_ASSERT(cib != NULL);
	crm_trace("Calculating cluster status");

	/* This will zero the entire struct without freeing anything first, so
	* callers should never call pcmk__schedule_actions() with a populated data
	* set unless pe_flag_have_status is set (i.e. cluster_status() was
	* previously called, whether directly or via pcmk__schedule_actions()).
	*/
	set_working_set_defaults(data_set);

	if (localhost_save) {
	data_set->localhost = localhost_save;
	}

	pe__set_working_set_flags(data_set, flags);
	data_set->input = cib;
	cluster_status(data_set); // Sets pe_flag_have_status
	}

	/*!
	* \internal
	* \brief Run the scheduler for a given CIB
	*
	* \param[in] cib CIB XML to use as scheduler input
	* \param[in] flags Working set flags to set in addition to defaults
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__schedule_actions(xmlNode *cib, unsigned long long flags,
	pe_working_set_t *data_set)
	{
	unpack_cib(cib, flags, data_set);
	pcmk__set_allocation_methods(data_set);
	pcmk__apply_node_health(data_set);
	pcmk__unpack_constraints(data_set);
	if (pcmk_is_set(data_set->flags, pe_flag_check_config)) {
	return;
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_quick_location) &&
	pcmk__is_daemon) {
	log_resource_details(data_set);
	}

	apply_node_criteria(data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_quick_location)) {
	return;
	}

	pcmk__create_internal_constraints(data_set);
	pcmk__handle_rsc_config_changes(data_set);
	allocate_resources(data_set);
	schedule_resource_actions(data_set);

	/* Remote ordering constraints need to happen prior to calculating fencing
	* because it is one more place we can mark nodes as needing fencing.
	*/
	pcmk__order_remote_connection_actions(data_set);

	schedule_fencing_and_shutdowns(data_set);
	pcmk__apply_orderings(data_set);
	log_all_actions(data_set);
	pcmk__create_graph(data_set);

	if (get_crm_log_level() == LOG_TRACE) {
	log_unrunnable_actions(data_set);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_recurring.c b/lib/pacemaker/pcmk_sched_recurring.c
	index 0019350fd9..177ceca9de 100644
	--- a/lib/pacemaker/pcmk_sched_recurring.c
	+++ b/lib/pacemaker/pcmk_sched_recurring.c
	@@ -1,714 +1,714 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Information parsed from an operation history entry in the CIB
	struct op_history {
	// XML attributes
	const char *id; // ID of history entry
	const char *name; // Action name

	// Parsed information
	char *key; // Operation key for action
	enum rsc_role_e role; // Action role (or RSC_ROLE_UNKNOWN for default)
	guint interval_ms; // Action interval
	};

	/*!
	* \internal
	* \brief Parse an interval from XML
	*
	* \param[in] xml XML containing an interval attribute
	*
	* \return Interval parsed from XML (or 0 as default)
	*/
	static guint
	xe_interval(const xmlNode *xml)
	{
	return crm_parse_interval_spec(crm_element_value(xml,
	XML_LRM_ATTR_INTERVAL));
	}

	/*!
	* \internal
	* \brief Check whether an operation exists multiple times in resource history
	*
	* \param[in] rsc Resource with history to search
	* \param[in] name Name of action to search for
	* \param[in] interval_ms Interval (in milliseconds) of action to search for
	*
	* \return true if an operation with \p name and \p interval_ms exists more than
	* once in the operation history of \p rsc, otherwise false
	*/
	static bool
	is_op_dup(const pe_resource_t rsc, const char name, guint interval_ms)
	{
	const char *id = NULL;

	for (xmlNode *op = first_named_child(rsc->ops_xml, "op");
	op != NULL; op = crm_next_same_xml(op)) {

	// Check whether action name and interval match
	if (!pcmk__str_eq(crm_element_value(op, "name"),
	name, pcmk__str_none)
	\|\| (xe_interval(op) != interval_ms)) {
	continue;
	}

	if (ID(op) == NULL) {
	continue; // Shouldn't be possible
	}

	if (id == NULL) {
	id = ID(op); // First matching op
	} else {
	pcmk__config_err("Operation %s is duplicate of %s (do not use "
	"same name and interval combination more "
	"than once per resource)", ID(op), id);
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether an action name is one that can be recurring
	*
	* \param[in] name Action name to check
	*
	* \return true if \p name is an action known to be unsuitable as a recurring
	* operation, otherwise false
	*
	* \note Pacemaker's current philosophy is to allow users to configure recurring
	* operations except for a short list of actions known not to be suitable
	* for that (as opposed to allowing only actions known to be suitable,
	* which includes only monitor). Among other things, this approach allows
	* users to define their own custom operations and make them recurring,
	* though that use case is not well tested.
	*/
	static bool
	op_cannot_recur(const char *name)
	{
	return pcmk__str_any_of(name, RSC_STOP, RSC_START, RSC_DEMOTE, RSC_PROMOTE,
	CRMD_ACTION_RELOAD_AGENT, CRMD_ACTION_MIGRATE,
	CRMD_ACTION_MIGRATED, NULL);
	}

	/*!
	* \internal
	* \brief Check whether a resource history entry is for a recurring action
	*
	* \param[in] rsc Resource that history entry is for
	* \param[in] xml XML of resource history entry to check
	* \param[out] op Where to store parsed info if recurring
	*
	* \return true if \p xml is for a recurring action, otherwise false
	*/
	static bool
	is_recurring_history(const pe_resource_t rsc, const xmlNode xml,
	struct op_history *op)
	{
	const char *role = NULL;

	op->interval_ms = xe_interval(xml);
	if (op->interval_ms == 0) {
	return false; // Not recurring
	}

	op->id = ID(xml);
	if (pcmk__str_empty(op->id)) {
	pcmk__config_err("Ignoring resource history entry without ID");
	return false; // Shouldn't be possible (unless CIB was manually edited)
	}

	op->name = crm_element_value(xml, "name");
	if (op_cannot_recur(op->name)) {
	pcmk__config_err("Ignoring %s because %s action cannot be recurring",
	op->id, pcmk__s(op->name, "unnamed"));
	return false;
	}

	// There should only be one recurring operation per action/interval
	if (is_op_dup(rsc, op->name, op->interval_ms)) {
	return false;
	}

	// Ensure role is valid if specified
	role = crm_element_value(xml, "role");
	if (role == NULL) {
	op->role = RSC_ROLE_UNKNOWN;
	} else {
	op->role = text2role(role);
	if (op->role == RSC_ROLE_UNKNOWN) {
	pcmk__config_err("Ignoring %s because %s is not a valid role",
	op->id, role);
	}
	}

	// Disabled resources don't get monitored
	op->key = pcmk__op_key(rsc->id, op->name, op->interval_ms);
	if (find_rsc_op_entry(rsc, op->key) == NULL) {
	crm_trace("Not creating recurring action %s for disabled resource %s",
	op->id, rsc->id);
	free(op->key);
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Check whether a recurring action for an active role should be optional
	*
	* \param[in] rsc Resource that recurring action is for
	* \param[in] node Node that \p rsc will be active on (if any)
	* \param[in] key Operation key for recurring action to check
	* \param[in] start Start action for \p rsc
	*
	* \return true if recurring action should be optional, otherwise false
	*/
	static bool
	active_recurring_should_be_optional(const pe_resource_t *rsc,
	const pe_node_t node, const char key,
	pe_action_t *start)
	{
	GList *possible_matches = NULL;

	if (node == NULL) { // Should only be possible if unmanaged and stopped
	pe_rsc_trace(rsc, "%s will be mandatory because resource is unmanaged",
	key);
	return false;
	}

	if (!pcmk_is_set(rsc->cmds->action_flags(start, NULL),
	pe_action_optional)) {
	pe_rsc_trace(rsc, "%s will be mandatory because %s is",
	key, start->uuid);
	return false;
	}

	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches == NULL) {
	pe_rsc_trace(rsc, "%s will be mandatory because it is not active on %s",
	key, pe__node_name(node));
	return false;
	}

	for (GList *iter = possible_matches; iter != NULL; iter = iter->next) {
	pe_action_t op = (pe_action_t ) iter->data;

	if (pcmk_is_set(op->flags, pe_action_reschedule)) {
	pe_rsc_trace(rsc,
	"%s will be mandatory because "
	"it needs to be rescheduled", key);
	g_list_free(possible_matches);
	return false;
	}
	}

	g_list_free(possible_matches);
	return true;
	}

	/*!
	* \internal
	* \brief Create recurring action from resource history entry for an active role
	*
	* \param[in,out] rsc Resource that resource history is for
	* \param[in] start Start action for \p rsc on \p node
	* \param[in] node Node that resource will be active on (if any)
	* \param[in] op Resource history entry
	*/
	static void
	recurring_op_for_active(pe_resource_t rsc, pe_action_t start,
	const pe_node_t node, const struct op_history op)
	{
	pe_action_t *mon = NULL;
	bool is_optional = true;

	// We're only interested in recurring actions for active roles
	if (op->role == RSC_ROLE_STOPPED) {
	return;
	}

	is_optional = active_recurring_should_be_optional(rsc, node, op->key,
	start);

	if (((op->role != RSC_ROLE_UNKNOWN) && (rsc->next_role != op->role))
	\|\| ((op->role == RSC_ROLE_UNKNOWN)
	&& (rsc->next_role == RSC_ROLE_PROMOTED))) {
	// Configured monitor role doesn't match role resource will have

	if (is_optional) { // It's running, so cancel it
	char *after_key = NULL;
	pe_action_t *cancel_op = pcmk__new_cancel_action(rsc, op->name,
	op->interval_ms,
	node);

	switch (rsc->role) {
	case RSC_ROLE_UNPROMOTED:
	case RSC_ROLE_STARTED:
	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	after_key = promote_key(rsc);

	} else if (rsc->next_role == RSC_ROLE_STOPPED) {
	after_key = stop_key(rsc);
	}

	break;
	case RSC_ROLE_PROMOTED:
	after_key = demote_key(rsc);
	break;
	default:
	break;
	}

	if (after_key) {
	pcmk__new_ordering(rsc, NULL, cancel_op, rsc, after_key, NULL,
	pe_order_runnable_left, rsc->cluster);
	}
	}

	do_crm_log((is_optional? LOG_INFO : LOG_TRACE),
	"%s recurring action %s because %s configured for %s role "
	"(not %s)",
	(is_optional? "Cancelling" : "Ignoring"), op->key, op->id,
	role2text((op->role == RSC_ROLE_UNKNOWN)? RSC_ROLE_UNPROMOTED : op->role),
	role2text(rsc->next_role));
	return;
	}

	pe_rsc_trace(rsc,
	"Creating %s recurring action %s for %s (%s %s on %s)",
	(is_optional? "optional" : "mandatory"), op->key,
	op->id, rsc->id, role2text(rsc->next_role),
	pe__node_name(node));

	mon = custom_action(rsc, strdup(op->key), op->name, node, is_optional, TRUE,
	rsc->cluster);

	if (!pcmk_is_set(start->flags, pe_action_runnable)) {
	pe_rsc_trace(rsc, "%s is unrunnable because start is", mon->uuid);
	pe__clear_action_flags(mon, pe_action_runnable);

	} else if ((node == NULL) \|\| !node->details->online
	\|\| node->details->unclean) {
	pe_rsc_trace(rsc, "%s is unrunnable because no node is available",
	mon->uuid);
	pe__clear_action_flags(mon, pe_action_runnable);

	} else if (!pcmk_is_set(mon->flags, pe_action_optional)) {
	pe_rsc_info(rsc, "Start %s-interval %s for %s on %s",
	pcmk__readable_interval(op->interval_ms), mon->task,
	rsc->id, pe__node_name(node));
	}

	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	pe__add_action_expected_result(mon, CRM_EX_PROMOTED);
	}

	// Order monitor relative to other actions
	if ((node == NULL) \|\| pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pcmk__new_ordering(rsc, start_key(rsc), NULL,
	NULL, strdup(mon->uuid), mon,
	pe_order_implies_then\|pe_order_runnable_left,
	rsc->cluster);

	pcmk__new_ordering(rsc, reload_key(rsc), NULL,
	NULL, strdup(mon->uuid), mon,
	pe_order_implies_then\|pe_order_runnable_left,
	rsc->cluster);

	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	pcmk__new_ordering(rsc, promote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional\|pe_order_runnable_left,
	rsc->cluster);

	} else if (rsc->role == RSC_ROLE_PROMOTED) {
	pcmk__new_ordering(rsc, demote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional\|pe_order_runnable_left,
	rsc->cluster);
	}
	}
	}

	/*!
	* \internal
	* \brief Cancel a recurring action if running on a node
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] node Node to cancel action on
	* \param[in] key Operation key for action
	* \param[in] name Action name
	* \param[in] interval_ms Action interval (in milliseconds)
	*/
	static void
	cancel_if_running(pe_resource_t rsc, const pe_node_t node, const char *key,
	const char *name, guint interval_ms)
	{
	GList *possible_matches = find_actions_exact(rsc->actions, key, node);
	pe_action_t *cancel_op = NULL;

	if (possible_matches == NULL) {
	return; // Recurring action isn't running on this node
	}
	g_list_free(possible_matches);

	cancel_op = pcmk__new_cancel_action(rsc, name, interval_ms, node);

	switch (rsc->next_role) {
	case RSC_ROLE_STARTED:
	case RSC_ROLE_UNPROMOTED:
	/* Order starts after cancel. If the current role is
	* stopped, this cancels the monitor before the resource
	* starts; if the current role is started, then this cancels
	* the monitor on a migration target before starting there.
	*/
	pcmk__new_ordering(rsc, NULL, cancel_op,
	rsc, start_key(rsc), NULL,
	pe_order_runnable_left, rsc->cluster);
	break;
	default:
	break;
	}
	pe_rsc_info(rsc,
	"Cancelling %s-interval %s action for %s on %s because "
	"configured for " RSC_ROLE_STOPPED_S " role (not %s)",
	pcmk__readable_interval(interval_ms), name, rsc->id,
	pe__node_name(node), role2text(rsc->next_role));
	}

	/*!
	* \internal
	* \brief Order an action after all probes of a resource on a node
	*
	* \param[in,out] rsc Resource to check for probes
	* \param[in] node Node to check for probes of \p rsc
	* \param[in,out] action Action to order after probes of \p rsc on \p node
	*/
	static void
	order_after_probes(pe_resource_t rsc, const pe_node_t node,
	pe_action_t *action)
	{
	GList *probes = pe__resource_actions(rsc, node, RSC_STATUS, FALSE);

	for (GList *iter = probes; iter != NULL; iter = iter->next) {
	order_actions((pe_action_t *) iter->data, action,
	pe_order_runnable_left);
	}
	g_list_free(probes);
	}

	/*!
	* \internal
	* \brief Order an action after all stops of a resource on a node
	*
	* \param[in,out] rsc Resource to check for stops
	* \param[in] node Node to check for stops of \p rsc
	* \param[in,out] action Action to order after stops of \p rsc on \p node
	*/
	static void
	order_after_stops(pe_resource_t rsc, const pe_node_t node,
	pe_action_t *action)
	{
	GList *stop_ops = pe__resource_actions(rsc, node, RSC_STOP, TRUE);

	for (GList *iter = stop_ops; iter != NULL; iter = iter->next) {
	pe_action_t stop = (pe_action_t ) iter->data;

	if (!pcmk_is_set(stop->flags, pe_action_optional)
	&& !pcmk_is_set(action->flags, pe_action_optional)
	&& !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "%s optional on %s: unmanaged",
	action->uuid, pe__node_name(node));
	pe__set_action_flags(action, pe_action_optional);
	}

	if (!pcmk_is_set(stop->flags, pe_action_runnable)) {
	crm_debug("%s unrunnable on %s: stop is unrunnable",
	action->uuid, pe__node_name(node));
	pe__clear_action_flags(action, pe_action_runnable);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pcmk__new_ordering(rsc, stop_key(rsc), stop,
	NULL, NULL, action,
	pe_order_implies_then\|pe_order_runnable_left,
	rsc->cluster);
	}
	}
	g_list_free(stop_ops);
	}

	/*!
	* \internal
	* \brief Create recurring action from resource history entry for inactive role
	*
	* \param[in,out] rsc Resource that resource history is for
	* \param[in] node Node that resource will be active on (if any)
	* \param[in] op Resource history entry
	*/
	static void
	recurring_op_for_inactive(pe_resource_t rsc, const pe_node_t node,
	const struct op_history *op)
	{
	GList *possible_matches = NULL;

	// We're only interested in recurring actions for the inactive role
	if (op->role != RSC_ROLE_STOPPED) {
	return;
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	crm_notice("Ignoring %s (recurring monitors for " RSC_ROLE_STOPPED_S
	" role are not supported for anonymous clones)", op->id);
	return; // @TODO add support
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s on nodes "
	"where it should not be running", op->id, rsc->id);

	for (GList *iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) {
	pe_node_t stop_node = (pe_node_t ) iter->data;

	bool is_optional = true;
	pe_action_t *stopped_mon = NULL;

	// Cancel action on node where resource will be active
	if ((node != NULL)
	&& pcmk__str_eq(stop_node->details->uname, node->details->uname,
	pcmk__str_casei)) {
	cancel_if_running(rsc, node, op->key, op->name, op->interval_ms);
	continue;
	}

	// Recurring action on this node is optional if it's already active here
	possible_matches = find_actions_exact(rsc->actions, op->key, stop_node);
	is_optional = (possible_matches != NULL);
	g_list_free(possible_matches);

	pe_rsc_trace(rsc,
	"Creating %s recurring action %s for %s (%s "
	RSC_ROLE_STOPPED_S " on %s)",
	(is_optional? "optional" : "mandatory"),
	op->key, op->id, rsc->id, pe__node_name(stop_node));

	stopped_mon = custom_action(rsc, strdup(op->key), op->name, stop_node,
	is_optional, TRUE, rsc->cluster);

	pe__add_action_expected_result(stopped_mon, CRM_EX_NOT_RUNNING);

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	order_after_probes(rsc, stop_node, stopped_mon);
	}

	/* The recurring action is for the inactive role, so it shouldn't be
	* performed until the resource is inactive.
	*/
	order_after_stops(rsc, stop_node, stopped_mon);

	if (!stop_node->details->online \|\| stop_node->details->unclean) {
	pe_rsc_debug(rsc, "%s unrunnable on %s: node unavailable)",
	stopped_mon->uuid, pe__node_name(stop_node));
	pe__clear_action_flags(stopped_mon, pe_action_runnable);
	}

	if (pcmk_is_set(stopped_mon->flags, pe_action_runnable)
	&& !pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
	crm_notice("Start recurring %s-interval %s for "
	RSC_ROLE_STOPPED_S " %s on %s",
	pcmk__readable_interval(op->interval_ms),
	stopped_mon->task, rsc->id, pe__node_name(stop_node));
	}
	}
	}

	/*!
	* \internal
	* \brief Create recurring actions for a resource
	*
	* \param[in,out] rsc Resource to create recurring actions for
	*/
	void
	pcmk__create_recurring_actions(pe_resource_t *rsc)
	{
	pe_action_t *start = NULL;

	if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	pe_rsc_trace(rsc, "Skipping recurring actions for blocked resource %s",
	rsc->id);
	return;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_maintenance)) {
	pe_rsc_trace(rsc, "Skipping recurring actions for %s "
	"in maintenance mode", rsc->id);
	return;
	}

	if (rsc->allocated_to == NULL) {
	// Recurring actions for active roles not needed

	} else if (rsc->allocated_to->details->maintenance) {
	pe_rsc_trace(rsc,
	"Skipping recurring actions for %s on %s "
	"in maintenance mode",
	rsc->id, pe__node_name(rsc->allocated_to));

	} else if ((rsc->next_role != RSC_ROLE_STOPPED)
	\|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	// Recurring actions for active roles needed
	start = start_action(rsc, rsc->allocated_to, TRUE);
	}

	pe_rsc_trace(rsc, "Creating any recurring actions needed for %s", rsc->id);

	for (xmlNode *op = first_named_child(rsc->ops_xml, "op");
	op != NULL; op = crm_next_same_xml(op)) {

	struct op_history op_history = { NULL, };

	if (!is_recurring_history(rsc, op, &op_history)) {
	continue;
	}

	if (start != NULL) {
	recurring_op_for_active(rsc, start, rsc->allocated_to, &op_history);
	}
	recurring_op_for_inactive(rsc, rsc->allocated_to, &op_history);

	free(op_history.key);
	}
	}

	/*!
	* \internal
	* \brief Create an executor cancel action
	*
	* \param[in,out] rsc Resource of action to cancel
	* \param[in] task Name of action to cancel
	* \param[in] interval_ms Interval of action to cancel
	* \param[in] node Node of action to cancel
	*
	* \return Created op
	*/
	pe_action_t *
	pcmk__new_cancel_action(pe_resource_t rsc, const char task, guint interval_ms,
	const pe_node_t *node)
	{
	pe_action_t *cancel_op = NULL;
	char *key = NULL;
	char *interval_ms_s = NULL;

	CRM_ASSERT((rsc != NULL) && (task != NULL) && (node != NULL));

	// @TODO dangerous if possible to schedule another action with this key
	key = pcmk__op_key(rsc->id, task, interval_ms);

	cancel_op = custom_action(rsc, key, RSC_CANCEL, node, FALSE, TRUE,
	rsc->cluster);

	pcmk__str_update(&cancel_op->task, RSC_CANCEL);
	pcmk__str_update(&cancel_op->cancel_task, task);

	interval_ms_s = crm_strdup_printf("%u", interval_ms);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, task);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL_MS, interval_ms_s);
	free(interval_ms_s);

	return cancel_op;
	}

	/*!
	* \internal
	* \brief Schedule cancellation of a recurring action
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] call_id Action's call ID from history
	* \param[in] task Action name
	* \param[in] interval_ms Action interval
	* \param[in] node Node that history entry is for
	* \param[in] reason Short description of why action is being cancelled
	*/
	void
	pcmk__schedule_cancel(pe_resource_t rsc, const char call_id, const char *task,
	guint interval_ms, const pe_node_t *node,
	const char *reason)
	{
	pe_action_t *cancel = NULL;

	CRM_CHECK((rsc != NULL) && (task != NULL)
	&& (node != NULL) && (reason != NULL),
	return);

	crm_info("Recurring %s-interval %s for %s will be stopped on %s: %s",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node), reason);
	cancel = pcmk__new_cancel_action(rsc, task, interval_ms, node);
	add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);

	// Cancellations happen after stops
	pcmk__new_ordering(rsc, stop_key(rsc), NULL, rsc, NULL, cancel,
	pe_order_optional, rsc->cluster);
	}

	/*!
	* \internal
	* \brief Reschedule a recurring action
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] task Name of action being rescheduled
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] node Node where action should be rescheduled
	*/
	void
	pcmk__reschedule_recurring(pe_resource_t rsc, const char task,
	- guint interval_ms, pe_node_t *node)
	+ guint interval_ms, const pe_node_t *node)
	{
	pe_action_t *op = NULL;

	trigger_unfencing(rsc, node, "Device parameters changed (reschedule)",
	NULL, rsc->cluster);
	op = custom_action(rsc, pcmk__op_key(rsc->id, task, interval_ms),
	task, node, TRUE, TRUE, rsc->cluster);
	pe__set_action_flags(op, pe_action_reschedule);
	}

	/*!
	* \internal
	* \brief Check whether an action is recurring
	*
	* \param[in] action Action to check
	*
	* \return true if \p action has a nonzero interval, otherwise false
	*/
	bool
	pcmk__action_is_recurring(const pe_action_t *action)
	{
	guint interval_ms = 0;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	return false;
	}
	return (interval_ms > 0);
	}
	diff --git a/lib/pengine/failcounts.c b/lib/pengine/failcounts.c
	index 8f1769609c..b97d221c6e 100644
	--- a/lib/pengine/failcounts.c
	+++ b/lib/pengine/failcounts.c
	@@ -1,400 +1,400 @@
	/*
	* Copyright 2008-2022 the Pacemaker project contributors
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/types.h>
	#include <regex.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/pengine/internal.h>

	static gboolean
	is_matched_failure(const char rsc_id, const xmlNode conf_op_xml,
	const xmlNode *lrm_op_xml)
	{
	gboolean matched = FALSE;
	const char *conf_op_name = NULL;
	const char *lrm_op_task = NULL;
	const char *conf_op_interval_spec = NULL;
	guint conf_op_interval_ms = 0;
	guint lrm_op_interval_ms = 0;
	const char *lrm_op_id = NULL;
	char *last_failure_key = NULL;

	if (rsc_id == NULL \|\| conf_op_xml == NULL \|\| lrm_op_xml == NULL) {
	return FALSE;
	}

	// Get name and interval from configured op
	conf_op_name = crm_element_value(conf_op_xml, "name");
	conf_op_interval_spec = crm_element_value(conf_op_xml,
	XML_LRM_ATTR_INTERVAL);
	conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);

	// Get name and interval from op history entry
	lrm_op_task = crm_element_value(lrm_op_xml, XML_LRM_ATTR_TASK);
	crm_element_value_ms(lrm_op_xml, XML_LRM_ATTR_INTERVAL_MS,
	&lrm_op_interval_ms);

	if ((conf_op_interval_ms != lrm_op_interval_ms)
	\|\| !pcmk__str_eq(conf_op_name, lrm_op_task, pcmk__str_casei)) {
	return FALSE;
	}

	lrm_op_id = ID(lrm_op_xml);
	last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);

	if (pcmk__str_eq(last_failure_key, lrm_op_id, pcmk__str_casei)) {
	matched = TRUE;

	} else {
	char *expected_op_key = pcmk__op_key(rsc_id, conf_op_name,
	conf_op_interval_ms);

	if (pcmk__str_eq(expected_op_key, lrm_op_id, pcmk__str_casei)) {
	int rc = 0;
	int target_rc = pe__target_rc_from_xml(lrm_op_xml);

	crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc);
	if (rc != target_rc) {
	matched = TRUE;
	}
	}
	free(expected_op_key);
	}

	free(last_failure_key);
	return matched;
	}

	static gboolean
	block_failure(const pe_node_t node, pe_resource_t rsc, const xmlNode *xml_op)
	{
	char *xml_name = clone_strip(rsc->id);

	/* @TODO This xpath search occurs after template expansion, but it is unable
	* to properly detect on-fail in id-ref, operation meta-attributes, or
	* op_defaults, or evaluate rules.
	*
	* Also, on-fail defaults to block (in unpack_operation()) for stop actions
	* when stonith is disabled.
	*
	* Ideally, we'd unpack the operation before this point, and pass in a
	* meta-attributes table that takes all that into consideration.
	*/
	char *xpath = crm_strdup_printf("//" XML_CIB_TAG_RESOURCE
	"[@" XML_ATTR_ID "='%s']"
	"//" XML_ATTR_OP
	"[@" XML_OP_ATTR_ON_FAIL "='block']",
	xml_name);

	xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
	gboolean should_block = FALSE;

	free(xpath);

	if (xpathObj) {
	int max = numXpathResults(xpathObj);
	int lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *pref = getXpathResult(xpathObj, lpc);

	if (xml_op) {
	should_block = is_matched_failure(xml_name, pref, xml_op);
	if (should_block) {
	break;
	}

	} else {
	const char *conf_op_name = NULL;
	const char *conf_op_interval_spec = NULL;
	guint conf_op_interval_ms = 0;
	char *lrm_op_xpath = NULL;
	xmlXPathObject *lrm_op_xpathObj = NULL;

	// Get name and interval from configured op
	conf_op_name = crm_element_value(pref, "name");
	conf_op_interval_spec = crm_element_value(pref, XML_LRM_ATTR_INTERVAL);
	conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);

	#define XPATH_FMT "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
	"//" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']" \
	"/" XML_LRM_TAG_RSC_OP "[@" XML_LRM_ATTR_TASK "='%s']" \
	"[@" XML_LRM_ATTR_INTERVAL "='%u']"

	lrm_op_xpath = crm_strdup_printf(XPATH_FMT,
	node->details->uname, xml_name,
	conf_op_name,
	conf_op_interval_ms);
	lrm_op_xpathObj = xpath_search(rsc->cluster->input, lrm_op_xpath);

	free(lrm_op_xpath);

	if (lrm_op_xpathObj) {
	int max2 = numXpathResults(lrm_op_xpathObj);
	int lpc2 = 0;

	for (lpc2 = 0; lpc2 < max2; lpc2++) {
	xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
	lpc2);

	should_block = is_matched_failure(xml_name, pref,
	lrm_op_xml);
	if (should_block) {
	break;
	}
	}
	}
	freeXpathObject(lrm_op_xpathObj);

	if (should_block) {
	break;
	}
	}
	}
	}

	free(xml_name);
	freeXpathObject(xpathObj);

	return should_block;
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(pe_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
	}

	/*!
	* \internal
	* \brief Compile regular expression to match a failure-related node attribute
	*
	* \param[in] prefix Attribute prefix to match
	* \param[in] rsc_name Resource name to match as used in failure attributes
	* \param[in] is_legacy Whether DC uses per-resource fail counts
	* \param[in] is_unique Whether the resource is a globally unique clone
	* \param[out] re Where to store resulting regular expression
	*
	* \return Standard Pacemaker return code
	* \note Fail attributes are named like PREFIX-RESOURCE#OP_INTERVAL.
	* The caller is responsible for freeing re with regfree().
	*/
	static int
	generate_fail_regex(const char prefix, const char rsc_name,
	gboolean is_legacy, gboolean is_unique, regex_t *re)
	{
	char *pattern;

	/* @COMPAT DC < 1.1.17: Fail counts used to be per-resource rather than
	* per-operation.
	*/
	const char *op_pattern = (is_legacy? "" : "#.+_[0-9]+");

	/* Ignore instance numbers for anything other than globally unique clones.
	* Anonymous clone fail counts could contain an instance number if the
	* clone was initially unique, failed, then was converted to anonymous.
	* @COMPAT Also, before 1.1.8, anonymous clone fail counts always contained
	* clone instance numbers.
	*/
	const char *instance_pattern = (is_unique? "" : "(:[0-9]+)?");

	pattern = crm_strdup_printf("^%s-%s%s%s$", prefix, rsc_name,
	instance_pattern, op_pattern);
	if (regcomp(re, pattern, REG_EXTENDED\|REG_NOSUB) != 0) {
	free(pattern);
	return EINVAL;
	}

	free(pattern);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Compile regular expressions to match failure-related node attributes
	*
	* \param[in] rsc Resource being checked for failures
	* \param[in] data_set Data set (for CRM feature set version)
	* \param[out] failcount_re Storage for regular expression for fail count
	* \param[out] lastfailure_re Storage for regular expression for last failure
	*
	* \return Standard Pacemaker return code
	* \note On success, the caller is responsible for freeing the expressions with
	* regfree().
	*/
	static int
	generate_fail_regexes(pe_resource_t rsc, pe_working_set_t data_set,
	regex_t failcount_re, regex_t lastfailure_re)
	{
	char *rsc_name = rsc_fail_name(rsc);
	const char *version = crm_element_value(data_set->input, XML_ATTR_CRM_VERSION);
	gboolean is_legacy = (compare_version(version, "3.0.13") < 0);
	int rc = pcmk_rc_ok;

	if (generate_fail_regex(PCMK__FAIL_COUNT_PREFIX, rsc_name, is_legacy,
	pcmk_is_set(rsc->flags, pe_rsc_unique),
	failcount_re) != pcmk_rc_ok) {
	rc = EINVAL;

	} else if (generate_fail_regex(PCMK__LAST_FAILURE_PREFIX, rsc_name,
	is_legacy,
	pcmk_is_set(rsc->flags, pe_rsc_unique),
	lastfailure_re) != pcmk_rc_ok) {
	rc = EINVAL;
	regfree(failcount_re);
	}

	free(rsc_name);
	return rc;
	}

	int
	pe_get_failcount(const pe_node_t node, pe_resource_t rsc,
	time_t last_failure, uint32_t flags, const xmlNode xml_op)
	{
	char *key = NULL;
	const char *value = NULL;
	regex_t failcount_re, lastfailure_re;
	int failcount = 0;
	time_t last = 0;
	GHashTableIter iter;

	CRM_CHECK(generate_fail_regexes(rsc, rsc->cluster, &failcount_re,
	&lastfailure_re) == pcmk_rc_ok,
	return 0);

	/* Resource fail count is sum of all matching operation fail counts */
	g_hash_table_iter_init(&iter, node->details->attrs);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	if (regexec(&failcount_re, key, 0, NULL, 0) == 0) {
	failcount = pcmk__add_scores(failcount, char2score(value));
	} else if (regexec(&lastfailure_re, key, 0, NULL, 0) == 0) {
	long long last_ll;

	if (pcmk__scan_ll(value, &last_ll, 0LL) == pcmk_rc_ok) {
	last = (time_t) QB_MAX(last, last_ll);
	}
	}
	}

	regfree(&failcount_re);
	regfree(&lastfailure_re);

	if ((failcount > 0) && (last > 0) && (last_failure != NULL)) {
	*last_failure = last;
	}

	/* If failure blocks the resource, disregard any failure timeout */
	if ((failcount > 0) && rsc->failure_timeout
	&& block_failure(node, rsc, xml_op)) {

	pe_warn("Ignoring failure timeout %d for %s because it conflicts with on-fail=block",
	rsc->failure_timeout, rsc->id);
	rsc->failure_timeout = 0;
	}

	/* If all failures have expired, ignore fail count */
	if (pcmk_is_set(flags, pe_fc_effective) && (failcount > 0) && (last > 0)
	&& rsc->failure_timeout) {

	time_t now = get_effective_time(rsc->cluster);

	if (now > (last + rsc->failure_timeout)) {
	crm_debug("Failcount for %s on %s expired after %ds",
	rsc->id, pe__node_name(node), rsc->failure_timeout);
	failcount = 0;
	}
	}

	/* We never want the fail counts of a bundle container's fillers to
	* count towards the container's fail count.
	*
	* Most importantly, a Pacemaker Remote connection to a bundle container
	* is a filler of the container, but can reside on a different node than the
	* container itself. Counting its fail count on its node towards the
	* container's fail count on that node could lead to attempting to stop the
	* container on the wrong node.
	*/

	if (pcmk_is_set(flags, pe_fc_fillers) && rsc->fillers
	&& !pe_rsc_is_bundled(rsc)) {

	GList *gIter = NULL;

	for (gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
	pe_resource_t filler = (pe_resource_t ) gIter->data;
	time_t filler_last_failure = 0;

	failcount += pe_get_failcount(node, filler, &filler_last_failure,
	flags, xml_op);

	if (last_failure && filler_last_failure > *last_failure) {
	*last_failure = filler_last_failure;
	}
	}

	if (failcount > 0) {
	crm_info("Container %s and the resources within it "
	"have failed %s time%s on %s",
	rsc->id, pcmk_readable_score(failcount),
	pcmk__plural_s(failcount), pe__node_name(node));
	}

	} else if (failcount > 0) {
	crm_info("%s has failed %s time%s on %s",
	rsc->id, pcmk_readable_score(failcount),
	pcmk__plural_s(failcount), pe__node_name(node));
	}

	return failcount;
	}

	/*!
	* \brief Schedule a controller operation to clear a fail count
	*
	- * \param[in] rsc Resource with failure
	- * \param[in] node Node failure occurred on
	- * \param[in] reason Readable description why needed (for logging)
	- * \param[in] data_set Working set for cluster
	+ * \param[in,out] rsc Resource with failure
	+ * \param[in] node Node failure occurred on
	+ * \param[in] reason Readable description why needed (for logging)
	+ * \param[in,out] data_set Working set for cluster
	*
	* \return Scheduled action
	*/
	pe_action_t *
	-pe__clear_failcount(pe_resource_t rsc, pe_node_t node,
	+pe__clear_failcount(pe_resource_t rsc, const pe_node_t node,
	const char reason, pe_working_set_t data_set)
	{
	char *key = NULL;
	pe_action_t *clear = NULL;

	CRM_CHECK(rsc && node && reason && data_set, return NULL);

	key = pcmk__op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
	clear = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE,
	data_set);
	add_hash_param(clear->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
	rsc->id, pe__node_name(node), reason, clear->uuid);
	return clear;
	}
	diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c
	index a003260bbf..d068681e41 100644
	--- a/lib/pengine/pe_actions.c
	+++ b/lib/pengine/pe_actions.c
	@@ -1,1683 +1,1684 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/internal.h>
	#include "pe_status_private.h"

	static void unpack_operation(pe_action_t action, xmlNode xml_obj,
	pe_resource_t *container,
	pe_working_set_t *data_set, guint interval_ms);

	static void
	add_singleton(pe_working_set_t data_set, pe_action_t action)
	{
	if (data_set->singletons == NULL) {
	data_set->singletons = pcmk__strkey_table(NULL, NULL);
	}
	g_hash_table_insert(data_set->singletons, action->uuid, action);
	}

	static pe_action_t *
	lookup_singleton(pe_working_set_t data_set, const char action_uuid)
	{
	if (data_set->singletons == NULL) {
	return NULL;
	}
	return g_hash_table_lookup(data_set->singletons, action_uuid);
	}

	/*!
	* \internal
	* \brief Find an existing action that matches arguments
	*
	* \param[in] key Action key to match
	* \param[in] rsc Resource to match (if any)
	* \param[in] node Node to match (if any)
	* \param[in] data_set Cluster working set
	*
	* \return Existing action that matches arguments (or NULL if none)
	*/
	static pe_action_t *
	find_existing_action(const char key, pe_resource_t rsc, const pe_node_t *node,
	pe_working_set_t *data_set)
	{
	GList *matches = NULL;
	pe_action_t *action = NULL;

	/* When rsc is NULL, it would be quicker to check data_set->singletons,
	* but checking all data_set->actions takes the node into account.
	*/
	matches = find_actions(((rsc == NULL)? data_set->actions : rsc->actions),
	key, node);
	if (matches == NULL) {
	return NULL;
	}
	CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));

	action = matches->data;
	g_list_free(matches);
	return action;
	}

	static xmlNode *
	find_rsc_op_entry_helper(const pe_resource_t rsc, const char key,
	gboolean include_disabled)
	{
	guint interval_ms = 0;
	gboolean do_retry = TRUE;
	char *local_key = NULL;
	const char *name = NULL;
	const char *interval_spec = NULL;
	char *match_key = NULL;
	xmlNode *op = NULL;
	xmlNode *operation = NULL;

	retry:
	for (operation = pcmk__xe_first_child(rsc->ops_xml); operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	bool enabled = false;

	name = crm_element_value(operation, "name");
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (!include_disabled && pcmk__xe_get_bool_attr(operation, "enabled", &enabled) == pcmk_rc_ok &&
	!enabled) {
	continue;
	}

	interval_ms = crm_parse_interval_spec(interval_spec);
	match_key = pcmk__op_key(rsc->id, name, interval_ms);
	if (pcmk__str_eq(key, match_key, pcmk__str_casei)) {
	op = operation;
	}
	free(match_key);

	if (rsc->clone_name) {
	match_key = pcmk__op_key(rsc->clone_name, name, interval_ms);
	if (pcmk__str_eq(key, match_key, pcmk__str_casei)) {
	op = operation;
	}
	free(match_key);
	}

	if (op != NULL) {
	free(local_key);
	return op;
	}
	}
	}

	free(local_key);
	if (do_retry == FALSE) {
	return NULL;
	}

	do_retry = FALSE;
	if (strstr(key, CRMD_ACTION_MIGRATE) \|\| strstr(key, CRMD_ACTION_MIGRATED)) {
	local_key = pcmk__op_key(rsc->id, "migrate", 0);
	key = local_key;
	goto retry;

	} else if (strstr(key, "_notify_")) {
	local_key = pcmk__op_key(rsc->id, "notify", 0);
	key = local_key;
	goto retry;
	}

	return NULL;
	}

	xmlNode *
	find_rsc_op_entry(const pe_resource_t rsc, const char key)
	{
	return find_rsc_op_entry_helper(rsc, key, FALSE);
	}

	/*!
	* \internal
	* \brief Create a new action object
	*
	* \param[in] key Action key
	* \param[in] task Action name
	* \param[in] rsc Resource that action is for (if any)
	* \param[in] node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in] for_graph Whether action should be recorded in transition graph
	* \param[in] data_set Cluster working set
	*
	* \return Newly allocated action
	* \note This function takes ownership of \p key. It is the caller's
	* responsibility to free the return value with pe_free_action().
	*/
	static pe_action_t *
	new_action(char key, const char task, pe_resource_t *rsc,
	const pe_node_t *node, bool optional, bool for_graph,
	pe_working_set_t *data_set)
	{
	pe_action_t *action = calloc(1, sizeof(pe_action_t));

	CRM_ASSERT(action != NULL);

	action->rsc = rsc;
	action->task = strdup(task); CRM_ASSERT(action->task != NULL);
	action->uuid = key;
	action->extra = pcmk__strkey_table(free, free);
	action->meta = pcmk__strkey_table(free, free);

	if (node) {
	action->node = pe__copy_node(node);
	}

	if (pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	// Resource history deletion for a node can be done on the DC
	pe__set_action_flags(action, pe_action_dc);
	}

	pe__set_action_flags(action, pe_action_runnable);
	if (optional) {
	pe__set_action_flags(action, pe_action_optional);
	} else {
	pe__clear_action_flags(action, pe_action_optional);
	}

	if (rsc != NULL) {
	guint interval_ms = 0;

	action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE);
	parse_op_key(key, NULL, NULL, &interval_ms);
	unpack_operation(action, action->op_entry, rsc->container, data_set,
	interval_ms);
	}

	if (for_graph) {
	pe_rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
	(optional? "optional" : "required"),
	data_set->action_id, key, task,
	((rsc == NULL)? "no resource" : rsc->id),
	pe__node_name(node));
	action->id = data_set->action_id++;

	data_set->actions = g_list_prepend(data_set->actions, action);
	if (rsc == NULL) {
	add_singleton(data_set, action);
	} else {
	rsc->actions = g_list_prepend(rsc->actions, action);
	}
	}
	return action;
	}

	/*!
	* \internal
	* \brief Evaluate node attribute values for an action
	*
	* \param[in] action Action to unpack attributes for
	* \param[in] data_set Cluster working set
	*/
	static void
	unpack_action_node_attributes(pe_action_t action, pe_working_set_t data_set)
	{
	if (!pcmk_is_set(action->flags, pe_action_have_node_attrs)
	&& (action->op_entry != NULL)) {

	pe_rule_eval_data_t rule_data = {
	.node_hash = action->node->details->attrs,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe__set_action_flags(action, pe_action_have_node_attrs);
	pe__unpack_dataset_nvpairs(action->op_entry, XML_TAG_ATTR_SETS,
	&rule_data, action->extra, NULL,
	FALSE, data_set);
	}
	}

	/*!
	* \internal
	* \brief Update an action's optional flag
	*
	* \param[in] action Action to update
	* \param[in] optional Requested optional status
	*/
	static void
	update_action_optional(pe_action_t *action, gboolean optional)
	{
	// Force a non-recurring action to be optional if its resource is unmanaged
	if ((action->rsc != NULL) && (action->node != NULL)
	&& !pcmk_is_set(action->flags, pe_action_pseudo)
	&& !pcmk_is_set(action->rsc->flags, pe_rsc_managed)
	&& (g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS) == NULL)) {
	pe_rsc_debug(action->rsc, "%s on %s is optional (%s is unmanaged)",
	action->uuid, pe__node_name(action->node),
	action->rsc->id);
	pe__set_action_flags(action, pe_action_optional);
	// We shouldn't clear runnable here because ... something

	// Otherwise require the action if requested
	} else if (!optional) {
	pe__clear_action_flags(action, pe_action_optional);
	}
	}

	static enum pe_quorum_policy
	effective_quorum_policy(pe_resource_t rsc, pe_working_set_t data_set)
	{
	enum pe_quorum_policy policy = data_set->no_quorum_policy;

	if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	policy = no_quorum_ignore;

	} else if (data_set->no_quorum_policy == no_quorum_demote) {
	switch (rsc->role) {
	case RSC_ROLE_PROMOTED:
	case RSC_ROLE_UNPROMOTED:
	if (rsc->next_role > RSC_ROLE_UNPROMOTED) {
	pe__set_next_role(rsc, RSC_ROLE_UNPROMOTED,
	"no-quorum-policy=demote");
	}
	policy = no_quorum_ignore;
	break;
	default:
	policy = no_quorum_stop;
	break;
	}
	}
	return policy;
	}

	/*!
	* \internal
	* \brief Update a resource action's runnable flag
	*
	* \param[in] action Action to update
	* \param[in] for_graph Whether action should be recorded in transition graph
	* \param[in] data_set Cluster working set
	*
	* \note This may also schedule fencing if a stop is unrunnable.
	*/
	static void
	update_resource_action_runnable(pe_action_t *action, bool for_graph,
	pe_working_set_t *data_set)
	{
	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	return;
	}

	if (action->node == NULL) {
	pe_rsc_trace(action->rsc, "%s is unrunnable (unallocated)",
	action->uuid);
	pe__clear_action_flags(action, pe_action_runnable);

	} else if (!pcmk_is_set(action->flags, pe_action_dc)
	&& !(action->node->details->online)
	&& (!pe__is_guest_node(action->node)
	\|\| action->node->details->remote_requires_reset)) {
	pe__clear_action_flags(action, pe_action_runnable);
	do_crm_log((for_graph? LOG_WARNING: LOG_TRACE),
	"%s on %s is unrunnable (node is offline)",
	action->uuid, pe__node_name(action->node));
	if (pcmk_is_set(action->rsc->flags, pe_rsc_managed)
	&& for_graph
	&& pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)
	&& !(action->node->details->unclean)) {
	pe_fence_node(data_set, action->node, "stop is unrunnable", false);
	}

	} else if (!pcmk_is_set(action->flags, pe_action_dc)
	&& action->node->details->pending) {
	pe__clear_action_flags(action, pe_action_runnable);
	do_crm_log((for_graph? LOG_WARNING: LOG_TRACE),
	"Action %s on %s is unrunnable (node is pending)",
	action->uuid, pe__node_name(action->node));

	} else if (action->needs == rsc_req_nothing) {
	pe_action_set_reason(action, NULL, TRUE);
	if (pe__is_guest_node(action->node)
	&& !pe_can_fence(data_set, action->node)) {
	/* An action that requires nothing usually does not require any
	* fencing in order to be runnable. However, there is an exception:
	* such an action cannot be completed if it is on a guest node whose
	* host is unclean and cannot be fenced.
	*/
	pe_rsc_debug(action->rsc, "%s on %s is unrunnable "
	"(node's host cannot be fenced)",
	action->uuid, pe__node_name(action->node));
	pe__clear_action_flags(action, pe_action_runnable);
	} else {
	pe_rsc_trace(action->rsc,
	"%s on %s does not require fencing or quorum",
	action->uuid, pe__node_name(action->node));
	pe__set_action_flags(action, pe_action_runnable);
	}

	} else {
	switch (effective_quorum_policy(action->rsc, data_set)) {
	case no_quorum_stop:
	pe_rsc_debug(action->rsc, "%s on %s is unrunnable (no quorum)",
	action->uuid, pe__node_name(action->node));
	pe__clear_action_flags(action, pe_action_runnable);
	pe_action_set_reason(action, "no quorum", true);
	break;

	case no_quorum_freeze:
	if (!action->rsc->fns->active(action->rsc, TRUE)
	\|\| (action->rsc->next_role > action->rsc->role)) {
	pe_rsc_debug(action->rsc,
	"%s on %s is unrunnable (no quorum)",
	action->uuid, pe__node_name(action->node));
	pe__clear_action_flags(action, pe_action_runnable);
	pe_action_set_reason(action, "quorum freeze", true);
	}
	break;

	default:
	//pe_action_set_reason(action, NULL, TRUE);
	pe__set_action_flags(action, pe_action_runnable);
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource object's flags for a new action on it
	*
	* \param[in] rsc Resource that action is for (if any)
	* \param[in] action New action
	*/
	static void
	update_resource_flags_for_action(pe_resource_t rsc, pe_action_t action)
	{
	/* @COMPAT pe_rsc_starting and pe_rsc_stopping are not actually used
	* within Pacemaker, and should be deprecated and eventually removed
	*/
	if (pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pe_rsc_stopping);

	} else if (pcmk__str_eq(action->task, CRMD_ACTION_START, pcmk__str_casei)) {
	if (pcmk_is_set(action->flags, pe_action_runnable)) {
	pe__set_resource_flags(rsc, pe_rsc_starting);
	} else {
	pe__clear_resource_flags(rsc, pe_rsc_starting);
	}
	}
	}

	static bool
	valid_stop_on_fail(const char *value)
	{
	return !pcmk__strcase_any_of(value, "standby", "demote", "stop", NULL);
	}

	static const char *
	unpack_operation_on_fail(pe_action_t * action)
	{
	const char *name = NULL;
	const char *role = NULL;
	const char *on_fail = NULL;
	const char *interval_spec = NULL;
	const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);

	if (pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)
	&& !valid_stop_on_fail(value)) {

	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
	"action to default value because '%s' is not "
	"allowed for stop", action->rsc->id, value);
	return NULL;

	} else if (pcmk__str_eq(action->task, CRMD_ACTION_DEMOTE, pcmk__str_casei) && !value) {
	// demote on_fail defaults to monitor value for promoted role if present
	xmlNode *operation = NULL;

	CRM_CHECK(action->rsc != NULL, return NULL);

	for (operation = pcmk__xe_first_child(action->rsc->ops_xml);
	(operation != NULL) && (value == NULL);
	operation = pcmk__xe_next(operation)) {
	bool enabled = false;

	if (!pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	continue;
	}
	name = crm_element_value(operation, "name");
	role = crm_element_value(operation, "role");
	on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (!on_fail) {
	continue;
	} else if (pcmk__xe_get_bool_attr(operation, "enabled", &enabled) == pcmk_rc_ok && !enabled) {
	continue;
	} else if (!pcmk__str_eq(name, "monitor", pcmk__str_casei)
	\|\| !pcmk__strcase_any_of(role, RSC_ROLE_PROMOTED_S,
	RSC_ROLE_PROMOTED_LEGACY_S,
	NULL)) {
	continue;
	} else if (crm_parse_interval_spec(interval_spec) == 0) {
	continue;
	} else if (pcmk__str_eq(on_fail, "demote", pcmk__str_casei)) {
	continue;
	}

	value = on_fail;
	}
	} else if (pcmk__str_eq(action->task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	value = "ignore";

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	name = crm_element_value(action->op_entry, "name");
	role = crm_element_value(action->op_entry, "role");
	interval_spec = crm_element_value(action->op_entry,
	XML_LRM_ATTR_INTERVAL);

	if (!pcmk__str_eq(name, CRMD_ACTION_PROMOTE, pcmk__str_casei)
	&& (!pcmk__str_eq(name, CRMD_ACTION_STATUS, pcmk__str_casei)
	\|\| !pcmk__strcase_any_of(role, RSC_ROLE_PROMOTED_S,
	RSC_ROLE_PROMOTED_LEGACY_S, NULL)
	\|\| (crm_parse_interval_spec(interval_spec) == 0))) {
	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s "
	"action to default value because 'demote' is not "
	"allowed for it", action->rsc->id, name);
	return NULL;
	}
	}

	return value;
	}

	static int
	unpack_timeout(const char *value)
	{
	int timeout_ms = crm_get_msec(value);

	if (timeout_ms < 0) {
	timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
	}
	return timeout_ms;
	}

	// true if value contains valid, non-NULL interval origin for recurring op
	static bool
	unpack_interval_origin(const char value, xmlNode xml_obj, guint interval_ms,
	crm_time_t now, long long start_delay)
	{
	long long result = 0;
	guint interval_sec = interval_ms / 1000;
	crm_time_t *origin = NULL;

	// Ignore unspecified values and non-recurring operations
	if ((value == NULL) \|\| (interval_ms == 0) \|\| (now == NULL)) {
	return false;
	}

	// Parse interval origin from text
	origin = crm_time_new(value);
	if (origin == NULL) {
	pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
	"'%s' because '%s' is not valid",
	(ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
	return false;
	}

	// Get seconds since origin (negative if origin is in the future)
	result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
	crm_time_free(origin);

	// Calculate seconds from closest interval to now
	result = result % interval_sec;

	// Calculate seconds remaining until next interval
	result = ((result <= 0)? 0 : interval_sec) - result;
	crm_info("Calculated a start delay of %llds for operation '%s'",
	result,
	(ID(xml_obj)? ID(xml_obj) : "(unspecified)"));

	if (start_delay != NULL) {
	start_delay = result 1000; // milliseconds
	}
	return true;
	}

	static int
	unpack_start_delay(const char value, GHashTable meta)
	{
	int start_delay = 0;

	if (value != NULL) {
	start_delay = crm_get_msec(value);

	if (start_delay < 0) {
	start_delay = 0;
	}

	if (meta) {
	g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY),
	pcmk__itoa(start_delay));
	}
	}

	return start_delay;
	}

	static xmlNode *
	find_min_interval_mon(pe_resource_t * rsc, gboolean include_disabled)
	{
	guint interval_ms = 0;
	guint min_interval_ms = G_MAXUINT;
	const char *name = NULL;
	const char *interval_spec = NULL;
	xmlNode *op = NULL;
	xmlNode *operation = NULL;

	for (operation = pcmk__xe_first_child(rsc->ops_xml);
	operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	bool enabled = false;

	name = crm_element_value(operation, "name");
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (!include_disabled && pcmk__xe_get_bool_attr(operation, "enabled", &enabled) == pcmk_rc_ok &&
	!enabled) {
	continue;
	}

	if (!pcmk__str_eq(name, RSC_STATUS, pcmk__str_casei)) {
	continue;
	}

	interval_ms = crm_parse_interval_spec(interval_spec);

	if (interval_ms && (interval_ms < min_interval_ms)) {
	min_interval_ms = interval_ms;
	op = operation;
	}
	}
	}

	return op;
	}

	/*!
	* \brief Unpack operation XML into an action structure
	*
	* Unpack an operation's meta-attributes (normalizing the interval, timeout,
	* and start delay values as integer milliseconds), requirements, and
	* failure policy.
	*
	* \param[in,out] action Action to unpack into
	* \param[in] xml_obj Operation XML (or NULL if all defaults)
	* \param[in] container Resource that contains affected resource, if any
	* \param[in] data_set Cluster state
	* \param[in] interval_ms How frequently to perform the operation
	*/
	static void
	unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
	pe_working_set_t * data_set, guint interval_ms)
	{
	int timeout_ms = 0;
	const char *value = NULL;
	bool is_probe = false;

	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(action->rsc->xml, XML_EXPR_ATTR_TYPE)
	};

	pe_op_eval_data_t op_rule_data = {
	.op_name = action->task,
	.interval = interval_ms
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = &op_rule_data
	};

	CRM_CHECK(action && action->rsc, return);

	is_probe = pcmk_is_probe(action->task, interval_ms);

	// Cluster-wide <op_defaults> <meta_attributes>
	pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, &rule_data,
	action->meta, NULL, FALSE, data_set);

	// Determine probe default timeout differently
	if (is_probe) {
	xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE);

	if (min_interval_mon) {
	value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT);
	if (value) {
	crm_trace("\t%s: Setting default timeout to minimum-interval "
	"monitor's timeout '%s'", action->uuid, value);
	g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
	strdup(value));
	}
	}
	}

	if (xml_obj) {
	xmlAttrPtr xIter = NULL;

	// <op> <meta_attributes> take precedence over defaults
	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_META_SETS, &rule_data,
	action->meta, NULL, TRUE, data_set);

	/* Anything set as an <op> XML property has highest precedence.
	* This ensures we use the name and interval from the <op> tag.
	*/
	for (xIter = xml_obj->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = crm_element_value(xml_obj, prop_name);

	g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value));
	}
	}

	g_hash_table_remove(action->meta, "id");

	// Normalize interval to milliseconds
	if (interval_ms > 0) {
	g_hash_table_replace(action->meta, strdup(XML_LRM_ATTR_INTERVAL),
	crm_strdup_printf("%u", interval_ms));
	} else {
	g_hash_table_remove(action->meta, XML_LRM_ATTR_INTERVAL);
	}

	/*
	* Timeout order of precedence:
	* 1. pcmk_monitor_timeout (if rsc has pcmk_ra_cap_fence_params
	* and task is start or a probe; pcmk_monitor_timeout works
	* by default for a recurring monitor)
	* 2. explicit op timeout on the primitive
	* 3. default op timeout
	* a. if probe, then min-interval monitor's timeout
	* b. else, in XML_CIB_TAG_OPCONFIG
	* 4. CRM_DEFAULT_OP_TIMEOUT_S
	*
	* #1 overrides general rule of <op> XML property having highest
	* precedence.
	*/
	if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
	pcmk_ra_cap_fence_params)
	&& (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
	\|\| is_probe)) {

	GHashTable *params = pe_rsc_params(action->rsc, action->node, data_set);

	value = g_hash_table_lookup(params, "pcmk_monitor_timeout");

	if (value) {
	crm_trace("\t%s: Setting timeout to pcmk_monitor_timeout '%s', "
	"overriding default", action->uuid, value);
	g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
	strdup(value));
	}
	}

	// Normalize timeout to positive milliseconds
	value = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
	timeout_ms = unpack_timeout(value);
	g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
	pcmk__itoa(timeout_ms));

	if (!pcmk__strcase_any_of(action->task, RSC_START, RSC_PROMOTE, NULL)) {
	action->needs = rsc_req_nothing;
	value = "nothing (not start or promote)";

	} else if (pcmk_is_set(action->rsc->flags, pe_rsc_needs_fencing)) {
	action->needs = rsc_req_stonith;
	value = "fencing";

	} else if (pcmk_is_set(action->rsc->flags, pe_rsc_needs_quorum)) {
	action->needs = rsc_req_quorum;
	value = "quorum";

	} else {
	action->needs = rsc_req_nothing;
	value = "nothing";
	}
	pe_rsc_trace(action->rsc, "%s requires %s", action->uuid, value);

	value = unpack_operation_on_fail(action);

	if (value == NULL) {

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	action->on_fail = action_fail_block;
	g_hash_table_insert(action->meta, strdup(XML_OP_ATTR_ON_FAIL), strdup("block"));
	value = "block"; // The above could destroy the original string

	} else if (pcmk__str_eq(value, "fence", pcmk__str_casei)) {
	action->on_fail = action_fail_fence;
	value = "node fencing";

	if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
	"operation '%s' to 'stop' because 'fence' is not "
	"valid when fencing is disabled", action->uuid);
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop resource";
	}

	} else if (pcmk__str_eq(value, "standby", pcmk__str_casei)) {
	action->on_fail = action_fail_standby;
	value = "node standby";

	} else if (pcmk__strcase_any_of(value, "ignore", PCMK__VALUE_NOTHING,
	NULL)) {
	action->on_fail = action_fail_ignore;
	value = "ignore";

	} else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
	action->on_fail = action_fail_migrate;
	value = "force migration";

	} else if (pcmk__str_eq(value, "stop", pcmk__str_casei)) {
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop resource";

	} else if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	action->on_fail = action_fail_recover;
	value = "restart (and possibly migrate)";

	} else if (pcmk__str_eq(value, "restart-container", pcmk__str_casei)) {
	if (container) {
	action->on_fail = action_fail_restart_container;
	value = "restart container (and possibly migrate)";

	} else {
	value = NULL;
	}

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	action->on_fail = action_fail_demote;
	value = "demote instance";

	} else {
	pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
	value = NULL;
	}

	/* defaults */
	if (value == NULL && container) {
	action->on_fail = action_fail_restart_container;
	value = "restart container (and possibly migrate) (default)";

	/* For remote nodes, ensure that any failure that results in dropping an
	* active connection to the node results in fencing of the node.
	*
	* There are only two action failures that don't result in fencing.
	* 1. probes - probe failures are expected.
	* 2. start - a start failure indicates that an active connection does not already
	* exist. The user can set op on-fail=fence if they really want to fence start
	* failures. */
	} else if (((value == NULL) \|\| !pcmk_is_set(action->rsc->flags, pe_rsc_managed))
	&& pe__resource_is_remote_conn(action->rsc, data_set)
	&& !(pcmk__str_eq(action->task, CRMD_ACTION_STATUS, pcmk__str_casei)
	&& (interval_ms == 0))
	&& !pcmk__str_eq(action->task, CRMD_ACTION_START, pcmk__str_casei)) {

	if (!pcmk_is_set(action->rsc->flags, pe_rsc_managed)) {
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop unmanaged remote node (enforcing default)";

	} else {
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	value = "fence remote node (default)";
	} else {
	value = "recover remote node connection (default)";
	}

	if (action->rsc->remote_reconnect_ms) {
	action->fail_role = RSC_ROLE_STOPPED;
	}
	action->on_fail = action_fail_reset_remote;
	}

	} else if (value == NULL && pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)) {
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	action->on_fail = action_fail_fence;
	value = "resource fence (default)";

	} else {
	action->on_fail = action_fail_block;
	value = "resource block (default)";
	}

	} else if (value == NULL) {
	action->on_fail = action_fail_recover;
	value = "restart (and possibly migrate) (default)";
	}

	pe_rsc_trace(action->rsc, "%s failure handling: %s",
	action->uuid, value);

	value = NULL;
	if (xml_obj != NULL) {
	value = g_hash_table_lookup(action->meta, "role_after_failure");
	if (value) {
	pe_warn_once(pe_wo_role_after,
	"Support for role_after_failure is deprecated and will be removed in a future release");
	}
	}
	if (value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) {
	action->fail_role = text2role(value);
	}
	/* defaults */
	if (action->fail_role == RSC_ROLE_UNKNOWN) {
	if (pcmk__str_eq(action->task, CRMD_ACTION_PROMOTE, pcmk__str_casei)) {
	action->fail_role = RSC_ROLE_UNPROMOTED;
	} else {
	action->fail_role = RSC_ROLE_STARTED;
	}
	}
	pe_rsc_trace(action->rsc, "%s failure results in: %s",
	action->uuid, role2text(action->fail_role));

	value = g_hash_table_lookup(action->meta, XML_OP_ATTR_START_DELAY);
	if (value) {
	unpack_start_delay(value, action->meta);
	} else {
	long long start_delay = 0;

	value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN);
	if (unpack_interval_origin(value, xml_obj, interval_ms, data_set->now,
	&start_delay)) {
	g_hash_table_replace(action->meta, strdup(XML_OP_ATTR_START_DELAY),
	crm_strdup_printf("%lld", start_delay));
	}
	}
	}

	/*!
	* \brief Create or update an action object
	*
	* \param[in] rsc Resource that action is for (if any)
	* \param[in] key Action key (must be non-NULL)
	* \param[in] task Action name (must be non-NULL)
	* \param[in] on_node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in] save_action Whether action should be recorded in transition graph
	* \param[in] data_set Cluster working set
	*
	* \return Action object corresponding to arguments
	* \note This function takes ownership of (and might free) \p key. If
	* \p save_action is true, \p data_set will own the returned action,
	* otherwise it is the caller's responsibility to free the return value
	* with pe_free_action().
	*/
	pe_action_t *
	custom_action(pe_resource_t rsc, char key, const char *task,
	const pe_node_t *on_node, gboolean optional, gboolean save_action,
	pe_working_set_t *data_set)
	{
	pe_action_t *action = NULL;

	CRM_ASSERT((key != NULL) && (task != NULL) && (data_set != NULL));

	if (save_action) {
	action = find_existing_action(key, rsc, on_node, data_set);
	}

	if (action == NULL) {
	action = new_action(key, task, rsc, on_node, optional, save_action,
	data_set);
	} else {
	free(key);
	}

	update_action_optional(action, optional);

	if (rsc != NULL) {
	if (action->node != NULL) {
	unpack_action_node_attributes(action, data_set);
	}

	update_resource_action_runnable(action, save_action, data_set);

	if (save_action) {
	update_resource_flags_for_action(rsc, action);
	}
	}

	return action;
	}

	pe_action_t *
	get_pseudo_op(const char name, pe_working_set_t data_set)
	{
	pe_action_t *op = lookup_singleton(data_set, name);

	if (op == NULL) {
	op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);
	}
	return op;
	}

	static GList *
	find_unfencing_devices(GList candidates, GList matches)
	{
	for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *candidate = gIter->data;

	if (candidate->children != NULL) {
	matches = find_unfencing_devices(candidate->children, matches);

	} else if (!pcmk_is_set(candidate->flags, pe_rsc_fence_device)) {
	continue;

	} else if (pcmk_is_set(candidate->flags, pe_rsc_needs_unfencing)) {
	matches = g_list_prepend(matches, candidate);

	} else if (pcmk__str_eq(g_hash_table_lookup(candidate->meta,
	PCMK_STONITH_PROVIDES),
	PCMK__VALUE_UNFENCING,
	pcmk__str_casei)) {
	matches = g_list_prepend(matches, candidate);
	}
	}
	return matches;
	}

	static int
	-node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set)
	+node_priority_fencing_delay(const pe_node_t *node,
	+ const pe_working_set_t *data_set)
	{
	int member_count = 0;
	int online_count = 0;
	int top_priority = 0;
	int lowest_priority = 0;
	GList *gIter = NULL;

	// `priority-fencing-delay` is disabled
	if (data_set->priority_fencing_delay <= 0) {
	return 0;
	}

	/* No need to request a delay if the fencing target is not a normal cluster
	* member, for example if it's a remote node or a guest node. */
	if (node->details->type != node_member) {
	return 0;
	}

	// No need to request a delay if the fencing target is in our partition
	if (node->details->online) {
	return 0;
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t *n = gIter->data;

	if (n->details->type != node_member) {
	continue;
	}

	member_count ++;

	if (n->details->online) {
	online_count++;
	}

	if (member_count == 1
	\|\| n->details->priority > top_priority) {
	top_priority = n->details->priority;
	}

	if (member_count == 1
	\|\| n->details->priority < lowest_priority) {
	lowest_priority = n->details->priority;
	}
	}

	// No need to delay if we have more than half of the cluster members
	if (online_count > member_count / 2) {
	return 0;
	}

	/* All the nodes have equal priority.
	* Any configured corresponding `pcmk_delay_base/max` will be applied. */
	if (lowest_priority == top_priority) {
	return 0;
	}

	if (node->details->priority < top_priority) {
	return 0;
	}

	return data_set->priority_fencing_delay;
	}

	pe_action_t *
	-pe_fence_op(pe_node_t * node, const char op, bool optional, const char reason,
	- bool priority_delay, pe_working_set_t * data_set)
	+pe_fence_op(const pe_node_t node, const char op, bool optional,
	+ const char reason, bool priority_delay, pe_working_set_t data_set)
	{
	char *op_key = NULL;
	pe_action_t *stonith_op = NULL;

	if(op == NULL) {
	op = data_set->stonith_action;
	}

	op_key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);

	stonith_op = lookup_singleton(data_set, op_key);
	if(stonith_op == NULL) {
	stonith_op = custom_action(NULL, op_key, CRM_OP_FENCE, node, TRUE, TRUE, data_set);

	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
	add_hash_param(stonith_op->meta, "stonith_action", op);

	if (pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)) {
	/* Extra work to detect device changes
	*/
	GString *digests_all = g_string_sized_new(1024);
	GString *digests_secure = g_string_sized_new(1024);

	GList *matches = find_unfencing_devices(data_set->resources, NULL);

	char *key = NULL;
	char *value = NULL;

	for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *match = gIter->data;
	const char *agent = g_hash_table_lookup(match->meta,
	XML_ATTR_TYPE);
	op_digest_cache_t *data = NULL;

	data = pe__compare_fencing_digest(match, agent, node, data_set);
	if(data->rc == RSC_DIGEST_ALL) {
	optional = FALSE;
	crm_notice("Unfencing Pacemaker Remote node %s "
	"because the definition of %s changed",
	pe__node_name(node), match->id);
	if (!pcmk__is_daemon && data_set->priv != NULL) {
	pcmk__output_t *out = data_set->priv;
	out->info(out,
	"notice: Unfencing Pacemaker Remote node %s "
	"because the definition of %s changed",
	pe__node_name(node), match->id);
	}
	}

	pcmk__g_strcat(digests_all,
	match->id, ":", agent, ":",
	data->digest_all_calc, ",", NULL);
	pcmk__g_strcat(digests_secure,
	match->id, ":", agent, ":",
	data->digest_secure_calc, ",", NULL);
	}
	key = strdup(XML_OP_ATTR_DIGESTS_ALL);
	value = strdup((const char *) digests_all->str);
	CRM_ASSERT((key != NULL) && (value != NULL));
	g_hash_table_insert(stonith_op->meta, key, value);
	g_string_free(digests_all, TRUE);

	key = strdup(XML_OP_ATTR_DIGESTS_SECURE);
	value = strdup((const char *) digests_secure->str);
	CRM_ASSERT((key != NULL) && (value != NULL));
	g_hash_table_insert(stonith_op->meta, key, value);
	g_string_free(digests_secure, TRUE);
	}

	} else {
	free(op_key);
	}

	if (data_set->priority_fencing_delay > 0

	/* It's a suitable case where `priority-fencing-delay` applies.
	* At least add `priority-fencing-delay` field as an indicator. */
	&& (priority_delay

	/* The priority delay needs to be recalculated if this function has
	* been called by schedule_fencing_and_shutdowns() after node
	* priority has already been calculated by native_add_running().
	*/
	\|\| g_hash_table_lookup(stonith_op->meta,
	XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) {

	/* Add `priority-fencing-delay` to the fencing op even if it's 0 for
	* the targeting node. So that it takes precedence over any possible
	* `pcmk_delay_base/max`.
	*/
	char *delay_s = pcmk__itoa(node_priority_fencing_delay(node, data_set));

	g_hash_table_insert(stonith_op->meta,
	strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY),
	delay_s);
	}

	if(optional == FALSE && pe_can_fence(data_set, node)) {
	pe__clear_action_flags(stonith_op, pe_action_optional);
	pe_action_set_reason(stonith_op, reason, false);

	} else if(reason && stonith_op->reason == NULL) {
	stonith_op->reason = strdup(reason);
	}

	return stonith_op;
	}

	void
	pe_free_action(pe_action_t * action)
	{
	if (action == NULL) {
	return;
	}
	g_list_free_full(action->actions_before, free); /* pe_action_wrapper_t* */
	g_list_free_full(action->actions_after, free); /* pe_action_wrapper_t* */
	if (action->extra) {
	g_hash_table_destroy(action->extra);
	}
	if (action->meta) {
	g_hash_table_destroy(action->meta);
	}
	free(action->cancel_task);
	free(action->reason);
	free(action->task);
	free(action->uuid);
	free(action->node);
	free(action);
	}

	int
	pe_get_configured_timeout(pe_resource_t rsc, const char action, pe_working_set_t *data_set)
	{
	xmlNode *child = NULL;
	GHashTable *action_meta = NULL;
	const char *timeout_spec = NULL;
	int timeout_ms = 0;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	child != NULL; child = crm_next_same_xml(child)) {
	if (pcmk__str_eq(action, crm_element_value(child, XML_NVPAIR_ATTR_NAME),
	pcmk__str_casei)) {
	timeout_spec = crm_element_value(child, XML_ATTR_TIMEOUT);
	break;
	}
	}

	if (timeout_spec == NULL && data_set->op_defaults) {
	action_meta = pcmk__strkey_table(free, free);
	pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS,
	&rule_data, action_meta, NULL, FALSE, data_set);
	timeout_spec = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
	}

	// @TODO check meta-attributes
	// @TODO maybe use min-interval monitor timeout as default for monitors

	timeout_ms = crm_get_msec(timeout_spec);
	if (timeout_ms < 0) {
	timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
	}

	if (action_meta != NULL) {
	g_hash_table_destroy(action_meta);
	}
	return timeout_ms;
	}

	enum action_tasks
	get_complex_task(pe_resource_t * rsc, const char *name, gboolean allow_non_atomic)
	{
	enum action_tasks task = text2task(name);

	if (rsc == NULL) {
	return task;

	} else if (allow_non_atomic == FALSE \|\| rsc->variant == pe_native) {
	switch (task) {
	case stopped_rsc:
	case started_rsc:
	case action_demoted:
	case action_promoted:
	crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id);
	return task - 1;
	default:
	break;
	}
	}
	return task;
	}

	/*!
	* \internal
	* \brief Find first matching action in a list
	*
	* \param[in] input List of actions to search
	* \param[in] uuid If not NULL, action must have this UUID
	* \param[in] task If not NULL, action must have this action name
	* \param[in] on_node If not NULL, action must be on this node
	*
	* \return First action in list that matches criteria, or NULL if none
	*/
	pe_action_t *
	find_first_action(const GList input, const char uuid, const char *task,
	const pe_node_t *on_node)
	{
	CRM_CHECK(uuid \|\| task, return NULL);

	for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	return action;

	} else if (action->node == NULL) {
	continue;

	} else if (on_node->details == action->node->details) {
	return action;
	}
	}

	return NULL;
	}

	GList *
	find_actions(GList input, const char key, const pe_node_t *on_node)
	{
	GList *gIter = input;
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	crm_trace("Action %s matches (ignoring node)", key);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	crm_trace("Action %s matches (unallocated, assigning to %s)",
	key, pe__node_name(on_node));

	action->node = pe__copy_node(on_node);
	result = g_list_prepend(result, action);

	} else if (on_node->details == action->node->details) {
	crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	GList *
	find_actions_exact(GList input, const char key, const pe_node_t *on_node)
	{
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	if (on_node == NULL) {
	return NULL;
	}

	for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if ((action->node != NULL)
	&& pcmk__str_eq(key, action->uuid, pcmk__str_casei)
	&& pcmk__str_eq(on_node->details->id, action->node->details->id,
	pcmk__str_casei)) {

	crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	/*!
	* \brief Find all actions of given type for a resource
	*
	* \param[in] rsc Resource to search
	* \param[in] node Find only actions scheduled on this node
	* \param[in] task Action name to search for
	* \param[in] require_node If TRUE, NULL node or action node will not match
	*
	* \return List of actions found (or NULL if none)
	* \note If node is not NULL and require_node is FALSE, matching actions
	* without a node will be assigned to node.
	*/
	GList *
	pe__resource_actions(const pe_resource_t rsc, const pe_node_t node,
	const char *task, bool require_node)
	{
	GList *result = NULL;
	char *key = pcmk__op_key(rsc->id, task, 0);

	if (require_node) {
	result = find_actions_exact(rsc->actions, key, node);
	} else {
	result = find_actions(rsc->actions, key, node);
	}
	free(key);
	return result;
	}

	/*!
	* \internal
	* \brief Create an action reason string based on the action itself
	*
	* \param[in] action Action to create reason string for
	* \param[in] flag Action flag that was cleared
	*
	* \return Newly allocated string suitable for use as action reason
	* \note It is the caller's responsibility to free() the result.
	*/
	char *
	pe__action2reason(pe_action_t *action, enum pe_action_flags flag)
	{
	const char *change = NULL;

	switch (flag) {
	case pe_action_runnable:
	case pe_action_migrate_runnable:
	change = "unrunnable";
	break;
	case pe_action_optional:
	change = "required";
	break;
	default:
	// Bug: caller passed unsupported flag
	CRM_CHECK(change != NULL, change = "");
	break;
	}
	return crm_strdup_printf("%s%s%s %s", change,
	(action->rsc == NULL)? "" : " ",
	(action->rsc == NULL)? "" : action->rsc->id,
	action->task);
	}

	void pe_action_set_reason(pe_action_t action, const char reason, bool overwrite)
	{
	if (action->reason != NULL && overwrite) {
	pe_rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
	action->uuid, action->reason, pcmk__s(reason, "(none)"));
	} else if (action->reason == NULL) {
	pe_rsc_trace(action->rsc, "Set %s reason to '%s'",
	action->uuid, pcmk__s(reason, "(none)"));
	} else {
	// crm_assert(action->reason != NULL && !overwrite);
	return;
	}

	pcmk__str_update(&action->reason, reason);
	}

	/*!
	* \internal
	* \brief Create an action to clear a resource's history from CIB
	*
	* \param[in] rsc Resource to clear
	* \param[in] node Node to clear history on
	*
	* \return New action to clear resource history
	*/
	pe_action_t *
	pe__clear_resource_history(pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set)
	{
	char *key = NULL;

	CRM_ASSERT(rsc && node);
	key = pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0);
	return custom_action(rsc, key, CRM_OP_LRM_DELETE, node, FALSE, TRUE,
	data_set);
	}

	#define sort_return(an_int, why) do { \
	free(a_uuid); \
	free(b_uuid); \
	crm_trace("%s (%d) %c %s (%d) : %s", \
	a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
	b_xml_id, b_call_id, why); \
	return an_int; \
	} while(0)

	int
	pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b,
	bool same_node_default)
	{
	int a_call_id = -1;
	int b_call_id = -1;

	char *a_uuid = NULL;
	char *b_uuid = NULL;

	const char *a_xml_id = crm_element_value(xml_a, XML_ATTR_ID);
	const char *b_xml_id = crm_element_value(xml_b, XML_ATTR_ID);

	const char *a_node = crm_element_value(xml_a, XML_LRM_ATTR_TARGET);
	const char *b_node = crm_element_value(xml_b, XML_LRM_ATTR_TARGET);
	bool same_node = true;

	/* @COMPAT The on_node attribute was added to last_failure as of 1.1.13 (via
	* 8b3ca1c) and the other entries as of 1.1.12 (via 0b07b5c).
	*
	* In case that any of the lrm_rsc_op entries doesn't have on_node
	* attribute, we need to explicitly tell whether the two operations are on
	* the same node.
	*/
	if (a_node == NULL \|\| b_node == NULL) {
	same_node = same_node_default;

	} else {
	same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei);
	}

	if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) {
	/* We have duplicate lrm_rsc_op entries in the status
	* section which is unlikely to be a good thing
	* - we can handle it easily enough, but we need to get
	* to the bottom of why it's happening.
	*/
	pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
	sort_return(0, "duplicate");
	}

	crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
	crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);

	if (a_call_id == -1 && b_call_id == -1) {
	/* both are pending ops so it doesn't matter since
	* stops are never pending
	*/
	sort_return(0, "pending");

	} else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) {
	sort_return(-1, "call id");

	} else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) {
	sort_return(1, "call id");

	} else if (a_call_id >= 0 && b_call_id >= 0
	&& (!same_node \|\| a_call_id == b_call_id)) {
	/*
	* The op and last_failed_op are the same
	* Order on last-rc-change
	*/
	time_t last_a = -1;
	time_t last_b = -1;

	crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
	crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);

	crm_trace("rc-change: %lld vs %lld",
	(long long) last_a, (long long) last_b);
	if (last_a >= 0 && last_a < last_b) {
	sort_return(-1, "rc-change");

	} else if (last_b >= 0 && last_a > last_b) {
	sort_return(1, "rc-change");
	}
	sort_return(0, "rc-change");

	} else {
	/* One of the inputs is a pending operation
	* Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
	*/

	int a_id = -1;
	int b_id = -1;

	const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
	const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);

	CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
	if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic a");
	}
	if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic b");
	}
	/* try to determine the relative age of the operation...
	* some pending operations (e.g. a start) may have been superseded
	* by a subsequent stop
	*
	* [a\|b]_id == -1 means it's a shutdown operation and _always_ comes last
	*/
	if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) \|\| a_id == b_id) {
	/*
	* some of the logic in here may be redundant...
	*
	* if the UUID from the TE doesn't match then one better
	* be a pending operation.
	* pending operations don't survive between elections and joins
	* because we query the LRM directly
	*/

	if (b_call_id == -1) {
	sort_return(-1, "transition + call");

	} else if (a_call_id == -1) {
	sort_return(1, "transition + call");
	}

	} else if ((a_id >= 0 && a_id < b_id) \|\| b_id == -1) {
	sort_return(-1, "transition");

	} else if ((b_id >= 0 && a_id > b_id) \|\| a_id == -1) {
	sort_return(1, "transition");
	}
	}

	/* we should never end up here */
	CRM_CHECK(FALSE, sort_return(0, "default"));
	}

	gint
	sort_op_by_callid(gconstpointer a, gconstpointer b)
	{
	const xmlNode *xml_a = a;
	const xmlNode *xml_b = b;

	return pe__is_newer_op(xml_a, xml_b, true);
	}

	/*!
	* \internal
	* \brief Create a new pseudo-action for a resource
	*
	* \param[in] rsc Resource to create action for
	* \param[in] task Action name
	* \param[in] optional Whether action should be considered optional
	* \param[in] runnable Whethe action should be considered runnable
	*
	* \return New action object corresponding to arguments
	*/
	pe_action_t *
	pe__new_rsc_pseudo_action(pe_resource_t rsc, const char task, bool optional,
	bool runnable)
	{
	pe_action_t *action = NULL;

	CRM_ASSERT((rsc != NULL) && (task != NULL));

	action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL,
	optional, TRUE, rsc->cluster);
	pe__set_action_flags(action, pe_action_pseudo);
	if (runnable) {
	pe__set_action_flags(action, pe_action_runnable);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Add the expected result to an action
	*
	* \param[in] action Action to add expected result to
	* \param[in] expected_result Expected result to add
	*
	* \note This is more efficient than calling add_hash_param().
	*/
	void
	pe__add_action_expected_result(pe_action_t *action, int expected_result)
	{
	char *name = NULL;

	CRM_ASSERT((action != NULL) && (action->meta != NULL));

	name = strdup(XML_ATTR_TE_TARGET_RC);
	CRM_ASSERT (name != NULL);

	g_hash_table_insert(action->meta, name, pcmk__itoa(expected_result));
	}
	diff --git a/lib/pengine/pe_digest.c b/lib/pengine/pe_digest.c
	index 16868865cd..dfc2dacfce 100644
	--- a/lib/pengine/pe_digest.c
	+++ b/lib/pengine/pe_digest.c
	@@ -1,591 +1,592 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/pengine/internal.h>
	#include "pe_status_private.h"

	extern bool pcmk__is_daemon;

	/*!
	* \internal
	* \brief Free an operation digest cache entry
	*
	- * \param[in] ptr Pointer to cache entry to free
	+ * \param[in,out] ptr Pointer to cache entry to free
	*
	* \note The argument is a gpointer so this can be used as a hash table
	* free function.
	*/
	void
	pe__free_digests(gpointer ptr)
	{
	op_digest_cache_t *data = ptr;

	if (data != NULL) {
	free_xml(data->params_all);
	free_xml(data->params_secure);
	free_xml(data->params_restart);

	free(data->digest_all_calc);
	free(data->digest_restart_calc);
	free(data->digest_secure_calc);

	free(data);
	}
	}

	// Return true if XML attribute name is not substring of a given string
	static bool
	attr_not_in_string(xmlAttrPtr a, void *user_data)
	{
	bool filter = false;
	char name = crm_strdup_printf(" %s ", (const char ) a->name);

	if (strstr((const char *) user_data, name) == NULL) {
	crm_trace("Filtering %s (not found in '%s')",
	(const char ) a->name, (const char ) user_data);
	filter = true;
	}
	free(name);
	return filter;
	}

	// Return true if XML attribute name is substring of a given string
	static bool
	attr_in_string(xmlAttrPtr a, void *user_data)
	{
	bool filter = false;
	char name = crm_strdup_printf(" %s ", (const char ) a->name);

	if (strstr((const char *) user_data, name) != NULL) {
	crm_trace("Filtering %s (found in '%s')",
	(const char ) a->name, (const char ) user_data);
	filter = true;
	}
	free(name);
	return filter;
	}

	/*!
	* \internal
	* \brief Add digest of all parameters to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	- * \param[in] rsc Resource that action was for
	+ * \param[in,out] rsc Resource that action was for
	* \param[in] node Node action was performed on
	* \param[in] params Resource parameters evaluated for node
	* \param[in] task Name of action performed
	* \param[in,out] interval_ms Action's interval (will be reset if in overrides)
	- * \param[in] xml_op XML of operation in CIB status (if available)
	+ * \param[in] xml_op Unused
	* \param[in] op_version CRM feature set to use for digest calculation
	* \param[in] overrides Key/value table to override resource parameters
	- * \param[in] data_set Cluster working set
	+ * \param[in,out] data_set Cluster working set
	*/
	static void
	calculate_main_digest(op_digest_cache_t data, pe_resource_t rsc,
	- pe_node_t node, GHashTable params,
	+ const pe_node_t node, GHashTable params,
	const char task, guint interval_ms,
	- xmlNode xml_op, const char op_version,
	+ const xmlNode xml_op, const char op_version,
	GHashTable overrides, pe_working_set_t data_set)
	{
	pe_action_t *action = NULL;

	data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);

	/* REMOTE_CONTAINER_HACK: Allow Pacemaker Remote nodes to run containers
	* that themselves are Pacemaker Remote nodes
	*/
	(void) pe__add_bundle_remote_name(rsc, data_set, data->params_all,
	XML_RSC_ATTR_REMOTE_RA_ADDR);

	// If interval was overridden, reset it
	if (overrides != NULL) {
	const char *interval_s = g_hash_table_lookup(overrides, CRM_META "_"
	XML_LRM_ATTR_INTERVAL);

	if (interval_s != NULL) {
	long long value_ll;

	if ((pcmk__scan_ll(interval_s, &value_ll, 0LL) == pcmk_rc_ok)
	&& (value_ll >= 0) && (value_ll <= G_MAXUINT)) {
	*interval_ms = (guint) value_ll;
	}
	}
	}

	action = custom_action(rsc, pcmk__op_key(rsc->id, task, *interval_ms),
	task, node, TRUE, FALSE, data_set);
	if (overrides != NULL) {
	g_hash_table_foreach(overrides, hash2field, data->params_all);
	}
	g_hash_table_foreach(params, hash2field, data->params_all);
	g_hash_table_foreach(action->extra, hash2field, data->params_all);
	g_hash_table_foreach(action->meta, hash2metafield, data->params_all);

	pcmk__filter_op_for_digest(data->params_all);

	/* Given a non-recurring operation with extra parameters configured,
	* in case that the main digest doesn't match, even if the restart
	* digest matches, enforce a restart rather than a reload-agent anyway.
	* So that it ensures any changes of the extra parameters get applied
	* for this specific operation, and the digests calculated for the
	* resulting lrm_rsc_op will be correct.
	* Mark the implied rc RSC_DIGEST_RESTART for the case that the main
	* digest doesn't match.
	*/
	if (*interval_ms == 0
	&& g_hash_table_size(action->extra) > 0) {
	data->rc = RSC_DIGEST_RESTART;
	}

	pe_free_action(action);

	data->digest_all_calc = calculate_operation_digest(data->params_all,
	op_version);
	}

	// Return true if XML attribute name is a Pacemaker-defined fencing parameter
	static bool
	is_fence_param(xmlAttrPtr attr, void *user_data)
	{
	return pcmk_stonith_param((const char *) attr->name);
	}

	/*!
	* \internal
	* \brief Add secure digest to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in] rsc Resource that action was for
	* \param[in] params Resource parameters evaluated for node
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] op_version CRM feature set to use for digest calculation
	* \param[in] overrides Key/value hash table to override resource parameters
	*/
	static void
	-calculate_secure_digest(op_digest_cache_t data, pe_resource_t rsc,
	- GHashTable params, xmlNode xml_op,
	+calculate_secure_digest(op_digest_cache_t data, const pe_resource_t rsc,
	+ GHashTable params, const xmlNode xml_op,
	const char op_version, GHashTable overrides)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *secure_list = NULL;
	bool old_version = (compare_version(op_version, "3.16.0") < 0);

	if (xml_op == NULL) {
	secure_list = " passwd password user ";
	} else {
	secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
	}

	if (old_version) {
	data->params_secure = create_xml_node(NULL, XML_TAG_PARAMS);
	if (overrides != NULL) {
	g_hash_table_foreach(overrides, hash2field, data->params_secure);
	}

	g_hash_table_foreach(params, hash2field, data->params_secure);

	} else {
	// Start with a copy of all parameters
	data->params_secure = copy_xml(data->params_all);
	}

	if (secure_list != NULL) {
	pcmk__xe_remove_matching_attrs(data->params_secure, attr_in_string,
	(void *) secure_list);
	}
	if (old_version
	&& pcmk_is_set(pcmk_get_ra_caps(class),
	pcmk_ra_cap_fence_params)) {
	/* For stonith resources, Pacemaker adds special parameters,
	* but these are not listed in fence agent meta-data, so with older
	* versions of DC, the controller will not hash them. That means we have
	* to filter them out before calculating our hash for comparison.
	*/
	pcmk__xe_remove_matching_attrs(data->params_secure, is_fence_param,
	NULL);
	}
	pcmk__filter_op_for_digest(data->params_secure);

	/* CRM_meta_timeout should be part of a digest for recurring operations.
	* However, with older versions of DC, the controller does not add timeout
	* to secure digests, because it only includes parameters declared by the
	* resource agent.
	* Remove any timeout that made it this far, to match.
	*/
	if (old_version) {
	xml_remove_prop(data->params_secure, CRM_META "_" XML_ATTR_TIMEOUT);
	}

	data->digest_secure_calc = calculate_operation_digest(data->params_secure,
	op_version);
	}

	/*!
	* \internal
	* \brief Add restart digest to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] op_version CRM feature set to use for digest calculation
	*
	* \note This function doesn't need to handle overrides because it starts with
	* data->params_all, which already has overrides applied.
	*/
	static void
	-calculate_restart_digest(op_digest_cache_t data, xmlNode xml_op,
	+calculate_restart_digest(op_digest_cache_t data, const xmlNode xml_op,
	const char *op_version)
	{
	const char *value = NULL;

	// We must have XML of resource operation history
	if (xml_op == NULL) {
	return;
	}

	// And the history must have a restart digest to compare against
	if (crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) == NULL) {
	return;
	}

	// Start with a copy of all parameters
	data->params_restart = copy_xml(data->params_all);

	// Then filter out reloadable parameters, if any
	value = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);
	if (value != NULL) {
	pcmk__xe_remove_matching_attrs(data->params_restart, attr_not_in_string,
	(void *) value);
	}

	value = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	data->digest_restart_calc = calculate_operation_digest(data->params_restart,
	value);
	}

	/*!
	* \internal
	* \brief Create a new digest cache entry with calculated digests
	*
	- * \param[in] rsc Resource that action was for
	+ * \param[in,out] rsc Resource that action was for
	* \param[in] task Name of action performed
	* \param[in,out] interval_ms Action's interval (will be reset if in overrides)
	* \param[in] node Node action was performed on
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] overrides Key/value table to override resource parameters
	* \param[in] calc_secure Whether to calculate secure digest
	- * \param[in] data_set Cluster working set
	+ * \param[in,out] data_set Cluster working set
	*
	* \return Pointer to new digest cache entry (or NULL on memory error)
	* \note It is the caller's responsibility to free the result using
	* pe__free_digests().
	*/
	op_digest_cache_t *
	pe__calculate_digests(pe_resource_t rsc, const char task, guint *interval_ms,
	- pe_node_t node, xmlNode xml_op, GHashTable *overrides,
	- bool calc_secure, pe_working_set_t *data_set)
	+ const pe_node_t node, const xmlNode xml_op,
	+ GHashTable *overrides, bool calc_secure,
	+ pe_working_set_t *data_set)
	{
	op_digest_cache_t *data = calloc(1, sizeof(op_digest_cache_t));
	const char *op_version = NULL;
	GHashTable *params = NULL;

	if (data == NULL) {
	return NULL;
	}

	data->rc = RSC_DIGEST_MATCH;

	if (xml_op != NULL) {
	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	}

	if (op_version == NULL && data_set != NULL && data_set->input != NULL) {
	op_version = crm_element_value(data_set->input, XML_ATTR_CRM_VERSION);
	}

	if (op_version == NULL) {
	op_version = CRM_FEATURE_SET;
	}

	params = pe_rsc_params(rsc, node, data_set);
	calculate_main_digest(data, rsc, node, params, task, interval_ms, xml_op,
	op_version, overrides, data_set);
	if (calc_secure) {
	calculate_secure_digest(data, rsc, params, xml_op, op_version,
	overrides);
	}
	calculate_restart_digest(data, xml_op, op_version);
	return data;
	}

	/*!
	* \internal
	* \brief Calculate action digests and store in node's digest cache
	*
	- * \param[in] rsc Resource that action was for
	- * \param[in] task Name of action performed
	- * \param[in] interval_ms Action's interval
	- * \param[in] node Node action was performed on
	- * \param[in] xml_op XML of operation in CIB status (if available)
	- * \param[in] calc_secure Whether to calculate secure digest
	- * \param[in] data_set Cluster working set
	+ * \param[in,out] rsc Resource that action was for
	+ * \param[in] task Name of action performed
	+ * \param[in] interval_ms Action's interval
	+ * \param[in] node Node action was performed on
	+ * \param[in] xml_op XML of operation in CIB status (if available)
	+ * \param[in] calc_secure Whether to calculate secure digest
	+ * \param[in,out] data_set Cluster working set
	*
	* \return Pointer to node's digest cache entry
	*/
	static op_digest_cache_t *
	rsc_action_digest(pe_resource_t rsc, const char task, guint interval_ms,
	- pe_node_t node, xmlNode xml_op, bool calc_secure,
	- pe_working_set_t *data_set)
	+ const pe_node_t node, const xmlNode xml_op,
	+ bool calc_secure, pe_working_set_t *data_set)
	{
	op_digest_cache_t *data = NULL;
	char *key = pcmk__op_key(rsc->id, task, interval_ms);

	data = g_hash_table_lookup(node->details->digest_cache, key);
	if (data == NULL) {
	data = pe__calculate_digests(rsc, task, &interval_ms, node, xml_op,
	NULL, calc_secure, data_set);
	CRM_ASSERT(data != NULL);
	g_hash_table_insert(node->details->digest_cache, strdup(key), data);
	}
	free(key);
	return data;
	}

	/*!
	* \internal
	* \brief Calculate operation digests and compare against an XML history entry
	*
	- * \param[in] rsc Resource to check
	- * \param[in] xml_op Resource history XML
	- * \param[in] node Node to use for digest calculation
	- * \param[in] data_set Cluster working set
	+ * \param[in,out] rsc Resource to check
	+ * \param[in] xml_op Resource history XML
	+ * \param[in] node Node to use for digest calculation
	+ * \param[in,out] data_set Cluster working set
	*
	* \return Pointer to node's digest cache entry, with comparison result set
	*/
	op_digest_cache_t *
	-rsc_action_digest_cmp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * node,
	- pe_working_set_t * data_set)
	+rsc_action_digest_cmp(pe_resource_t rsc, const xmlNode xml_op,
	+ const pe_node_t node, pe_working_set_t data_set)
	{
	op_digest_cache_t *data = NULL;
	guint interval_ms = 0;

	const char *op_version;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *digest_all;
	const char *digest_restart;

	CRM_ASSERT(node != NULL);

	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
	digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	data = rsc_action_digest(rsc, task, interval_ms, node, xml_op,
	pcmk_is_set(data_set->flags, pe_flag_sanitized),
	data_set);

	if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) {
	pe_rsc_info(rsc, "Parameters to %ums-interval %s action for %s on %s "
	"changed: hash was %s vs. now %s (restart:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_restart, "missing"),
	data->digest_restart_calc,
	op_version,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	data->rc = RSC_DIGEST_RESTART;

	} else if (digest_all == NULL) {
	/* it is unknown what the previous op digest was */
	data->rc = RSC_DIGEST_UNKNOWN;

	} else if (strcmp(digest_all, data->digest_all_calc) != 0) {
	/* Given a non-recurring operation with extra parameters configured,
	* in case that the main digest doesn't match, even if the restart
	* digest matches, enforce a restart rather than a reload-agent anyway.
	* So that it ensures any changes of the extra parameters get applied
	* for this specific operation, and the digests calculated for the
	* resulting lrm_rsc_op will be correct.
	* Preserve the implied rc RSC_DIGEST_RESTART for the case that the main
	* digest doesn't match.
	*/
	if (interval_ms == 0
	&& data->rc == RSC_DIGEST_RESTART) {
	pe_rsc_info(rsc, "Parameters containing extra ones to %ums-interval"
	" %s action for %s on %s "
	"changed: hash was %s vs. now %s (restart:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_all, "missing"), data->digest_all_calc,
	op_version,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));

	} else {
	pe_rsc_info(rsc, "Parameters to %ums-interval %s action for %s on %s "
	"changed: hash was %s vs. now %s (%s:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_all, "missing"), data->digest_all_calc,
	(interval_ms > 0)? "reschedule" : "reload",
	op_version,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	data->rc = RSC_DIGEST_ALL;
	}

	} else {
	data->rc = RSC_DIGEST_MATCH;
	}
	return data;
	}

	/*!
	* \internal
	* \brief Create an unfencing summary for use in special node attribute
	*
	* Create a string combining a fence device's resource ID, agent type, and
	* parameter digest (whether for all parameters or just non-private parameters).
	* This can be stored in a special node attribute, allowing us to detect changes
	* in either the agent type or parameters, to know whether unfencing must be
	* redone or can be safely skipped when the device's history is cleaned.
	*
	* \param[in] rsc_id Fence device resource ID
	* \param[in] agent_type Fence device agent
	* \param[in] param_digest Fence device parameter digest
	*
	* \return Newly allocated string with unfencing digest
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	create_unfencing_summary(const char rsc_id, const char agent_type,
	const char *param_digest)
	{
	return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest);
	}

	/*!
	* \internal
	* \brief Check whether a node can skip unfencing
	*
	* Check whether a fence device's current definition matches a node's
	* stored summary of when it was last unfenced by the device.
	*
	* \param[in] rsc_id Fence device's resource ID
	* \param[in] agent Fence device's agent type
	* \param[in] digest_calc Fence device's current parameter digest
	* \param[in] node_summary Value of node's special unfencing node attribute
	* (a comma-separated list of unfencing summaries for
	* all devices that have unfenced this node)
	*
	* \return TRUE if digest matches, FALSE otherwise
	*/
	static bool
	unfencing_digest_matches(const char rsc_id, const char agent,
	const char digest_calc, const char node_summary)
	{
	bool matches = FALSE;

	if (rsc_id && agent && digest_calc && node_summary) {
	char *search_secure = create_unfencing_summary(rsc_id, agent,
	digest_calc);

	/* The digest was calculated including the device ID and agent,
	* so there is no risk of collision using strstr().
	*/
	matches = (strstr(node_summary, search_secure) != NULL);
	crm_trace("Calculated unfencing digest '%s' %sfound in '%s'",
	search_secure, matches? "" : "not ", node_summary);
	free(search_secure);
	}
	return matches;
	}

	/* Magic string to use as action name for digest cache entries used for
	* unfencing checks. This is not a real action name (i.e. "on"), so
	* pcmk__check_action_config() won't confuse these entries with real actions.
	*/
	#define STONITH_DIGEST_TASK "stonith-on"

	/*!
	* \internal
	* \brief Calculate fence device digests and digest comparison result
	*
	- * \param[in] rsc Fence device resource
	- * \param[in] agent Fence device's agent type
	- * \param[in] node Node with digest cache to use
	- * \param[in] data_set Cluster working set
	+ * \param[in,out] rsc Fence device resource
	+ * \param[in] agent Fence device's agent type
	+ * \param[in] node Node with digest cache to use
	+ * \param[in,out] data_set Cluster working set
	*
	* \return Node's digest cache entry
	*/
	op_digest_cache_t *
	pe__compare_fencing_digest(pe_resource_t rsc, const char agent,
	- pe_node_t node, pe_working_set_t data_set)
	+ const pe_node_t node, pe_working_set_t data_set)
	{
	const char *node_summary = NULL;

	// Calculate device's current parameter digests
	op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, 0U,
	node, NULL, TRUE, data_set);

	// Check whether node has special unfencing summary node attribute
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL);
	if (node_summary == NULL) {
	data->rc = RSC_DIGEST_UNKNOWN;
	return data;
	}

	// Check whether full parameter digest matches
	if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc,
	node_summary)) {
	data->rc = RSC_DIGEST_MATCH;
	return data;
	}

	// Check whether secure parameter digest matches
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE);
	if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc,
	node_summary)) {
	data->rc = RSC_DIGEST_MATCH;
	if (!pcmk__is_daemon && data_set->priv != NULL) {
	pcmk__output_t *out = data_set->priv;
	out->info(out, "Only 'private' parameters to %s "
	"for unfencing %s changed", rsc->id,
	pe__node_name(node));
	}
	return data;
	}

	// Parameters don't match
	data->rc = RSC_DIGEST_ALL;
	if (pcmk_is_set(data_set->flags, pe_flag_sanitized) && data->digest_secure_calc) {
	if (data_set->priv != NULL) {
	pcmk__output_t *out = data_set->priv;
	char *digest = create_unfencing_summary(rsc->id, agent,
	data->digest_secure_calc);

	out->info(out, "Parameters to %s for unfencing "
	"%s changed, try '%s'", rsc->id,
	pe__node_name(node), digest);
	free(digest);
	} else if (!pcmk__is_daemon) {
	char *digest = create_unfencing_summary(rsc->id, agent,
	data->digest_secure_calc);

	printf("Parameters to %s for unfencing %s changed, try '%s'\n",
	rsc->id, pe__node_name(node), digest);
	free(digest);
	}
	}
	return data;
	}
	diff --git a/lib/pengine/pe_status_private.h b/lib/pengine/pe_status_private.h
	index 7f441b7480..5e41e5eae6 100644
	--- a/lib/pengine/pe_status_private.h
	+++ b/lib/pengine/pe_status_private.h
	@@ -1,83 +1,83 @@
	/*
	* Copyright 2018-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PE_STATUS_PRIVATE__H
	# define PE_STATUS_PRIVATE__H

	/* This header is for the sole use of libpe_status, so that functions can be
	* declared with G_GNUC_INTERNAL for efficiency.
	*/

	#if defined(PCMK__UNIT_TESTING)
	#undef G_GNUC_INTERNAL
	#define G_GNUC_INTERNAL
	#endif

	/*!
	* \internal
	* \deprecated This macro will be removed in a future release
	*/
	# define status_print(fmt, args...) \
	if(options & pe_print_html) { \
	FILE *stream = print_data; \
	fprintf(stream, fmt, ##args); \
	} else if(options & pe_print_printf \|\| options & pe_print_ncurses) { \
	FILE *stream = print_data; \
	fprintf(stream, fmt, ##args); \
	} else if(options & pe_print_xml) { \
	FILE *stream = print_data; \
	fprintf(stream, fmt, ##args); \
	} else if(options & pe_print_log) { \
	int log_level = (int)print_data; \
	do_crm_log(log_level, fmt, ##args); \
	}

	G_GNUC_INTERNAL
	pe_resource_t pe__create_clone_child(pe_resource_t rsc,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pe__force_anon(const char standard, pe_resource_t rsc, const char *rid,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	gint pe__cmp_rsc_priority(gconstpointer a, gconstpointer b);

	G_GNUC_INTERNAL
	gboolean pe__unpack_resource(xmlNode xml_obj, pe_resource_t *rsc,
	pe_resource_t parent, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	gboolean unpack_remote_nodes(xmlNode xml_resources, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	gboolean unpack_resources(xmlNode xml_resources, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	gboolean unpack_config(xmlNode config, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	gboolean unpack_nodes(xmlNode xml_nodes, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	gboolean unpack_tags(xmlNode xml_tags, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	gboolean unpack_status(xmlNode status, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	op_digest_cache_t pe__compare_fencing_digest(pe_resource_t rsc,
	const char *agent,
	- pe_node_t *node,
	+ const pe_node_t *node,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pe__unpack_node_health_scores(pe_working_set_t *data_set);

	#endif // PE_STATUS_PRIVATE__H
	diff --git a/lib/pengine/remote.c b/lib/pengine/remote.c
	index ab19ff94da..3f3f395d3c 100644
	--- a/lib/pengine/remote.c
	+++ b/lib/pengine/remote.c
	@@ -1,267 +1,268 @@
	/*
	- * Copyright 2013-2019 the Pacemaker project contributors
	+ * Copyright 2013-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/internal.h>
	#include <glib.h>

	bool
	pe__resource_is_remote_conn(const pe_resource_t *rsc,
	const pe_working_set_t *data_set)
	{
	return (rsc != NULL) && rsc->is_remote_node
	&& pe__is_remote_node(pe_find_node(data_set->nodes, rsc->id));
	}

	bool
	pe__is_remote_node(const pe_node_t *node)
	{
	return (node != NULL) && (node->details->type == node_remote)
	&& ((node->details->remote_rsc == NULL)
	\|\| (node->details->remote_rsc->container == NULL));
	}

	bool
	pe__is_guest_node(const pe_node_t *node)
	{
	return (node != NULL) && (node->details->type == node_remote)
	&& (node->details->remote_rsc != NULL)
	&& (node->details->remote_rsc->container != NULL);
	}

	bool
	pe__is_guest_or_remote_node(const pe_node_t *node)
	{
	return (node != NULL) && (node->details->type == node_remote);
	}

	bool
	pe__is_bundle_node(const pe_node_t *node)
	{
	return pe__is_guest_node(node)
	&& pe_rsc_is_bundled(node->details->remote_rsc);
	}

	/*!
	* \internal
	* \brief Check whether a resource creates a guest node
	*
	* If a given resource contains a filler resource that is a remote connection,
	* return that filler resource (or NULL if none is found).
	*
	* \param[in] data_set Working set of cluster
	* \param[in] rsc Resource to check
	*
	* \return Filler resource with remote connection, or NULL if none found
	*/
	pe_resource_t *
	pe__resource_contains_guest_node(const pe_working_set_t *data_set,
	const pe_resource_t *rsc)
	{
	if ((rsc != NULL) && (data_set != NULL)
	&& pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {

	for (GList *gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *filler = gIter->data;

	if (filler->is_remote_node) {
	return filler;
	}
	}
	}
	return NULL;
	}

	bool
	xml_contains_remote_node(xmlNode *xml)
	{
	const char *value = NULL;

	if (xml == NULL) {
	return false;
	}

	value = crm_element_value(xml, XML_ATTR_TYPE);
	if (!pcmk__str_eq(value, "remote", pcmk__str_casei)) {
	return false;
	}

	value = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
	if (!pcmk__str_eq(value, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
	return false;
	}

	value = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
	if (!pcmk__str_eq(value, "pacemaker", pcmk__str_casei)) {
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Execute a supplied function for each guest node running on a host
	*
	* \param[in] data_set Working set for cluster
	* \param[in] host Host node to check
	* \param[in] helper Function to call for each guest node
	* \param[in,out] user_data Pointer to pass to helper function
	*/
	void
	pe_foreach_guest_node(const pe_working_set_t data_set, const pe_node_t host,
	void (helper)(const pe_node_t, void), void user_data)
	{
	GList *iter;

	CRM_CHECK(data_set && host && host->details && helper, return);
	if (!pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	return;
	}
	for (iter = host->details->running_rsc; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	if (rsc->is_remote_node && (rsc->container != NULL)) {
	pe_node_t *guest_node = pe_find_node(data_set->nodes, rsc->id);

	if (guest_node) {
	(*helper)(guest_node, user_data);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Create CIB XML for an implicit remote connection
	*
	* \param[in] parent If not NULL, use as parent XML element
	* \param[in] uname Name of Pacemaker Remote node
	* \param[in] container If not NULL, use this as connection container
	* \param[in] migrateable If not NULL, use as allow-migrate value
	* \param[in] is_managed If not NULL, use as is-managed value
	* \param[in] start_timeout If not NULL, use as remote connect timeout
	* \param[in] server If not NULL, use as remote server value
	* \param[in] port If not NULL, use as remote port value
	*/
	xmlNode *
	pe_create_remote_xml(xmlNode parent, const char uname,
	const char container_id, const char migrateable,
	const char is_managed, const char start_timeout,
	const char server, const char port)
	{
	xmlNode *remote;
	xmlNode *xml_sub;

	remote = create_xml_node(parent, XML_CIB_TAG_RESOURCE);

	// Add identity
	crm_xml_add(remote, XML_ATTR_ID, uname);
	crm_xml_add(remote, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(remote, XML_AGENT_ATTR_PROVIDER, "pacemaker");
	crm_xml_add(remote, XML_ATTR_TYPE, "remote");

	// Add meta-attributes
	xml_sub = create_xml_node(remote, XML_TAG_META_SETS);
	crm_xml_set_id(xml_sub, "%s-%s", uname, XML_TAG_META_SETS);
	crm_create_nvpair_xml(xml_sub, NULL,
	XML_RSC_ATTR_INTERNAL_RSC, XML_BOOLEAN_TRUE);
	if (container_id) {
	crm_create_nvpair_xml(xml_sub, NULL,
	XML_RSC_ATTR_CONTAINER, container_id);
	}
	if (migrateable) {
	crm_create_nvpair_xml(xml_sub, NULL,
	XML_OP_ATTR_ALLOW_MIGRATE, migrateable);
	}
	if (is_managed) {
	crm_create_nvpair_xml(xml_sub, NULL, XML_RSC_ATTR_MANAGED, is_managed);
	}

	// Add instance attributes
	if (port \|\| server) {
	xml_sub = create_xml_node(remote, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_sub, "%s-%s", uname, XML_TAG_ATTR_SETS);
	if (server) {
	crm_create_nvpair_xml(xml_sub, NULL, XML_RSC_ATTR_REMOTE_RA_ADDR,
	server);
	}
	if (port) {
	crm_create_nvpair_xml(xml_sub, NULL, "port", port);
	}
	}

	// Add operations
	xml_sub = create_xml_node(remote, "operations");
	crm_create_op_xml(xml_sub, uname, "monitor", "30s", "30s");
	if (start_timeout) {
	crm_create_op_xml(xml_sub, uname, "start", "0", start_timeout);
	}
	return remote;
	}

	// History entry to be checked for fail count clearing
	struct check_op {
	- xmlNode *rsc_op; // History entry XML
	- pe_resource_t *rsc; // Known resource corresponding to history entry
	- pe_node_t *node; // Known node corresponding to history entry
	+ const xmlNode *rsc_op; // History entry XML
	+ pe_resource_t *rsc; // Known resource corresponding to history entry
	+ const pe_node_t *node; // Known node corresponding to history entry
	enum pe_check_parameters check_type; // What needs checking
	};

	void
	-pe__add_param_check(xmlNode rsc_op, pe_resource_t rsc, pe_node_t *node,
	- enum pe_check_parameters flag, pe_working_set_t *data_set)
	+pe__add_param_check(const xmlNode rsc_op, pe_resource_t rsc,
	+ const pe_node_t *node, enum pe_check_parameters flag,
	+ pe_working_set_t *data_set)
	{
	struct check_op *check_op = NULL;

	CRM_CHECK(data_set && rsc_op && rsc && node, return);

	check_op = calloc(1, sizeof(struct check_op));
	CRM_ASSERT(check_op != NULL);

	crm_trace("Deferring checks of %s until after allocation", ID(rsc_op));
	check_op->rsc_op = rsc_op;
	check_op->rsc = rsc;
	check_op->node = node;
	check_op->check_type = flag;
	data_set->param_check = g_list_prepend(data_set->param_check, check_op);
	}

	/*!
	* \internal
	* \brief Call a function for each action to be checked for addr substitution
	*
	- * \param[in] data_set Working set for cluster
	- * \param[in] cb Function to be called
	+ * \param[in,out] data_set Working set for cluster
	+ * \param[in] cb Function to be called
	*/
	void
	pe__foreach_param_check(pe_working_set_t *data_set,
	- void (cb)(pe_resource_t, pe_node_t, xmlNode,
	- enum pe_check_parameters, pe_working_set_t*))
	+ void (cb)(pe_resource_t, const pe_node_t*,
	+ const xmlNode*, enum pe_check_parameters))
	{
	CRM_CHECK(data_set && cb, return);

	for (GList *item = data_set->param_check; item != NULL; item = item->next) {
	struct check_op *check_op = item->data;

	cb(check_op->rsc, check_op->node, check_op->rsc_op,
	- check_op->check_type, data_set);
	+ check_op->check_type);
	}
	}

	void
	pe__free_param_checks(pe_working_set_t *data_set)
	{
	if (data_set && data_set->param_check) {
	g_list_free_full(data_set->param_check, free);
	data_set->param_check = NULL;
	}
	}
	diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
	index 7e8b89dd37..fe83cab830 100644
	--- a/lib/pengine/utils.c
	+++ b/lib/pengine/utils.c
	@@ -1,905 +1,906 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>

	#include "pe_status_private.h"

	extern bool pcmk__is_daemon;

	gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);

	/*!
	* \internal
	* \brief Check whether we can fence a particular node
	*
	* \param[in] data_set Working set for cluster
	* \param[in] node Name of node to check
	*
	* \return true if node can be fenced, false otherwise
	*/
	bool
	-pe_can_fence(pe_working_set_t data_set, pe_node_t node)
	+pe_can_fence(const pe_working_set_t data_set, const pe_node_t node)
	{
	if (pe__is_guest_node(node)) {
	/* Guest nodes are fenced by stopping their container resource. We can
	* do that if the container's host is either online or fenceable.
	*/
	pe_resource_t *rsc = node->details->remote_rsc->container;

	for (GList *n = rsc->running_on; n != NULL; n = n->next) {
	pe_node_t *container_node = n->data;

	if (!container_node->details->online
	&& !pe_can_fence(data_set, container_node)) {
	return false;
	}
	}
	return true;

	} else if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	return false; /* Turned off */

	} else if (!pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {
	return false; /* No devices */

	} else if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	return true;

	} else if (data_set->no_quorum_policy == no_quorum_ignore) {
	return true;

	} else if(node == NULL) {
	return false;

	} else if(node->details->online) {
	crm_notice("We can fence %s without quorum because they're in our membership",
	pe__node_name(node));
	return true;
	}

	crm_trace("Cannot fence %s", pe__node_name(node));
	return false;
	}

	/*!
	* \internal
	* \brief Copy a node object
	*
	* \param[in] this_node Node object to copy
	*
	* \return Newly allocated shallow copy of this_node
	* \note This function asserts on errors and is guaranteed to return non-NULL.
	*/
	pe_node_t *
	pe__copy_node(const pe_node_t *this_node)
	{
	pe_node_t *new_node = NULL;

	CRM_ASSERT(this_node != NULL);

	new_node = calloc(1, sizeof(pe_node_t));
	CRM_ASSERT(new_node != NULL);

	new_node->rsc_discover_mode = this_node->rsc_discover_mode;
	new_node->weight = this_node->weight;
	new_node->fixed = this_node->fixed; // @COMPAT deprecated and unused
	new_node->details = this_node->details;

	return new_node;
	}

	/* any node in list1 or list2 and not in the other gets a score of -INFINITY */
	void
	node_list_exclude(GHashTable * hash, GList *list, gboolean merge_scores)
	{
	GHashTable *result = hash;
	pe_node_t *other_node = NULL;
	GList *gIter = list;

	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, hash);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {

	other_node = pe_find_node_id(list, node->details->id);
	if (other_node == NULL) {
	node->weight = -INFINITY;
	} else if (merge_scores) {
	node->weight = pcmk__add_scores(node->weight, other_node->weight);
	}
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	other_node = pe_hash_table_lookup(result, node->details->id);

	if (other_node == NULL) {
	pe_node_t *new_node = pe__copy_node(node);

	new_node->weight = -INFINITY;
	g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
	}
	}
	}

	/*!
	* \internal
	* \brief Create a node hash table from a node list
	*
	* \param[in] list Node list
	*
	* \return Hash table equivalent of node list
	*/
	GHashTable *
	pe__node_list2table(GList *list)
	{
	GHashTable *result = NULL;

	result = pcmk__strkey_table(NULL, free);
	for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pe_node_t new_node = pe__copy_node((pe_node_t ) gIter->data);

	g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Compare two nodes by name, with numeric portions sorted numerically
	*
	* Sort two node names case-insensitively like strcasecmp(), but with any
	* numeric portions of the name sorted numerically. For example, "node10" will
	* sort higher than "node9" but lower than "remotenode9".
	*
	* \param[in] a First node to compare (can be \c NULL)
	* \param[in] b Second node to compare (can be \c NULL)
	*
	* \retval -1 \c a comes before \c b (or \c a is \c NULL and \c b is not)
	* \retval 0 \c a and \c b are equal (or both are \c NULL)
	* \retval 1 \c a comes after \c b (or \c b is \c NULL and \c a is not)
	*/
	gint
	pe__cmp_node_name(gconstpointer a, gconstpointer b)
	{
	const pe_node_t node1 = (const pe_node_t ) a;
	const pe_node_t node2 = (const pe_node_t ) b;

	if ((node1 == NULL) && (node2 == NULL)) {
	return 0;
	}

	if (node1 == NULL) {
	return -1;
	}

	if (node2 == NULL) {
	return 1;
	}

	return pcmk__numeric_strcasecmp(node1->details->uname,
	node2->details->uname);
	}

	/*!
	* \internal
	* \brief Output node weights to stdout
	*
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	*/
	static void
	pe__output_node_weights(pe_resource_t rsc, const char comment,
	GHashTable nodes, pe_working_set_t data_set)
	{
	pcmk__output_t *out = data_set->priv;

	// Sort the nodes so the output is consistent for regression tests
	GList *list = g_list_sort(g_hash_table_get_values(nodes),
	pe__cmp_node_name);

	for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	out->message(out, "node-weight", rsc, comment, node->details->uname,
	pcmk_readable_score(node->weight));
	}
	g_list_free(list);
	}

	/*!
	* \internal
	* \brief Log node weights at trace level
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	*/
	static void
	pe__log_node_weights(const char file, const char function, int line,
	pe_resource_t rsc, const char comment, GHashTable *nodes)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	// Don't waste time if we're not tracing at this point
	pcmk__log_else(LOG_TRACE, return);

	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (rsc) {
	qb_log_from_external_source(function, file,
	"%s: %s allocation score on %s: %s",
	LOG_TRACE, line, 0,
	comment, rsc->id,
	pe__node_name(node),
	pcmk_readable_score(node->weight));
	} else {
	qb_log_from_external_source(function, file, "%s: %s = %s",
	LOG_TRACE, line, 0,
	comment, pe__node_name(node),
	pcmk_readable_score(node->weight));
	}
	}
	}

	/*!
	* \internal
	* \brief Log or output node weights
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] to_log Log if true, otherwise output
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes Use these nodes
	*/
	void
	pe__show_node_weights_as(const char file, const char function, int line,
	bool to_log, pe_resource_t rsc, const char comment,
	GHashTable nodes, pe_working_set_t data_set)
	{
	if (rsc != NULL && pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	// Don't show allocation scores for orphans
	return;
	}
	if (nodes == NULL) {
	// Nothing to show
	return;
	}

	if (to_log) {
	pe__log_node_weights(file, function, line, rsc, comment, nodes);
	} else {
	pe__output_node_weights(rsc, comment, nodes, data_set);
	}

	// If this resource has children, repeat recursively for each
	if (rsc && rsc->children) {
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	pe__show_node_weights_as(file, function, line, to_log, child,
	comment, child->allowed_nodes, data_set);
	}
	}
	}

	/*!
	* \internal
	* \brief Compare two resources by priority
	*
	* \param[in] a First resource to compare (can be \c NULL)
	* \param[in] b Second resource to compare (can be \c NULL)
	*
	* \retval -1 \c a->priority > \c b->priority (or \c b is \c NULL and \c a is
	* not)
	* \retval 0 \c a->priority == \c b->priority (or both \c a and \c b are
	* \c NULL)
	* \retval 1 \c a->priority < \c b->priority (or \c a is \c NULL and \c b is
	* not)
	*/
	gint
	pe__cmp_rsc_priority(gconstpointer a, gconstpointer b)
	{
	const pe_resource_t resource1 = (const pe_resource_t )a;
	const pe_resource_t resource2 = (const pe_resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->priority > resource2->priority) {
	return -1;
	}

	if (resource1->priority < resource2->priority) {
	return 1;
	}

	return 0;
	}

	static void
	resource_node_score(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag)
	{
	pe_node_t *match = NULL;

	if ((rsc->exclusive_discover \|\| (node->rsc_discover_mode == pe_discover_never))
	&& pcmk__str_eq(tag, "symmetric_default", pcmk__str_casei)) {
	/* This string comparision may be fragile, but exclusive resources and
	* exclusive nodes should not have the symmetric_default constraint
	* applied to them.
	*/
	return;

	} else if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	resource_node_score(child_rsc, node, score, tag);
	}
	}

	pe_rsc_trace(rsc, "Setting %s for %s on %s: %d",
	tag, rsc->id, pe__node_name(node), score);
	match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match == NULL) {
	match = pe__copy_node(node);
	g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
	}
	match->weight = pcmk__add_scores(match->weight, score);
	}

	void
	resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
	pe_working_set_t * data_set)
	{
	if (node != NULL) {
	resource_node_score(rsc, node, score, tag);

	} else if (data_set != NULL) {
	GList *gIter = data_set->nodes;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node_iter = (pe_node_t ) gIter->data;

	resource_node_score(rsc, node_iter, score, tag);
	}

	} else {
	GHashTableIter iter;
	pe_node_t *node_iter = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
	resource_node_score(rsc, node_iter, score, tag);
	}
	}

	if (node == NULL && score == -INFINITY) {
	if (rsc->allocated_to) {
	crm_info("Deallocating %s from %s",
	rsc->id, pe__node_name(rsc->allocated_to));
	free(rsc->allocated_to);
	rsc->allocated_to = NULL;
	}
	}
	}

	time_t
	get_effective_time(pe_working_set_t * data_set)
	{
	if(data_set) {
	if (data_set->now == NULL) {
	crm_trace("Recording a new 'now'");
	data_set->now = crm_time_new(NULL);
	}
	return crm_time_get_seconds_since_epoch(data_set->now);
	}

	crm_trace("Defaulting to 'now'");
	return time(NULL);
	}

	gboolean
	get_target_role(pe_resource_t * rsc, enum rsc_role_e * role)
	{
	enum rsc_role_e local_role = RSC_ROLE_UNKNOWN;
	const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);

	CRM_CHECK(role != NULL, return FALSE);

	if (pcmk__str_eq(value, "started", pcmk__str_null_matches \| pcmk__str_casei)
	\|\| pcmk__str_eq("default", value, pcmk__str_casei)) {
	return FALSE;
	}

	local_role = text2role(value);
	if (local_role == RSC_ROLE_UNKNOWN) {
	pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
	"because '%s' is not valid", rsc->id, value);
	return FALSE;

	} else if (local_role > RSC_ROLE_STARTED) {
	if (pcmk_is_set(uber_parent(rsc)->flags, pe_rsc_promotable)) {
	if (local_role > RSC_ROLE_UNPROMOTED) {
	/* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
	return FALSE;
	}

	} else {
	pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
	"because '%s' only makes sense for promotable "
	"clones", rsc->id, value);
	return FALSE;
	}
	}

	*role = local_role;
	return TRUE;
	}

	gboolean
	order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order)
	{
	GList *gIter = NULL;
	pe_action_wrapper_t *wrapper = NULL;
	GList *list = NULL;

	if (order == pe_order_none) {
	return FALSE;
	}

	if (lh_action == NULL \|\| rh_action == NULL) {
	return FALSE;
	}

	crm_trace("Creating action wrappers for ordering: %s then %s",
	lh_action->uuid, rh_action->uuid);

	/* Ensure we never create a dependency on ourselves... it's happened */
	CRM_ASSERT(lh_action != rh_action);

	/* Filter dups, otherwise update_action_states() has too much work to do */
	gIter = lh_action->actions_after;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_wrapper_t after = (pe_action_wrapper_t ) gIter->data;

	if (after->action == rh_action && (after->type & order)) {
	return FALSE;
	}
	}

	wrapper = calloc(1, sizeof(pe_action_wrapper_t));
	wrapper->action = rh_action;
	wrapper->type = order;
	list = lh_action->actions_after;
	list = g_list_prepend(list, wrapper);
	lh_action->actions_after = list;

	wrapper = calloc(1, sizeof(pe_action_wrapper_t));
	wrapper->action = lh_action;
	wrapper->type = order;
	list = rh_action->actions_before;
	list = g_list_prepend(list, wrapper);
	rh_action->actions_before = list;
	return TRUE;
	}

	void
	destroy_ticket(gpointer data)
	{
	pe_ticket_t *ticket = data;

	if (ticket->state) {
	g_hash_table_destroy(ticket->state);
	}
	free(ticket->id);
	free(ticket);
	}

	pe_ticket_t *
	ticket_new(const char ticket_id, pe_working_set_t data_set)
	{
	pe_ticket_t *ticket = NULL;

	if (pcmk__str_empty(ticket_id)) {
	return NULL;
	}

	if (data_set->tickets == NULL) {
	data_set->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {

	ticket = calloc(1, sizeof(pe_ticket_t));
	if (ticket == NULL) {
	crm_err("Cannot allocate ticket '%s'", ticket_id);
	return NULL;
	}

	crm_trace("Creaing ticket entry for %s", ticket_id);

	ticket->id = strdup(ticket_id);
	ticket->granted = FALSE;
	ticket->last_granted = -1;
	ticket->standby = FALSE;
	ticket->state = pcmk__strkey_table(free, free);

	g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket);
	}

	return ticket;
	}

	const char rsc_printable_id(pe_resource_t rsc)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	return ID(rsc->xml);
	}
	return rsc->id;
	}

	void
	pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags)
	{
	pe__clear_resource_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__clear_resource_flags_recursive((pe_resource_t *) gIter->data, flags);
	}
	}

	void
	pe__clear_resource_flags_on_all(pe_working_set_t *data_set, uint64_t flag)
	{
	for (GList *lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t r = (pe_resource_t ) lpc->data;
	pe__clear_resource_flags_recursive(r, flag);
	}
	}

	void
	pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags)
	{
	pe__set_resource_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__set_resource_flags_recursive((pe_resource_t *) gIter->data, flags);
	}
	}

	void
	-trigger_unfencing(
	- pe_resource_t * rsc, pe_node_t node, const char reason, pe_action_t dependency, pe_working_set_t data_set)
	+trigger_unfencing(pe_resource_t rsc, const pe_node_t node,
	+ const char reason, pe_action_t dependency,
	+ pe_working_set_t *data_set)
	{
	if (!pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)) {
	/* No resources require it */
	return;

	} else if ((rsc != NULL)
	&& !pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
	/* Wasn't a stonith device */
	return;

	} else if(node
	&& node->details->online
	&& node->details->unclean == FALSE
	&& node->details->shutdown == FALSE) {
	pe_action_t *unfence = pe_fence_op(node, "on", FALSE, reason, FALSE, data_set);

	if(dependency) {
	order_actions(unfence, dependency, pe_order_optional);
	}

	} else if(rsc) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
	trigger_unfencing(rsc, node, reason, dependency, data_set);
	}
	}
	}
	}

	gboolean
	add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref)
	{
	pe_tag_t *tag = NULL;
	GList *gIter = NULL;
	gboolean is_existing = FALSE;

	CRM_CHECK(tags && tag_name && obj_ref, return FALSE);

	tag = g_hash_table_lookup(tags, tag_name);
	if (tag == NULL) {
	tag = calloc(1, sizeof(pe_tag_t));
	if (tag == NULL) {
	return FALSE;
	}
	tag->id = strdup(tag_name);
	tag->refs = NULL;
	g_hash_table_insert(tags, strdup(tag_name), tag);
	}

	for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
	const char existing_ref = (const char ) gIter->data;

	if (pcmk__str_eq(existing_ref, obj_ref, pcmk__str_none)){
	is_existing = TRUE;
	break;
	}
	}

	if (is_existing == FALSE) {
	tag->refs = g_list_append(tag->refs, strdup(obj_ref));
	crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Check whether shutdown has been requested for a node
	*
	* \param[in] node Node to check
	*
	* \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
	* \note This differs from simply using node->details->shutdown in that it can
	* be used before that has been determined (and in fact to determine it),
	* and it can also be used to distinguish requested shutdown from implicit
	* shutdown of remote nodes by virtue of their connection stopping.
	*/
	bool
	pe__shutdown_requested(pe_node_t *node)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);

	return !pcmk__str_eq(shutdown, "0", pcmk__str_null_matches);
	}

	/*!
	* \internal
	* \brief Update a data set's "recheck by" time
	*
	* \param[in] recheck Epoch time when recheck should happen
	* \param[in,out] data_set Current working set
	*/
	void
	pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set)
	{
	if ((recheck > get_effective_time(data_set))
	&& ((data_set->recheck_by == 0)
	\|\| (data_set->recheck_by > recheck))) {
	data_set->recheck_by = recheck;
	}
	}

	/*!
	* \internal
	* \brief Wrapper for pe_unpack_nvpairs() using a cluster working set
	*/
	void
	pe__unpack_dataset_nvpairs(const xmlNode xml_obj, const char set_name,
	pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	pe_working_set_t *data_set)
	{
	crm_time_t *next_change = crm_time_new_undefined();

	pe_eval_nvpairs(data_set->input, xml_obj, set_name, rule_data, hash,
	always_first, overwrite, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, data_set);
	}
	crm_time_free(next_change);
	}

	bool
	pe__resource_is_disabled(pe_resource_t *rsc)
	{
	const char *target_role = NULL;

	CRM_CHECK(rsc != NULL, return false);
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	if (target_role) {
	enum rsc_role_e target_role_e = text2role(target_role);

	if ((target_role_e == RSC_ROLE_STOPPED)
	\|\| ((target_role_e == RSC_ROLE_UNPROMOTED)
	&& pcmk_is_set(uber_parent(rsc)->flags, pe_rsc_promotable))) {
	return true;
	}
	}
	return false;
	}

	bool
	pe__rsc_running_on_any(pe_resource_t rsc, GList node_list)
	{
	for (GList *ele = rsc->running_on; ele; ele = ele->next) {
	pe_node_t node = (pe_node_t ) ele->data;
	if (pcmk__str_in_list(node->details->uname, node_list,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	return true;
	}
	}

	return false;
	}

	bool
	pcmk__rsc_filtered_by_node(pe_resource_t rsc, GList only_node)
	{
	return (rsc->fns->active(rsc, FALSE) && !pe__rsc_running_on_any(rsc, only_node));
	}

	GList *
	pe__filter_rsc_list(GList rscs, GList filter)
	{
	GList *retval = NULL;

	for (GList *gIter = rscs; gIter; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* I think the second condition is safe here for all callers of this
	* function. If not, it needs to move into pe__node_text.
	*/
	if (pcmk__str_in_list(rsc_printable_id(rsc), filter, pcmk__str_star_matches) \|\|
	(rsc->parent && pcmk__str_in_list(rsc_printable_id(rsc->parent), filter, pcmk__str_star_matches))) {
	retval = g_list_prepend(retval, rsc);
	}
	}

	return retval;
	}

	GList *
	pe__build_node_name_list(pe_working_set_t data_set, const char s) {
	GList *nodes = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	/* Nothing was given so return a list of all node names. Or, '*' was
	* given. This would normally fall into the pe__unames_with_tag branch
	* where it will return an empty list. Catch it here instead.
	*/
	nodes = g_list_prepend(nodes, strdup("*"));
	} else {
	pe_node_t *node = pe_find_node(data_set->nodes, s);

	if (node) {
	/* The given string was a valid uname for a node. Return a
	* singleton list containing just that uname.
	*/
	nodes = g_list_prepend(nodes, strdup(s));
	} else {
	/* The given string was not a valid uname. It's either a tag or
	* it's a typo or something. In the first case, we'll return a
	* list of all the unames of the nodes with the given tag. In the
	* second case, we'll return a NULL pointer and nothing will
	* get displayed.
	*/
	nodes = pe__unames_with_tag(data_set, s);
	}
	}

	return nodes;
	}

	GList *
	pe__build_rsc_list(pe_working_set_t data_set, const char s) {
	GList *resources = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	resources = g_list_prepend(resources, strdup("*"));
	} else {
	pe_resource_t *rsc = pe_find_resource_with_flags(data_set->resources, s,
	pe_find_renamed\|pe_find_any);

	if (rsc) {
	/* A colon in the name we were given means we're being asked to filter
	* on a specific instance of a cloned resource. Put that exact string
	* into the filter list. Otherwise, use the printable ID of whatever
	* resource was found that matches what was asked for.
	*/
	if (strstr(s, ":") != NULL) {
	resources = g_list_prepend(resources, strdup(rsc->id));
	} else {
	resources = g_list_prepend(resources, strdup(rsc_printable_id(rsc)));
	}
	} else {
	/* The given string was not a valid resource name. It's a tag or a
	* typo or something. See pe__build_node_name_list() for more
	* detail.
	*/
	resources = pe__rscs_with_tag(data_set, s);
	}
	}

	return resources;
	}

	xmlNode *
	pe__failed_probe_for_rsc(pe_resource_t rsc, const char name)
	{
	pe_resource_t *parent = uber_parent(rsc);
	const char *rsc_id = rsc->id;

	if (rsc->variant == pe_clone) {
	rsc_id = pe__clone_child_id(rsc);
	} else if (parent->variant == pe_clone) {
	rsc_id = pe__clone_child_id(parent);
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(rsc->cluster->failed); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	const char *value = NULL;
	char *op_id = NULL;

	/* This resource operation is not a failed probe. */
	if (!pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	/* This resource operation was not run on the given node. Note that if name is
	* NULL, this will always succeed.
	*/
	value = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);
	if (value == NULL \|\| !pcmk__str_eq(value, name, pcmk__str_casei\|pcmk__str_null_matches)) {
	continue;
	}

	/* This resource operation has no operation_key. */
	value = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
	if (!parse_op_key(value ? value : ID(xml_op), &op_id, NULL, NULL)) {
	continue;
	}

	/* This resource operation's ID does not match the rsc_id we are looking for. */
	if (!pcmk__str_eq(op_id, rsc_id, pcmk__str_none)) {
	free(op_id);
	continue;
	}

	free(op_id);
	return xml_op;
	}

	return NULL;
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 6:11 PM (21 h, 12 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1983153
Default Alt Text: (328 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions