No OneTemporary
Actions

Size

378 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
	index ab7238f423..c464295aac 100644
	--- a/include/crm/pengine/internal.h
	+++ b/include/crm/pengine/internal.h
	@@ -1,598 +1,598 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PE_INTERNAL__H
	# define PE_INTERNAL__H
	# include <string.h>
	# include <crm/pengine/status.h>
	# include <crm/pengine/remote_internal.h>
	# include <crm/common/internal.h>
	# include <crm/common/output_internal.h>

	# define pe_rsc_info(rsc, fmt, args...) crm_log_tag(LOG_INFO, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_debug(rsc, fmt, args...) crm_log_tag(LOG_DEBUG, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_trace(rsc, fmt, args...) crm_log_tag(LOG_TRACE, rsc ? rsc->id : "<NULL>", fmt, ##args)

	# define pe_err(fmt...) do { \
	was_processing_error = TRUE; \
	pcmk__config_err(fmt); \
	} while (0)

	# define pe_warn(fmt...) do { \
	was_processing_warning = TRUE; \
	pcmk__config_warn(fmt); \
	} while (0)

	# define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
	# define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }

	#define pe__set_working_set_flags(working_set, flags_to_set) do { \
	(working_set)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_working_set_flags(working_set, flags_to_clear) do { \
	(working_set)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_resource_flags(resource, flags_to_set) do { \
	(resource)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_resource_flags(resource, flags_to_clear) do { \
	(resource)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_action_flags(action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags(action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_raw_action_flags(action_flags, action_name, flags_to_set) do { \
	action_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action", action_name, \
	(action_flags), \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_raw_action_flags(action_flags, action_name, flags_to_clear) do { \
	action_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", action_name, \
	(action_flags), \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_action_flags_as(function, line, action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags_as(function, line, action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_order_flags(order_flags, flags_to_set) do { \
	order_flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_order_flags(order_flags, flags_to_clear) do { \
	order_flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_graph_flags(graph_flags, gr_action, flags_to_set) do { \
	graph_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Graph", \
	(gr_action)->uuid, graph_flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_graph_flags(graph_flags, gr_action, flags_to_clear) do { \
	graph_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Graph", \
	(gr_action)->uuid, graph_flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	// Some warnings we don't want to print every transition

	enum pe_warn_once_e {
	pe_wo_blind = (1 << 0),
	pe_wo_restart_type = (1 << 1),
	pe_wo_role_after = (1 << 2),
	pe_wo_poweroff = (1 << 3),
	pe_wo_require_all = (1 << 4),
	pe_wo_order_score = (1 << 5),
	pe_wo_neg_threshold = (1 << 6),
	pe_wo_remove_after = (1 << 7),
	};

	extern uint32_t pe_wo;

	#define pe_warn_once(pe_wo_bit, fmt...) do { \
	if (!pcmk_is_set(pe_wo, pe_wo_bit)) { \
	if (pe_wo_bit == pe_wo_blind) { \
	crm_warn(fmt); \
	} else { \
	pe_warn(fmt); \
	} \
	pe_wo = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Warn-once", "logging", pe_wo, \
	(pe_wo_bit), #pe_wo_bit); \
	} \
	} while (0);


	typedef struct pe__location_constraint_s {
	char *id; // Constraint XML ID
	pe_resource_t *rsc_lh; // Resource being located
	enum rsc_role_e role_filter; // Role to locate
	enum pe_discover_e discover_mode; // Resource discovery
	GList node_list_rh; // List of pe_node_t
	} pe__location_t;

	typedef struct pe__order_constraint_s {
	int id;
	enum pe_ordering type;

	void *lh_opaque;
	pe_resource_t *lh_rsc;
	pe_action_t *lh_action;
	char *lh_action_task;

	void *rh_opaque;
	pe_resource_t *rh_rsc;
	pe_action_t *rh_action;
	char *rh_action_task;
	} pe__ordering_t;

	typedef struct notify_data_s {
	GSList *keys; // Environment variable name/value pairs

	const char *action;

	pe_action_t *pre;
	pe_action_t *post;
	pe_action_t *pre_done;
	pe_action_t *post_done;

	GList active; / notify_entry_t* */
	GList inactive; / notify_entry_t* */
	GList start; / notify_entry_t* */
	GList stop; / notify_entry_t* */
	GList demote; / notify_entry_t* */
	GList promote; / notify_entry_t* */
	GList promoted; / notify_entry_t* */
	GList unpromoted; / notify_entry_t* */
	GHashTable *allowed_nodes;

	} notify_data_t;

	bool pe_can_fence(pe_working_set_t data_set, pe_node_t node);

	int pe__add_scores(int score1, int score2);
	void add_hash_param(GHashTable * hash, const char name, const char value);

	char native_parameter(pe_resource_t rsc, pe_node_t * node, gboolean create, const char *name,
	pe_working_set_t * data_set);
	pe_node_t native_location(const pe_resource_t rsc, GList **list, int current);

	void pe_metadata(void);
	void verify_pe_options(GHashTable * options);

	void common_update_score(pe_resource_t * rsc, const char *id, int score);
	void native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set);

	gboolean native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean group_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean pe__unpack_bundle(pe_resource_t rsc, pe_working_set_t data_set);

	pe_resource_t native_find_rsc(pe_resource_t rsc, const char id, const pe_node_t node,
	int flags);

	gboolean native_active(pe_resource_t * rsc, gboolean all);
	gboolean group_active(pe_resource_t * rsc, gboolean all);
	gboolean clone_active(pe_resource_t * rsc, gboolean all);
	gboolean pe__bundle_active(pe_resource_t *rsc, gboolean all);

	void native_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void group_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void clone_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void pe__print_bundle(pe_resource_t rsc, const char pre_text, long options,
	void *print_data);

	gchar * pcmk__native_output_string(pe_resource_t rsc, const char name, pe_node_t *node,
	long options, const char *target_role, bool show_nodes);

	int pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...);
	char pe__node_display_name(pe_node_t node, bool print_detail);

	static inline const char *
	pe__rsc_bool_str(pe_resource_t *rsc, uint64_t rsc_flag)
	{
	return pcmk__btoa(pcmk_is_set(rsc->flags, rsc_flag));
	}

	int pe__clone_xml(pcmk__output_t *out, va_list args);
	int pe__clone_html(pcmk__output_t *out, va_list args);
	int pe__clone_text(pcmk__output_t *out, va_list args);
	int pe__group_xml(pcmk__output_t *out, va_list args);
	int pe__group_html(pcmk__output_t *out, va_list args);
	int pe__group_text(pcmk__output_t *out, va_list args);
	int pe__bundle_xml(pcmk__output_t *out, va_list args);
	int pe__bundle_html(pcmk__output_t *out, va_list args);
	int pe__bundle_text(pcmk__output_t *out, va_list args);
	int pe__node_html(pcmk__output_t *out, va_list args);
	int pe__node_text(pcmk__output_t *out, va_list args);
	int pe__node_xml(pcmk__output_t *out, va_list args);
	int pe__resource_xml(pcmk__output_t *out, va_list args);
	int pe__resource_html(pcmk__output_t *out, va_list args);
	int pe__resource_text(pcmk__output_t *out, va_list args);

	/* Exported for crm_mon to reference */
	int pe__ban_text(pcmk__output_t *out, va_list args);
	int pe__cluster_counts_text(pcmk__output_t *out, va_list args);
	int pe__cluster_dc_text(pcmk__output_t *out, va_list args);
	int pe__cluster_maint_mode_text(pcmk__output_t *out, va_list args);
	int pe__cluster_options_text(pcmk__output_t *out, va_list args);
	int pe__cluster_stack_text(pcmk__output_t *out, va_list args);
	int pe__cluster_summary(pcmk__output_t *out, va_list args);
	int pe__cluster_times_text(pcmk__output_t *out, va_list args);
	int pe__failed_action_text(pcmk__output_t *out, va_list args);
	int pe__node_attribute_text(pcmk__output_t *out, va_list args);
	int pe__node_list_text(pcmk__output_t *out, va_list args);
	int pe__op_history_text(pcmk__output_t *out, va_list args);
	int pe__resource_history_text(pcmk__output_t *out, va_list args);
	int pe__ticket_text(pcmk__output_t *out, va_list args);

	void native_free(pe_resource_t * rsc);
	void group_free(pe_resource_t * rsc);
	void clone_free(pe_resource_t * rsc);
	void pe__free_bundle(pe_resource_t *rsc);

	enum rsc_role_e native_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e group_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e clone_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e pe__bundle_resource_state(const pe_resource_t *rsc,
	gboolean current);

	void pe__count_common(pe_resource_t *rsc);
	void pe__count_bundle(pe_resource_t *rsc);

	gboolean common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc, pe_resource_t * parent,
	pe_working_set_t * data_set);
	void common_free(pe_resource_t * rsc);

	pe_node_t pe__copy_node(const pe_node_t this_node);
	extern time_t get_effective_time(pe_working_set_t * data_set);

	/* Failure handling utilities (from failcounts.c) */

	// bit flags for fail count handling options
	enum pe_fc_flags_e {
	pe_fc_default = (1 << 0),
	pe_fc_effective = (1 << 1), // don't count expired failures
	pe_fc_fillers = (1 << 2), // if container, include filler failures in count
	};

	int pe_get_failcount(pe_node_t node, pe_resource_t rsc, time_t *last_failure,
	uint32_t flags, xmlNode *xml_op,
	pe_working_set_t *data_set);

	pe_action_t pe__clear_failcount(pe_resource_t rsc, pe_node_t *node,
	const char *reason,
	pe_working_set_t *data_set);

	/* Functions for finding/counting a resource's active nodes */

	pe_node_t pe__find_active_on(const pe_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);
	pe_node_t pe__find_active_requires(const pe_resource_t rsc,
	unsigned int *count);

	static inline pe_node_t *
	pe__current_node(const pe_resource_t *rsc)
	{
	return pe__find_active_on(rsc, NULL, NULL);
	}


	/* Binary like operators for lists of nodes */
	extern void node_list_exclude(GHashTable * list, GList *list2, gboolean merge_scores);

	GHashTable pe__node_list2table(GList list);

	static inline gpointer
	pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
	{
	if (hash) {
	return g_hash_table_lookup(hash, key);
	}
	return NULL;
	}

	extern pe_action_t get_pseudo_op(const char name, pe_working_set_t * data_set);
	extern gboolean order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order);

	/* Printing functions for debug */
	extern void print_str_str(gpointer key, gpointer value, gpointer user_data);
	extern void pe__output_node(pe_node_t * node, gboolean details, pcmk__output_t *out);

	void pe__show_node_weights_as(const char file, const char function,
	int line, bool to_log, pe_resource_t *rsc,
	const char comment, GHashTable nodes,
	pe_working_set_t *data_set);

	#define pe__show_node_weights(level, rsc, text, nodes, data_set) \
	pe__show_node_weights_as(__FILE__, __func__, __LINE__, \
	(level), (rsc), (text), (nodes), (data_set))

	/* Sorting functions */
	extern gint sort_rsc_priority(gconstpointer a, gconstpointer b);
	extern gint sort_rsc_index(gconstpointer a, gconstpointer b);

	extern xmlNode find_rsc_op_entry(pe_resource_t rsc, const char *key);

	extern pe_action_t custom_action(pe_resource_t rsc, char key, const char task, pe_node_t * on_node,
	gboolean optional, gboolean foo, pe_working_set_t * data_set);

	# define delete_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DELETE, 0)
	# define delete_action(rsc, node, optional) custom_action( \
	rsc, delete_key(rsc), CRMD_ACTION_DELETE, node, \
	optional, TRUE, data_set);

	# define stopped_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
	# define stopped_action(rsc, node, optional) custom_action( \
	rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node, \
	optional, TRUE, data_set);

	# define stop_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOP, 0)
	# define stop_action(rsc, node, optional) custom_action( \
	rsc, stop_key(rsc), CRMD_ACTION_STOP, node, \
	optional, TRUE, data_set);

	-# define reload_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_RELOAD, 0)
	+# define reload_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_RELOAD_AGENT, 0)
	# define start_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_START, 0)
	# define start_action(rsc, node, optional) custom_action( \
	rsc, start_key(rsc), CRMD_ACTION_START, node, \
	optional, TRUE, data_set)

	# define started_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STARTED, 0)
	# define started_action(rsc, node, optional) custom_action( \
	rsc, started_key(rsc), CRMD_ACTION_STARTED, node, \
	optional, TRUE, data_set)

	# define promote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
	# define promote_action(rsc, node, optional) custom_action( \
	rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node, \
	optional, TRUE, data_set)

	# define promoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
	# define promoted_action(rsc, node, optional) custom_action( \
	rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node, \
	optional, TRUE, data_set)

	# define demote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
	# define demote_action(rsc, node, optional) custom_action( \
	rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node, \
	optional, TRUE, data_set)

	# define demoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
	# define demoted_action(rsc, node, optional) custom_action( \
	rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node, \
	optional, TRUE, data_set)

	extern int pe_get_configured_timeout(pe_resource_t rsc, const char action,
	pe_working_set_t *data_set);

	extern pe_action_t find_first_action(GList input, const char uuid, const char task,
	pe_node_t * on_node);
	extern enum action_tasks get_complex_task(pe_resource_t * rsc, const char *name,
	gboolean allow_non_atomic);

	extern GList find_actions(GList input, const char key, const pe_node_t on_node);
	GList find_actions_exact(GList input, const char *key,
	const pe_node_t *on_node);
	extern GList find_recurring_actions(GList input, pe_node_t * not_on_node);
	GList pe__resource_actions(const pe_resource_t rsc, const pe_node_t *node,
	const char *task, bool require_node);

	extern void pe_free_action(pe_action_t * action);

	extern void resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
	pe_working_set_t * data_set);

	extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
	extern gboolean get_target_role(pe_resource_t * rsc, enum rsc_role_e *role);

	extern pe_resource_t find_clone_instance(pe_resource_t rsc, const char *sub_id,
	pe_working_set_t * data_set);

	extern void destroy_ticket(gpointer data);
	extern pe_ticket_t ticket_new(const char ticket_id, pe_working_set_t * data_set);

	// Resources for manipulating resource names
	const char pe_base_name_end(const char id);
	char clone_strip(const char last_rsc_id);
	char clone_zero(const char last_rsc_id);

	static inline bool
	pe_base_name_eq(pe_resource_t rsc, const char id)
	{
	if (id && rsc && rsc->id) {
	// Number of characters in rsc->id before any clone suffix
	size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;

	return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
	}
	return FALSE;
	}

	int pe__target_rc_from_xml(xmlNode *xml_op);

	gint sort_node_uname(gconstpointer a, gconstpointer b);
	bool is_set_recursive(pe_resource_t * rsc, long long flag, bool any);

	enum rsc_digest_cmp_val {
	/! Digests are the same /
	RSC_DIGEST_MATCH = 0,
	/! Params that require a restart changed /
	RSC_DIGEST_RESTART,
	/! Some parameter changed. /
	RSC_DIGEST_ALL,
	/*! rsc op didn't have a digest associated with it, so
	* it is unknown if parameters changed or not. */
	RSC_DIGEST_UNKNOWN,
	};

	typedef struct op_digest_cache_s {
	enum rsc_digest_cmp_val rc;
	xmlNode *params_all;
	xmlNode *params_secure;
	xmlNode *params_restart;
	char *digest_all_calc;
	char *digest_secure_calc;
	char *digest_restart_calc;
	} op_digest_cache_t;

	op_digest_cache_t pe__calculate_digests(pe_resource_t rsc, const char *task,
	guint interval_ms, pe_node_t node,
	xmlNode xml_op, GHashTable overrides,
	bool calc_secure,
	pe_working_set_t *data_set);

	void pe__free_digests(gpointer ptr);

	op_digest_cache_t rsc_action_digest_cmp(pe_resource_t rsc, xmlNode * xml_op, pe_node_t * node,
	pe_working_set_t * data_set);

	pe_action_t pe_fence_op(pe_node_t node, const char op, bool optional, const char reason, bool priority_delay, pe_working_set_t * data_set);
	void trigger_unfencing(
	pe_resource_t * rsc, pe_node_t node, const char reason, pe_action_t dependency, pe_working_set_t data_set);

	void pe_action_set_reason(pe_action_t action, const char reason, bool overwrite);
	void pe_action_set_flag_reason(const char function, long line, pe_action_t action, pe_action_t reason, const char text, enum pe_action_flags flags, bool overwrite);

	#define pe_action_required(action, reason, text) \
	pe_action_set_flag_reason(__func__, __LINE__, action, reason, text, \
	pe_action_optional, FALSE)
	#define pe_action_implies(action, reason, flag) \
	pe_action_set_flag_reason(__func__, __LINE__, action, reason, NULL, \
	flag, FALSE)

	void pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_on_all(pe_working_set_t *data_set, uint64_t flag);

	gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref);

	void print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void * print_data, gboolean print_all);
	int pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, long options, gboolean print_all);
	void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay);

	pe_node_t pe_create_node(const char id, const char uname, const char type,
	const char score, pe_working_set_t data_set);
	void common_print(pe_resource_t * rsc, const char pre_text, const char name, pe_node_t node, long options, void print_data);
	int pe__common_output_text(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, long options);
	int pe__common_output_html(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, long options);
	pe_resource_t pe__find_bundle_replica(const pe_resource_t bundle,
	const pe_node_t *node);
	bool pe__bundle_needs_remote_name(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	const char pe__add_bundle_remote_name(pe_resource_t rsc,
	pe_working_set_t *data_set,
	xmlNode xml, const char field);
	const char pe_node_attribute_calculated(const pe_node_t node,
	const char *name,
	const pe_resource_t *rsc);
	const char pe_node_attribute_raw(pe_node_t node, const char *name);
	bool pe__is_universal_clone(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	void pe__add_param_check(xmlNode rsc_op, pe_resource_t rsc, pe_node_t *node,
	enum pe_check_parameters, pe_working_set_t *data_set);
	void pe__foreach_param_check(pe_working_set_t *data_set,
	void (cb)(pe_resource_t, pe_node_t, xmlNode,
	enum pe_check_parameters,
	pe_working_set_t*));
	void pe__free_param_checks(pe_working_set_t *data_set);

	bool pe__shutdown_requested(pe_node_t *node);
	void pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set);

	/*!
	* \internal
	* \brief Register xml formatting message functions.
	*/
	void pe__register_messages(pcmk__output_t *out);

	void pe__unpack_dataset_nvpairs(xmlNode xml_obj, const char set_name,
	pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	pe_working_set_t *data_set);

	bool pe__resource_is_disabled(pe_resource_t *rsc);
	pe_action_t pe__clear_resource_history(pe_resource_t rsc, pe_node_t *node,
	pe_working_set_t *data_set);

	GList pe__rscs_with_tag(pe_working_set_t data_set, const char *tag_name);
	GList pe__unames_with_tag(pe_working_set_t data_set, const char *tag_name);
	bool pe__rsc_has_tag(pe_working_set_t data_set, const char rsc, const char *tag);
	bool pe__uname_has_tag(pe_working_set_t data_set, const char node, const char *tag);

	bool pe__rsc_running_on_any_node_in_list(pe_resource_t rsc, GList node_list);
	GList pe__filter_rsc_list(GList rscs, GList *filter);

	bool pcmk__rsc_filtered_by_node(pe_resource_t rsc, GList only_node);

	gboolean pe__bundle_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__clone_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__group_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__native_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);

	#endif
	diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c
	index 73c23b8fd0..f51955b666 100644
	--- a/lib/pacemaker/pcmk_sched_allocate.c
	+++ b/lib/pacemaker/pcmk_sched_allocate.c
	@@ -1,3092 +1,3093 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	CRM_TRACE_INIT_DATA(pacemaker);

	extern bool pcmk__is_daemon;

	void set_alloc_actions(pe_working_set_t * data_set);
	extern void ReloadRsc(pe_resource_t * rsc, pe_node_t node, pe_working_set_t data_set);
	extern gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
	static void apply_remote_node_ordering(pe_working_set_t *data_set);
	static enum remote_connection_state get_remote_node_state(pe_node_t *node);

	enum remote_connection_state {
	remote_state_unknown = 0,
	remote_state_alive = 1,
	remote_state_resting = 2,
	remote_state_failed = 3,
	remote_state_stopped = 4
	};

	static const char *
	state2text(enum remote_connection_state state)
	{
	switch (state) {
	case remote_state_unknown:
	return "unknown";
	case remote_state_alive:
	return "alive";
	case remote_state_resting:
	return "resting";
	case remote_state_failed:
	return "failed";
	case remote_state_stopped:
	return "stopped";
	}

	return "impossible";
	}

	resource_alloc_functions_t resource_class_alloc_functions[] = {
	{
	pcmk__native_merge_weights,
	pcmk__native_allocate,
	native_create_actions,
	native_create_probe,
	native_internal_constraints,
	native_rsc_colocation_lh,
	native_rsc_colocation_rh,
	native_rsc_location,
	native_action_flags,
	native_update_actions,
	native_expand,
	native_append_meta,
	},
	{
	pcmk__group_merge_weights,
	pcmk__group_allocate,
	group_create_actions,
	native_create_probe,
	group_internal_constraints,
	group_rsc_colocation_lh,
	group_rsc_colocation_rh,
	group_rsc_location,
	group_action_flags,
	group_update_actions,
	group_expand,
	group_append_meta,
	},
	{
	pcmk__native_merge_weights,
	pcmk__clone_allocate,
	clone_create_actions,
	clone_create_probe,
	clone_internal_constraints,
	clone_rsc_colocation_lh,
	clone_rsc_colocation_rh,
	clone_rsc_location,
	clone_action_flags,
	pcmk__multi_update_actions,
	clone_expand,
	clone_append_meta,
	},
	{
	pcmk__native_merge_weights,
	pcmk__bundle_allocate,
	pcmk__bundle_create_actions,
	pcmk__bundle_create_probe,
	pcmk__bundle_internal_constraints,
	pcmk__bundle_rsc_colocation_lh,
	pcmk__bundle_rsc_colocation_rh,
	pcmk__bundle_rsc_location,
	pcmk__bundle_action_flags,
	pcmk__multi_update_actions,
	pcmk__bundle_expand,
	pcmk__bundle_append_meta,
	}
	};

	gboolean
	update_action_flags(pe_action_t * action, enum pe_action_flags flags, const char *source, int line)
	{
	static unsigned long calls = 0;
	gboolean changed = FALSE;
	gboolean clear = pcmk_is_set(flags, pe_action_clear);
	enum pe_action_flags last = action->flags;

	if (clear) {
	pe__clear_action_flags_as(source, line, action, flags);
	} else {
	pe__set_action_flags_as(source, line, action, flags);
	}

	if (last != action->flags) {
	calls++;
	changed = TRUE;
	/* Useful for tracking down _who_ changed a specific flag */
	/* CRM_ASSERT(calls != 534); */
	pe__clear_raw_action_flags(flags, "action update", pe_action_clear);
	crm_trace("%s on %s: %sset flags 0x%.6x (was 0x%.6x, now 0x%.6x, %lu, %s)",
	action->uuid, action->node ? action->node->details->uname : "[none]",
	clear ? "un-" : "", flags, last, action->flags, calls, source);
	}

	return changed;
	}

	static gboolean
	check_rsc_parameters(pe_resource_t * rsc, pe_node_t * node, xmlNode * rsc_entry,
	gboolean active_here, pe_working_set_t * data_set)
	{
	int attr_lpc = 0;
	gboolean force_restart = FALSE;
	gboolean delete_resource = FALSE;
	gboolean changed = FALSE;

	const char *value = NULL;
	const char *old_value = NULL;

	const char *attr_list[] = {
	XML_ATTR_TYPE,
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER
	};

	for (; attr_lpc < PCMK__NELEM(attr_list); attr_lpc++) {
	value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
	old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
	if (value == old_value /* i.e. NULL */
	\|\| pcmk__str_eq(value, old_value, pcmk__str_none)) {
	continue;
	}

	changed = TRUE;
	trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
	if (active_here) {
	force_restart = TRUE;
	crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
	rsc->id, node->details->uname, attr_list[attr_lpc],
	crm_str(old_value), crm_str(value));
	}
	}
	if (force_restart) {
	/* make sure the restart happens */
	stop_action(rsc, node, FALSE);
	pe__set_resource_flags(rsc, pe_rsc_start_pending);
	delete_resource = TRUE;

	} else if (changed) {
	delete_resource = TRUE;
	}
	return delete_resource;
	}

	static void
	CancelXmlOp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * active_node,
	const char reason, pe_working_set_t data_set)
	{
	guint interval_ms = 0;
	pe_action_t *cancel = NULL;

	const char *task = NULL;
	const char *call_id = NULL;

	CRM_CHECK(xml_op != NULL, return);
	CRM_CHECK(active_node != NULL, return);

	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	crm_info("Action " PCMK__OP_FMT " on %s will be stopped: %s",
	rsc->id, task, interval_ms,
	active_node->details->uname, (reason? reason : "unknown"));

	cancel = pe_cancel_op(rsc, task, interval_ms, active_node, data_set);
	add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
	custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
	}

	static gboolean
	check_action_definition(pe_resource_t * rsc, pe_node_t * active_node, xmlNode * xml_op,
	pe_working_set_t * data_set)
	{
	char *key = NULL;
	guint interval_ms = 0;
	const op_digest_cache_t *digest_data = NULL;
	gboolean did_change = FALSE;

	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *digest_secure = NULL;

	CRM_CHECK(active_node != NULL, return FALSE);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	if (interval_ms > 0) {
	xmlNode *op_match = NULL;

	/* we need to reconstruct the key because of the way we used to construct resource IDs */
	key = pcmk__op_key(rsc->id, task, interval_ms);

	pe_rsc_trace(rsc, "Checking parameters for %s", key);
	op_match = find_rsc_op_entry(rsc, key);

	if ((op_match == NULL)
	&& pcmk_is_set(data_set->flags, pe_flag_stop_action_orphans)) {
	CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
	free(key);
	return TRUE;

	} else if (op_match == NULL) {
	pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
	free(key);
	return TRUE;
	}
	free(key);
	key = NULL;
	}

	crm_trace("Testing " PCMK__OP_FMT " on %s",
	rsc->id, task, interval_ms, active_node->details->uname);
	if ((interval_ms == 0) && pcmk__str_eq(task, RSC_STATUS, pcmk__str_casei)) {
	/* Reload based on the start action not a probe */
	task = RSC_START;

	} else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_MIGRATED, pcmk__str_casei)) {
	/* Reload based on the start action not a migrate */
	task = RSC_START;
	} else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_PROMOTE, pcmk__str_casei)) {
	/* Reload based on the start action not a promote */
	task = RSC_START;
	}

	digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_sanitized)) {
	digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
	}

	if(digest_data->rc != RSC_DIGEST_MATCH
	&& digest_secure
	&& digest_data->digest_secure_calc
	&& strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
	if (!pcmk__is_daemon && data_set->priv != NULL) {
	pcmk__output_t *out = data_set->priv;
	out->info(out, "Only 'private' parameters to "
	PCMK__OP_FMT " on %s changed: %s", rsc->id, task,
	interval_ms, active_node->details->uname,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	}

	} else if (digest_data->rc == RSC_DIGEST_RESTART) {
	/* Changes that force a restart */
	pe_action_t *required = NULL;

	did_change = TRUE;
	key = pcmk__op_key(rsc->id, task, interval_ms);
	crm_log_xml_info(digest_data->params_restart, "params:restart");
	required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
	pe_action_set_flag_reason(__func__, __LINE__, required, NULL,
	"resource definition change", pe_action_optional, TRUE);

	trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set);

	} else if ((digest_data->rc == RSC_DIGEST_ALL) \|\| (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
	- /* Changes that can potentially be handled by a reload */
	+ // Changes that can potentially be handled by an agent reload
	const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	did_change = TRUE;
	trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set);
	crm_log_xml_info(digest_data->params_all, "params:reload");
	key = pcmk__op_key(rsc->id, task, interval_ms);

	if (interval_ms > 0) {
	pe_action_t *op = NULL;

	#if 0
	/* Always reload/restart the entire resource */
	ReloadRsc(rsc, active_node, data_set);
	#else
	/* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
	op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_reschedule);
	#endif

	} else if (digest_restart) {
	pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);

	/* Reload this resource */
	ReloadRsc(rsc, active_node, data_set);
	free(key);

	} else {
	pe_action_t *required = NULL;
	- pe_rsc_trace(rsc, "Resource %s doesn't know how to reload", rsc->id);
	+ pe_rsc_trace(rsc, "Resource %s doesn't support agent reloads",
	+ rsc->id);

	/* Re-send the start/demote/promote op
	* Recurring ops will be detected independently
	*/
	required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
	pe_action_set_flag_reason(__func__, __LINE__, required, NULL,
	"resource definition change", pe_action_optional, TRUE);
	}
	}

	return did_change;
	}

	/*!
	* \internal
	* \brief Do deferred action checks after allocation
	*
	* \param[in] data_set Working set for cluster
	*/
	static void
	check_params(pe_resource_t rsc, pe_node_t node, xmlNode *rsc_op,
	enum pe_check_parameters check, pe_working_set_t *data_set)
	{
	const char *reason = NULL;
	op_digest_cache_t *digest_data = NULL;

	switch (check) {
	case pe_check_active:
	if (check_action_definition(rsc, node, rsc_op, data_set)
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {

	reason = "action definition changed";
	}
	break;

	case pe_check_last_failure:
	digest_data = rsc_action_digest_cmp(rsc, rsc_op, node, data_set);
	switch (digest_data->rc) {
	case RSC_DIGEST_UNKNOWN:
	crm_trace("Resource %s history entry %s on %s has no digest to compare",
	rsc->id, ID(rsc_op), node->details->id);
	break;
	case RSC_DIGEST_MATCH:
	break;
	default:
	reason = "resource parameters have changed";
	break;
	}
	break;
	}

	if (reason) {
	pe__clear_failcount(rsc, node, reason, data_set);
	}
	}

	static void
	check_actions_for(xmlNode * rsc_entry, pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	int offset = -1;
	int stop_index = 0;
	int start_index = 0;

	const char *task = NULL;

	xmlNode *rsc_op = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	CRM_CHECK(node != NULL, return);

	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_resource_t *parent = uber_parent(rsc);
	if(parent == NULL
	\|\| pe_rsc_is_clone(parent) == FALSE
	\|\| pcmk_is_set(parent->flags, pe_rsc_unique)) {
	pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
	DeleteRsc(rsc, node, FALSE, data_set);
	} else {
	pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
	}
	return;

	} else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
	if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
	DeleteRsc(rsc, node, FALSE, data_set);
	}
	pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
	rsc->id, node->details->uname);
	return;
	}

	pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);

	if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
	DeleteRsc(rsc, node, FALSE, data_set);
	}

	for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
	rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;
	guint interval_ms = 0;

	offset++;

	if (start_index < stop_index) {
	/* stopped */
	continue;
	} else if (offset < start_index) {
	/* action occurred prior to a start */
	continue;
	}

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	if ((interval_ms > 0) &&
	(pcmk_is_set(rsc->flags, pe_rsc_maintenance) \|\| node->details->maintenance)) {
	// Maintenance mode cancels recurring operations
	CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);

	} else if ((interval_ms > 0) \|\| pcmk__strcase_any_of(task, RSC_STATUS, RSC_START,
	RSC_PROMOTE, RSC_MIGRATED, NULL)) {
	/* If a resource operation failed, and the operation's definition
	* has changed, clear any fail count so they can be retried fresh.
	*/

	if (pe__bundle_needs_remote_name(rsc, data_set)) {
	/* We haven't allocated resources to nodes yet, so if the
	* REMOTE_CONTAINER_HACK is used, we may calculate the digest
	* based on the literal "#uname" value rather than the properly
	* substituted value. That would mistakenly make the action
	* definition appear to have been changed. Defer the check until
	* later in this case.
	*/
	pe__add_param_check(rsc_op, rsc, node, pe_check_active,
	data_set);

	} else if (check_action_definition(rsc, node, rsc_op, data_set)
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {
	pe__clear_failcount(rsc, node, "action definition changed",
	data_set);
	}
	}
	}
	g_list_free(sorted_op_list);
	}

	static GList *
	find_rsc_list(GList result, pe_resource_t rsc, const char *id, gboolean renamed_clones,
	gboolean partial, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	gboolean match = FALSE;

	if (id == NULL) {
	return NULL;
	}

	if (rsc == NULL) {
	if (data_set == NULL) {
	return NULL;
	}
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	result = find_rsc_list(result, child, id, renamed_clones, partial,
	NULL);
	}
	return result;
	}

	if (partial) {
	if (strstr(rsc->id, id)) {
	match = TRUE;

	} else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
	match = TRUE;
	}

	} else {
	if (strcmp(rsc->id, id) == 0) {
	match = TRUE;

	} else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = TRUE;
	}
	}

	if (match) {
	result = g_list_prepend(result, rsc);
	}

	if (rsc->children) {
	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
	}
	}

	return result;
	}

	static void
	check_actions(pe_working_set_t * data_set)
	{
	const char *id = NULL;
	pe_node_t *node = NULL;
	xmlNode *lrm_rscs = NULL;
	xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);

	xmlNode *node_state = NULL;

	for (node_state = pcmk__xe_first_child(status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {

	if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE,
	pcmk__str_none)) {
	id = crm_element_value(node_state, XML_ATTR_ID);
	lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);

	node = pe_find_node_id(data_set->nodes, id);

	if (node == NULL) {
	continue;

	/* Still need to check actions for a maintenance node to cancel existing monitor operations */
	} else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
	crm_trace("Skipping param check for %s: can't run resources",
	node->details->uname);
	continue;
	}

	crm_trace("Processing node %s", node->details->uname);
	if (node->details->online
	\|\| pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	xmlNode *rsc_entry = NULL;

	for (rsc_entry = pcmk__xe_first_child(lrm_rscs);
	rsc_entry != NULL;
	rsc_entry = pcmk__xe_next(rsc_entry)) {

	if (pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	if (xml_has_children(rsc_entry)) {
	GList *gIter = NULL;
	GList *result = NULL;
	const char *rsc_id = ID(rsc_entry);

	CRM_CHECK(rsc_id != NULL, return);

	result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
	for (gIter = result; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	if (rsc->variant != pe_native) {
	continue;
	}
	check_actions_for(rsc_entry, rsc, node, data_set);
	}
	g_list_free(result);
	}
	}
	}
	}
	}
	}
	}

	static void
	apply_placement_constraints(pe_working_set_t * data_set)
	{
	for (GList *gIter = data_set->placement_constraints;
	gIter != NULL; gIter = gIter->next) {
	pe__location_t *cons = gIter->data;

	cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
	}
	}

	static gboolean
	failcount_clear_action_exists(pe_node_t * node, pe_resource_t * rsc)
	{
	gboolean rc = FALSE;
	GList *list = pe__resource_actions(rsc, node, CRM_OP_CLEAR_FAILCOUNT, TRUE);

	if (list) {
	rc = TRUE;
	}
	g_list_free(list);
	return rc;
	}

	/*!
	* \internal
	* \brief Force resource away if failures hit migration threshold
	*
	* \param[in,out] rsc Resource to check for failures
	* \param[in,out] node Node to check for failures
	* \param[in,out] data_set Cluster working set to update
	*/
	static void
	check_migration_threshold(pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set)
	{
	int fail_count, countdown;
	pe_resource_t *failed;

	/* Migration threshold of 0 means never force away */
	if (rsc->migration_threshold == 0) {
	return;
	}

	// If we're ignoring failures, also ignore the migration threshold
	if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
	return;
	}

	/* If there are no failures, there's no need to force away */
	fail_count = pe_get_failcount(node, rsc, NULL,
	pe_fc_effective\|pe_fc_fillers, NULL,
	data_set);
	if (fail_count <= 0) {
	return;
	}

	/* How many more times recovery will be tried on this node */
	countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);

	/* If failed resource has a parent, we'll force the parent away */
	failed = rsc;
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	failed = uber_parent(rsc);
	}

	if (countdown == 0) {
	resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
	crm_warn("Forcing %s away from %s after %d failures (max=%d)",
	failed->id, node->details->uname, fail_count,
	rsc->migration_threshold);
	} else {
	crm_info("%s can fail %d more times on %s before being forced off",
	failed->id, countdown, node->details->uname);
	}
	}

	static void
	common_apply_stickiness(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	common_apply_stickiness(child_rsc, node, data_set);
	}
	return;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)
	&& rsc->stickiness != 0 && pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
	pe_node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (current == NULL) {

	} else if ((match != NULL)
	\|\| pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	pe_resource_t *sticky_rsc = rsc;

	resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
	pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
	" (node=%s, weight=%d)", sticky_rsc->id,
	node->details->uname, rsc->stickiness);
	} else {
	GHashTableIter iter;
	pe_node_t *nIter = NULL;

	pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
	" and node %s is not explicitly allowed", rsc->id, node->details->uname);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
	crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
	}
	}
	}

	/* Check the migration threshold only if a failcount clear action
	* has not already been placed for this resource on the node.
	* There is no sense in potentially forcing the resource from this
	* node if the failcount is being reset anyway.
	*
	* @TODO A clear_failcount operation can be scheduled in stage4() via
	* check_actions_for(), or in stage5() via check_params(). This runs in
	* stage2(), so it cannot detect those, meaning we might check the migration
	* threshold when we shouldn't -- worst case, we stop or move the resource,
	* then move it back next transition.
	*/
	if (failcount_clear_action_exists(node, rsc) == FALSE) {
	check_migration_threshold(rsc, node, data_set);
	}
	}

	void
	complex_set_cmds(pe_resource_t * rsc)
	{
	GList *gIter = rsc->children;

	rsc->cmds = &resource_class_alloc_functions[rsc->variant];

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	complex_set_cmds(child_rsc);
	}
	}

	void
	set_alloc_actions(pe_working_set_t * data_set)
	{

	GList *gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	complex_set_cmds(rsc);
	}
	}

	static void
	calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
	{
	const char key = (const char )gKey;
	const char value = (const char )gValue;
	int system_health = (int )user_data;

	if (!gKey \|\| !gValue \|\| !user_data) {
	return;
	}

	if (pcmk__starts_with(key, "#health")) {
	int score;

	/* Convert the value into an integer */
	score = char2score(value);

	/* Add it to the running total */
	system_health = pe__add_scores(score, system_health);
	}
	}

	static gboolean
	apply_system_health(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
	int base_health = 0;

	if (pcmk__str_eq(health_strategy, "none", pcmk__str_null_matches \| pcmk__str_casei)) {
	/* Prevent any accidental health -> score translation */
	pcmk__score_red = 0;
	pcmk__score_yellow = 0;
	pcmk__score_green = 0;
	return TRUE;

	} else if (pcmk__str_eq(health_strategy, "migrate-on-red", pcmk__str_casei)) {

	/* Resources on nodes which have health values of red are
	* weighted away from that node.
	*/
	pcmk__score_red = -INFINITY;
	pcmk__score_yellow = 0;
	pcmk__score_green = 0;

	} else if (pcmk__str_eq(health_strategy, "only-green", pcmk__str_casei)) {

	/* Resources on nodes which have health values of red or yellow
	* are forced away from that node.
	*/
	pcmk__score_red = -INFINITY;
	pcmk__score_yellow = -INFINITY;
	pcmk__score_green = 0;

	} else if (pcmk__str_eq(health_strategy, "progressive", pcmk__str_casei)) {
	/* Same as the above, but use the r/y/g scores provided by the user
	* Defaults are provided by the pe_prefs table
	* Also, custom health "base score" can be used
	*/
	base_health = char2score(pe_pref(data_set->config_hash,
	"node-health-base"));

	} else if (pcmk__str_eq(health_strategy, "custom", pcmk__str_casei)) {

	/* Requires the admin to configure the rsc_location constaints for
	* processing the stored health scores
	*/
	/* TODO: Check for the existence of appropriate node health constraints */
	return TRUE;

	} else {
	crm_err("Unknown node health strategy: %s", health_strategy);
	return FALSE;
	}

	crm_info("Applying automated node health strategy: %s", health_strategy);

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	int system_health = base_health;
	pe_node_t node = (pe_node_t ) gIter->data;

	/* Search through the node hash table for system health entries. */
	g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);

	crm_info(" Node %s has an combined system health of %d",
	node->details->uname, system_health);

	/* If the health is non-zero, then create a new rsc2node so that the
	* weight will be added later on.
	*/
	if (system_health != 0) {

	GList *gIter2 = data_set->resources;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
	}
	}
	}

	return TRUE;
	}

	gboolean
	stage0(pe_working_set_t * data_set)
	{
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	if (data_set->input == NULL) {
	return FALSE;
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_have_status)) {
	crm_trace("Calculating status");
	cluster_status(data_set);
	}

	set_alloc_actions(data_set);
	apply_system_health(data_set);
	unpack_constraints(cib_constraints, data_set);

	return TRUE;
	}

	/*
	* Check nodes for resources started outside of the LRM
	*/
	gboolean
	probe_resources(pe_working_set_t * data_set)
	{
	pe_action_t *probe_node_complete = NULL;

	for (GList *gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;
	const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED);

	if (node->details->online == FALSE) {

	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& (get_remote_node_state(node) == remote_state_failed)) {

	pe_fence_node(data_set, node, "the connection is unrecoverable", FALSE);
	}
	continue;

	} else if (node->details->unclean) {
	continue;

	} else if (node->details->rsc_discovery_enabled == FALSE) {
	/* resource discovery is disabled for this node */
	continue;
	}

	if (probed != NULL && crm_is_true(probed) == FALSE) {
	pe_action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
	CRM_OP_REPROBE, node, FALSE, TRUE, data_set);

	add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	continue;
	}

	for (GList *gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
	}
	}
	return TRUE;
	}

	static void
	rsc_discover_filter(pe_resource_t rsc, pe_node_t node)
	{
	GList *gIter = rsc->children;
	pe_resource_t *top = uber_parent(rsc);
	pe_node_t *match;

	if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
	return;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	rsc_discover_filter(child_rsc, node);
	}

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match && match->rsc_discover_mode != pe_discover_exclusive) {
	match->weight = -INFINITY;
	}
	}

	static time_t
	shutdown_time(pe_node_t node, pe_working_set_t data_set)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
	time_t result = 0;

	if (shutdown) {
	long long result_ll;

	if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
	result = (time_t) result_ll;
	}
	}
	return result? result : get_effective_time(data_set);
	}

	static void
	apply_shutdown_lock(pe_resource_t rsc, pe_working_set_t data_set)
	{
	const char *class;

	// Only primitives and (uncloned) groups may be locked
	if (rsc->variant == pe_group) {
	for (GList *item = rsc->children; item != NULL;
	item = item->next) {
	apply_shutdown_lock((pe_resource_t *) item->data, data_set);
	}
	} else if (rsc->variant != pe_native) {
	return;
	}

	// Fence devices and remote connections can't be locked
	class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
	\|\| pe__resource_is_remote_conn(rsc, data_set)) {
	return;
	}

	if (rsc->lock_node != NULL) {
	// The lock was obtained from resource history

	if (rsc->running_on != NULL) {
	/* The resource was started elsewhere even though it is now
	* considered locked. This shouldn't be possible, but as a
	* failsafe, we don't want to disturb the resource now.
	*/
	pe_rsc_info(rsc,
	"Cancelling shutdown lock because %s is already active",
	rsc->id);
	pe__clear_resource_history(rsc, rsc->lock_node, data_set);
	rsc->lock_node = NULL;
	rsc->lock_time = 0;
	}

	// Only a resource active on exactly one node can be locked
	} else if (pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *node = rsc->running_on->data;

	if (node->details->shutdown) {
	if (node->details->unclean) {
	pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
	rsc->id, node->details->uname);
	} else {
	rsc->lock_node = node;
	rsc->lock_time = shutdown_time(node, data_set);
	}
	}
	}

	if (rsc->lock_node == NULL) {
	// No lock needed
	return;
	}

	if (data_set->shutdown_lock > 0) {
	time_t lock_expiration = rsc->lock_time + data_set->shutdown_lock;

	pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
	rsc->id, rsc->lock_node->details->uname,
	(long long) lock_expiration);
	pe__update_recheck_time(++lock_expiration, data_set);
	} else {
	pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
	rsc->id, rsc->lock_node->details->uname);
	}

	// If resource is locked to one node, ban it from all other nodes
	for (GList *item = data_set->nodes; item != NULL; item = item->next) {
	pe_node_t *node = item->data;

	if (strcmp(node->details->uname, rsc->lock_node->details->uname)) {
	resource_location(rsc, node, -CRM_SCORE_INFINITY,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK, data_set);
	}
	}
	}

	/*
	* \internal
	* \brief Stage 2 of cluster status: apply node-specific criteria
	*
	* Count known nodes, and apply location constraints, stickiness, and exclusive
	* resource discovery.
	*/
	gboolean
	stage2(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	apply_shutdown_lock((pe_resource_t *) gIter->data, data_set);
	}
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_no_compat)) {
	// @COMPAT API backward compatibility
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (node && (node->weight >= 0) && node->details->online
	&& (node->details->type != node_ping)) {
	data_set->max_valid_nodes++;
	}
	}
	}

	apply_placement_constraints(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	GList *gIter2 = NULL;
	pe_node_t node = (pe_node_t ) gIter->data;

	gIter2 = data_set->resources;
	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	common_apply_stickiness(rsc, node, data_set);
	rsc_discover_filter(rsc, node);
	}
	}

	return TRUE;
	}

	/*
	* Create internal resource constraints before allocation
	*/
	gboolean
	stage3(pe_working_set_t * data_set)
	{

	GList *gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	rsc->cmds->internal_constraints(rsc, data_set);
	}

	return TRUE;
	}

	/*
	* Check for orphaned or redefined actions
	*/
	gboolean
	stage4(pe_working_set_t * data_set)
	{
	check_actions(data_set);
	return TRUE;
	}

	static void *
	convert_const_pointer(const void *ptr)
	{
	/* Worst function ever */
	return (void *)ptr;
	}

	static gint
	sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
	{
	int rc = 0;
	int r1_weight = -INFINITY;
	int r2_weight = -INFINITY;

	const char *reason = "existence";

	GList nodes = (GList ) data;
	const pe_resource_t *resource1 = a;
	const pe_resource_t *resource2 = b;

	pe_node_t *r1_node = NULL;
	pe_node_t *r2_node = NULL;
	GList *gIter = NULL;
	GHashTable *r1_nodes = NULL;
	GHashTable *r2_nodes = NULL;

	reason = "priority";
	r1_weight = resource1->priority;
	r2_weight = resource2->priority;

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}

	reason = "no node list";
	if (nodes == NULL) {
	goto done;
	}

	r1_nodes = pcmk__native_merge_weights(convert_const_pointer(resource1),
	resource1->id, NULL, NULL, 1,
	pe_weights_forward \| pe_weights_init);
	pe__show_node_weights(true, NULL, resource1->id, r1_nodes,
	resource1->cluster);

	r2_nodes = pcmk__native_merge_weights(convert_const_pointer(resource2),
	resource2->id, NULL, NULL, 1,
	pe_weights_forward \| pe_weights_init);
	pe__show_node_weights(true, NULL, resource2->id, r2_nodes,
	resource2->cluster);

	/* Current location score */
	reason = "current location";
	r1_weight = -INFINITY;
	r2_weight = -INFINITY;

	if (resource1->running_on) {
	r1_node = pe__current_node(resource1);
	r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
	if (r1_node != NULL) {
	r1_weight = r1_node->weight;
	}
	}
	if (resource2->running_on) {
	r2_node = pe__current_node(resource2);
	r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
	if (r2_node != NULL) {
	r2_weight = r2_node->weight;
	}
	}

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}

	reason = "score";
	for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	r1_node = NULL;
	r2_node = NULL;

	r1_weight = -INFINITY;
	if (r1_nodes) {
	r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
	}
	if (r1_node) {
	r1_weight = r1_node->weight;
	}

	r2_weight = -INFINITY;
	if (r2_nodes) {
	r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
	}
	if (r2_node) {
	r2_weight = r2_node->weight;
	}

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}
	}

	done:
	crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
	resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
	rc < 0 ? '>' : rc > 0 ? '<' : '=',
	resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);

	if (r1_nodes) {
	g_hash_table_destroy(r1_nodes);
	}
	if (r2_nodes) {
	g_hash_table_destroy(r2_nodes);
	}

	return rc;
	}

	static void
	allocate_resources(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	if (pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	/* Allocate remote connection resources first (which will also allocate
	* any colocation dependencies). If the connection is migrating, always
	* prefer the partial migration target.
	*/
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;
	if (rsc->is_remote_node == FALSE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating remote connection resource '%s'",
	rsc->id);
	rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
	}
	}

	/* now do the rest of the resources */
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;
	if (rsc->is_remote_node == TRUE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating %s resource '%s'",
	crm_element_name(rsc->xml), rsc->id);
	rsc->cmds->allocate(rsc, NULL, data_set);
	}
	}

	/* We always use pe_order_preserve with these convenience functions to exempt
	* internally generated constraints from the prohibition of user constraints
	* involving remote connection resources.
	*
	* The start ordering additionally uses pe_order_runnable_left so that the
	* specified action is not runnable if the start is not runnable.
	*/

	static inline void
	order_start_then_action(pe_resource_t lh_rsc, pe_action_t rh_action,
	enum pe_ordering extra, pe_working_set_t *data_set)
	{
	if (lh_rsc && rh_action && data_set) {
	custom_action_order(lh_rsc, start_key(lh_rsc), NULL,
	rh_action->rsc, NULL, rh_action,
	pe_order_preserve \| pe_order_runnable_left \| extra,
	data_set);
	}
	}

	static inline void
	order_action_then_stop(pe_action_t lh_action, pe_resource_t rh_rsc,
	enum pe_ordering extra, pe_working_set_t *data_set)
	{
	if (lh_action && rh_rsc && data_set) {
	custom_action_order(lh_action->rsc, NULL, lh_action,
	rh_rsc, stop_key(rh_rsc), NULL,
	pe_order_preserve \| extra, data_set);
	}
	}

	// Clear fail counts for orphaned rsc on all online nodes
	static void
	cleanup_orphans(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (node->details->online
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {

	pe_action_t *clear_op = NULL;

	clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
	data_set);

	/* We can't use order_action_then_stop() here because its
	* pe_order_preserve breaks things
	*/
	custom_action_order(clear_op->rsc, NULL, clear_op,
	rsc, stop_key(rsc), NULL,
	pe_order_optional, data_set);
	}
	}
	}

	gboolean
	stage5(pe_working_set_t * data_set)
	{
	pcmk__output_t *out = data_set->priv;
	GList *gIter = NULL;

	if (!pcmk__str_eq(data_set->placement_strategy, "default", pcmk__str_casei)) {
	GList *nodes = g_list_copy(data_set->nodes);

	nodes = sort_nodes_by_weight(nodes, NULL, data_set);
	data_set->resources =
	g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);

	g_list_free(nodes);
	}

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	out->message(out, "node-capacity", node, "Original");
	}
	}

	crm_trace("Allocating services");
	/* Take (next) highest resource, assign it and create its actions */

	allocate_resources(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	out->message(out, "node-capacity", node, "Remaining");
	}
	}

	// Process deferred action checks
	pe__foreach_param_check(data_set, check_params);
	pe__free_param_checks(data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
	crm_trace("Calculating needed probes");
	/* This code probably needs optimization
	* ptest -x with 100 nodes, 100 clones and clone-max=100:

	With probes:

	ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
	ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
	ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
	ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
	ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
	ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
	ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
	ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
	36s
	ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph

	Without probes:

	ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
	ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
	ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
	ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
	*/

	probe_resources(data_set);
	}

	crm_trace("Handle orphans");
	if (pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* There's no need to recurse into rsc->children because those
	* should just be unallocated clone instances.
	*/
	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	cleanup_orphans(rsc, data_set);
	}
	}
	}

	crm_trace("Creating actions");

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	rsc->cmds->create_actions(rsc, data_set);
	}

	crm_trace("Creating done");
	return TRUE;
	}

	static gboolean
	is_managed(const pe_resource_t * rsc)
	{
	GList *gIter = rsc->children;

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	return TRUE;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (is_managed(child_rsc)) {
	return TRUE;
	}
	}

	return FALSE;
	}

	static gboolean
	any_managed_resources(pe_working_set_t * data_set)
	{

	GList *gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	if (is_managed(rsc)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Create pseudo-op for guest node fence, and order relative to it
	*
	* \param[in] node Guest node to fence
	* \param[in] data_set Working set of CIB state
	*/
	static void
	fence_guest(pe_node_t node, pe_working_set_t data_set)
	{
	pe_resource_t *container = node->details->remote_rsc->container;
	pe_action_t *stop = NULL;
	pe_action_t *stonith_op = NULL;

	/* The fence action is just a label; we don't do anything differently for
	* off vs. reboot. We specify it explicitly, rather than let it default to
	* cluster's default action, because we are not _initiating_ fencing -- we
	* are creating a pseudo-event to describe fencing that is already occurring
	* by other means (container recovery).
	*/
	const char *fence_action = "off";

	/* Check whether guest's container resource has any explicit stop or
	* start (the stop may be implied by fencing of the guest's host).
	*/
	if (container) {
	stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL);

	if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) {
	fence_action = "reboot";
	}
	}

	/* Create a fence pseudo-event, so we have an event to order actions
	* against, and the controller can always detect it.
	*/
	stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", FALSE, data_set);
	update_action_flags(stonith_op, pe_action_pseudo \| pe_action_runnable,
	__func__, __LINE__);

	/* We want to imply stops/demotes after the guest is stopped, not wait until
	* it is restarted, so we always order pseudo-fencing after stop, not start
	* (even though start might be closer to what is done for a real reboot).
	*/
	if ((stop != NULL) && pcmk_is_set(stop->flags, pe_action_pseudo)) {
	pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, FALSE, data_set);
	crm_info("Implying guest node %s is down (action %d) after %s fencing",
	node->details->uname, stonith_op->id, stop->node->details->uname);
	order_actions(parent_stonith_op, stonith_op,
	pe_order_runnable_left\|pe_order_implies_then);

	} else if (stop) {
	order_actions(stop, stonith_op,
	pe_order_runnable_left\|pe_order_implies_then);
	crm_info("Implying guest node %s is down (action %d) "
	"after container %s is stopped (action %d)",
	node->details->uname, stonith_op->id,
	container->id, stop->id);
	} else {
	/* If we're fencing the guest node but there's no stop for the guest
	* resource, we must think the guest is already stopped. However, we may
	* think so because its resource history was just cleaned. To avoid
	* unnecessarily considering the guest node down if it's really up,
	* order the pseudo-fencing after any stop of the connection resource,
	* which will be ordered after any container (re-)probe.
	*/
	stop = find_first_action(node->details->remote_rsc->actions, NULL,
	RSC_STOP, NULL);

	if (stop) {
	order_actions(stop, stonith_op, pe_order_optional);
	crm_info("Implying guest node %s is down (action %d) "
	"after connection is stopped (action %d)",
	node->details->uname, stonith_op->id, stop->id);
	} else {
	/* Not sure why we're fencing, but everything must already be
	* cleanly stopped.
	*/
	crm_info("Implying guest node %s is down (action %d) ",
	node->details->uname, stonith_op->id);
	}
	}

	/* Order/imply other actions relative to pseudo-fence as with real fence */
	pcmk__order_vs_fence(stonith_op, data_set);
	}

	/*
	* Create dependencies for stonith and shutdown operations
	*/
	gboolean
	stage6(pe_working_set_t * data_set)
	{
	pe_action_t *dc_down = NULL;
	pe_action_t *stonith_op = NULL;
	gboolean integrity_lost = FALSE;
	gboolean need_stonith = TRUE;
	GList *gIter;
	GList *stonith_ops = NULL;
	GList *shutdown_ops = NULL;

	/* Remote ordering constraints need to happen prior to calculating fencing
	* because it is one more place we will mark the node as dirty.
	*
	* A nice side effect of doing them early is that apply_*_ordering() can be
	* simpler because pe_fence_node() has already done some of the work.
	*/
	crm_trace("Creating remote ordering constraints");
	apply_remote_node_ordering(data_set);

	crm_trace("Processing fencing and shutdown cases");
	if (any_managed_resources(data_set) == FALSE) {
	crm_notice("Delaying fencing operations until there are resources to manage");
	need_stonith = FALSE;
	}

	/* Check each node for stonith/shutdown */
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	/* Guest nodes are "fenced" by recovering their container resource,
	* so handle them separately.
	*/
	if (pe__is_guest_node(node)) {
	if (node->details->remote_requires_reset && need_stonith
	&& pe_can_fence(data_set, node)) {
	fence_guest(node, data_set);
	}
	continue;
	}

	stonith_op = NULL;

	if (node->details->unclean
	&& need_stonith && pe_can_fence(data_set, node)) {

	stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", FALSE, data_set);
	pe_warn("Scheduling Node %s for STONITH", node->details->uname);

	pcmk__order_vs_fence(stonith_op, data_set);

	if (node->details->is_dc) {
	// Remember if the DC is being fenced
	dc_down = stonith_op;

	} else {

	if (!pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)
	&& (stonith_ops != NULL)) {
	/* Concurrent fencing is disabled, so order each non-DC
	* fencing in a chain. If there is any DC fencing or
	* shutdown, it will be ordered after the last action in the
	* chain later.
	*/
	order_actions((pe_action_t *) stonith_ops->data,
	stonith_op, pe_order_optional);
	}

	// Remember all non-DC fencing actions in a separate list
	stonith_ops = g_list_prepend(stonith_ops, stonith_op);
	}

	} else if (node->details->online && node->details->shutdown &&
	/* TODO define what a shutdown op means for a remote node.
	* For now we do not send shutdown operations for remote nodes, but
	* if we can come up with a good use for this in the future, we will. */
	pe__is_guest_or_remote_node(node) == FALSE) {

	pe_action_t *down_op = sched_shutdown_op(node, data_set);

	if (node->details->is_dc) {
	// Remember if the DC is being shut down
	dc_down = down_op;
	} else {
	// Remember non-DC shutdowns for later ordering
	shutdown_ops = g_list_prepend(shutdown_ops, down_op);
	}
	}

	if (node->details->unclean && stonith_op == NULL) {
	integrity_lost = TRUE;
	pe_warn("Node %s is unclean!", node->details->uname);
	}
	}

	if (integrity_lost) {
	if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
	pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");

	} else if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	crm_notice("Cannot fence unclean nodes until quorum is"
	" attained (or no-quorum-policy is set to ignore)");
	}
	}

	if (dc_down != NULL) {
	/* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
	* DC elections. However, we don't want to order non-DC shutdowns before
	* a DC fencing, because even though we don't want a node that's
	* shutting down to become DC, the DC fencing could be ordered before a
	* clone stop that's also ordered before the shutdowns, thus leading to
	* a graph loop.
	*/
	if (pcmk__str_eq(dc_down->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	for (gIter = shutdown_ops; gIter != NULL; gIter = gIter->next) {
	pe_action_t node_stop = (pe_action_t ) gIter->data;

	crm_debug("Ordering shutdown on %s before %s on DC %s",
	node_stop->node->details->uname,
	dc_down->task, dc_down->node->details->uname);

	order_actions(node_stop, dc_down, pe_order_optional);
	}
	}

	// Order any non-DC fencing before any DC fencing or shutdown

	if (pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)) {
	/* With concurrent fencing, order each non-DC fencing action
	* separately before any DC fencing or shutdown.
	*/
	for (gIter = stonith_ops; gIter != NULL; gIter = gIter->next) {
	order_actions((pe_action_t *) gIter->data, dc_down,
	pe_order_optional);
	}
	} else if (stonith_ops) {
	/* Without concurrent fencing, the non-DC fencing actions are
	* already ordered relative to each other, so we just need to order
	* the DC fencing after the last action in the chain (which is the
	* first item in the list).
	*/
	order_actions((pe_action_t *) stonith_ops->data, dc_down,
	pe_order_optional);
	}
	}
	g_list_free(stonith_ops);
	g_list_free(shutdown_ops);
	return TRUE;
	}

	/*
	* Determine the sets of independent actions and the correct order for the
	* actions in each set.
	*
	* Mark dependencies of un-runnable actions un-runnable
	*
	*/
	static GList *
	find_actions_by_task(GList actions, pe_resource_t rsc, const char *original_key)
	{
	GList *list = NULL;

	list = find_actions(actions, original_key, NULL);
	if (list == NULL) {
	/* we're potentially searching a child of the original resource */
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
	key = pcmk__op_key(rsc->id, task, interval_ms);
	list = find_actions(actions, key, NULL);

	} else {
	crm_err("search key: %s", original_key);
	}

	free(key);
	free(task);
	}

	return list;
	}

	static void
	rsc_order_then(pe_action_t lh_action, pe_resource_t rsc,
	pe__ordering_t *order)
	{
	GList *gIter = NULL;
	GList *rh_actions = NULL;
	pe_action_t *rh_action = NULL;
	enum pe_ordering type;

	CRM_CHECK(rsc != NULL, return);
	CRM_CHECK(order != NULL, return);

	type = order->type;
	rh_action = order->rh_action;
	crm_trace("Processing RH of ordering constraint %d", order->id);

	if (rh_action != NULL) {
	rh_actions = g_list_prepend(NULL, rh_action);

	} else if (rsc != NULL) {
	rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
	}

	if (rh_actions == NULL) {
	pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..."
	" ignoring", rsc->id, order->rh_action_task);
	if (lh_action) {
	pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid);
	}
	return;
	}

	if ((lh_action != NULL) && (lh_action->rsc == rsc)
	&& pcmk_is_set(lh_action->flags, pe_action_dangle)) {

	pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
	order->rh_action_task);
	pe__clear_order_flags(type, pe_order_implies_then);
	}

	gIter = rh_actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t rh_action_iter = (pe_action_t ) gIter->data;

	if (lh_action) {
	order_actions(lh_action, rh_action_iter, type);

	} else if (type & pe_order_implies_then) {
	update_action_flags(rh_action_iter, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
	} else {
	crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
	}
	}

	g_list_free(rh_actions);
	}

	static void
	rsc_order_first(pe_resource_t lh_rsc, pe__ordering_t order,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;
	GList *lh_actions = NULL;
	pe_action_t *lh_action = order->lh_action;
	pe_resource_t *rh_rsc = order->rh_rsc;

	crm_trace("Processing LH of ordering constraint %d", order->id);
	CRM_ASSERT(lh_rsc != NULL);

	if (lh_action != NULL) {
	lh_actions = g_list_prepend(NULL, lh_action);

	} else {
	lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
	}

	if (lh_actions == NULL && lh_rsc != rh_rsc) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;

	parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(lh_rsc->id, op_type, interval_ms);

	if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && pcmk__str_eq(op_type, RSC_STOP, pcmk__str_casei)) {
	free(key);
	pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
	lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);

	} else if ((lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_UNPROMOTED)
	&& pcmk__str_eq(op_type, RSC_DEMOTE, pcmk__str_casei)) {
	free(key);
	pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
	lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);

	} else {
	pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating",
	lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
	lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
	lh_actions = g_list_prepend(NULL, lh_action);
	}

	free(op_type);
	}

	gIter = lh_actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t lh_action_iter = (pe_action_t ) gIter->data;

	if (rh_rsc == NULL && order->rh_action) {
	rh_rsc = order->rh_action->rsc;
	}
	if (rh_rsc) {
	rsc_order_then(lh_action_iter, rh_rsc, order);

	} else if (order->rh_action) {
	order_actions(lh_action_iter, order->rh_action, order->type);
	}
	}

	g_list_free(lh_actions);
	}

	extern void update_colo_start_chain(pe_action_t *action,
	pe_working_set_t *data_set);

	static int
	is_recurring_action(pe_action_t *action)
	{
	guint interval_ms;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	return 0;
	}
	return (interval_ms > 0);
	}

	static void
	apply_container_ordering(pe_action_t action, pe_working_set_t data_set)
	{
	/* VMs are also classified as containers for these purposes... in
	* that they both involve a 'thing' running on a real or remote
	* cluster node.
	*
	* This allows us to be smarter about the type and extent of
	* recovery actions required in various scenarios
	*/
	pe_resource_t *remote_rsc = NULL;
	pe_resource_t *container = NULL;
	enum action_tasks task = text2task(action->task);

	CRM_ASSERT(action->rsc);
	CRM_ASSERT(action->node);
	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc);

	container = remote_rsc->container;
	CRM_ASSERT(container);

	if (pcmk_is_set(container->flags, pe_rsc_failed)) {
	pe_fence_node(data_set, action->node, "container failed", FALSE);
	}

	crm_trace("Order %s action %s relative to %s%s for %s%s",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
	remote_rsc->id,
	pcmk_is_set(container->flags, pe_rsc_failed)? "failed " : "",
	container->id);

	if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
	/* Migration ops map to "no_action", but we need to apply the same
	* ordering as for stop or demote (see get_router_node()).
	*/
	task = stop_rsc;
	}

	switch (task) {
	case start_rsc:
	case action_promote:
	/* Force resource recovery if the container is recovered */
	order_start_then_action(container, action, pe_order_implies_then,
	data_set);

	/* Wait for the connection resource to be up too */
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	break;

	case stop_rsc:
	case action_demote:
	if (pcmk_is_set(container->flags, pe_rsc_failed)) {
	/* When the container representing a guest node fails, any stop
	* or demote actions for resources running on the guest node
	* are implied by the container stopping. This is similar to
	* how fencing operations work for cluster nodes and remote
	* nodes.
	*/
	} else {
	/* Ensure the operation happens before the connection is brought
	* down.
	*
	* If we really wanted to, we could order these after the
	* connection start, IFF the container's current role was
	* stopped (otherwise we re-introduce an ordering loop when the
	* connection is restarting).
	*/
	order_action_then_stop(action, remote_rsc, pe_order_none,
	data_set);
	}
	break;

	default:
	/* Wait for the connection resource to be up */
	if (is_recurring_action(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	if(task != no_action) {
	order_start_then_action(remote_rsc, action,
	pe_order_implies_then, data_set);
	}
	} else {
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	}
	break;
	}
	}

	static enum remote_connection_state
	get_remote_node_state(pe_node_t *node)
	{
	pe_resource_t *remote_rsc = NULL;
	pe_node_t *cluster_node = NULL;

	CRM_ASSERT(node);

	remote_rsc = node->details->remote_rsc;
	CRM_ASSERT(remote_rsc);

	cluster_node = pe__current_node(remote_rsc);

	/* If the cluster node the remote connection resource resides on
	* is unclean or went offline, we can't process any operations
	* on that remote node until after it starts elsewhere.
	*/
	if(remote_rsc->next_role == RSC_ROLE_STOPPED \|\| remote_rsc->allocated_to == NULL) {
	/* The connection resource is not going to run anywhere */

	if (cluster_node && cluster_node->details->unclean) {
	/* The remote connection is failed because its resource is on a
	* failed node and can't be recovered elsewhere, so we must fence.
	*/
	return remote_state_failed;
	}

	if (!pcmk_is_set(remote_rsc->flags, pe_rsc_failed)) {
	/* Connection resource is cleanly stopped */
	return remote_state_stopped;
	}

	/* Connection resource is failed */

	if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
	&& remote_rsc->remote_reconnect_ms
	&& node->details->remote_was_fenced
	&& !pe__shutdown_requested(node)) {

	/* We won't know whether the connection is recoverable until the
	* reconnect interval expires and we reattempt connection.
	*/
	return remote_state_unknown;
	}

	/* The remote connection is in a failed state. If there are any
	* resources known to be active on it (stop) or in an unknown state
	* (probe), we must assume the worst and fence it.
	*/
	return remote_state_failed;

	} else if (cluster_node == NULL) {
	/* Connection is recoverable but not currently running anywhere, see if we can recover it first */
	return remote_state_unknown;

	} else if(cluster_node->details->unclean == TRUE
	\|\| cluster_node->details->online == FALSE) {
	/* Connection is running on a dead node, see if we can recover it first */
	return remote_state_resting;

	} else if (pcmk__list_of_multiple(remote_rsc->running_on)
	&& remote_rsc->partial_migration_source
	&& remote_rsc->partial_migration_target) {
	/* We're in the middle of migrating a connection resource,
	* wait until after the resource migrates before performing
	* any actions.
	*/
	return remote_state_resting;

	}
	return remote_state_alive;
	}

	/*!
	* \internal
	* \brief Order actions on remote node relative to actions for the connection
	*/
	static void
	apply_remote_ordering(pe_action_t action, pe_working_set_t data_set)
	{
	pe_resource_t *remote_rsc = NULL;
	enum action_tasks task = text2task(action->task);
	enum remote_connection_state state = get_remote_node_state(action->node);

	enum pe_ordering order_opts = pe_order_none;

	if (action->rsc == NULL) {
	return;
	}

	CRM_ASSERT(action->node);
	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc);

	crm_trace("Order %s action %s relative to %s%s (state: %s)",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
	remote_rsc->id, state2text(state));

	if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
	/* Migration ops map to "no_action", but we need to apply the same
	* ordering as for stop or demote (see get_router_node()).
	*/
	task = stop_rsc;
	}

	switch (task) {
	case start_rsc:
	case action_promote:
	order_opts = pe_order_none;

	if (state == remote_state_failed) {
	/* Force recovery, by making this action required */
	pe__set_order_flags(order_opts, pe_order_implies_then);
	}

	/* Ensure connection is up before running this action */
	order_start_then_action(remote_rsc, action, order_opts, data_set);
	break;

	case stop_rsc:
	if(state == remote_state_alive) {
	order_action_then_stop(action, remote_rsc,
	pe_order_implies_first, data_set);

	} else if(state == remote_state_failed) {
	/* The resource is active on the node, but since we don't have a
	* valid connection, the only way to stop the resource is by
	* fencing the node. There is no need to order the stop relative
	* to the remote connection, since the stop will become implied
	* by the fencing.
	*/
	pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable", FALSE);

	} else if(remote_rsc->next_role == RSC_ROLE_STOPPED) {
	/* State must be remote_state_unknown or remote_state_stopped.
	* Since the connection is not coming back up in this
	* transition, stop this resource first.
	*/
	order_action_then_stop(action, remote_rsc,
	pe_order_implies_first, data_set);

	} else {
	/* The connection is going to be started somewhere else, so
	* stop this resource after that completes.
	*/
	order_start_then_action(remote_rsc, action, pe_order_none, data_set);
	}
	break;

	case action_demote:
	/* Only order this demote relative to the connection start if the
	* connection isn't being torn down. Otherwise, the demote would be
	* blocked because the connection start would not be allowed.
	*/
	if(state == remote_state_resting \|\| state == remote_state_unknown) {
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	} /* Otherwise we can rely on the stop ordering */
	break;

	default:
	/* Wait for the connection resource to be up */
	if (is_recurring_action(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	order_start_then_action(remote_rsc, action,
	pe_order_implies_then, data_set);

	} else {
	pe_node_t *cluster_node = pe__current_node(remote_rsc);

	if(task == monitor_rsc && state == remote_state_failed) {
	/* We would only be here if we do not know the
	* state of the resource on the remote node.
	* Since we have no way to find out, it is
	* necessary to fence the node.
	*/
	pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable", FALSE);
	}

	if(cluster_node && state == remote_state_stopped) {
	/* The connection is currently up, but is going
	* down permanently.
	*
	* Make sure we check services are actually
	* stopped _before_ we let the connection get
	* closed
	*/
	order_action_then_stop(action, remote_rsc,
	pe_order_runnable_left, data_set);

	} else {
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	}
	}
	break;
	}
	}

	static void
	apply_remote_node_ordering(pe_working_set_t *data_set)
	{
	if (!pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	return;
	}

	for (GList *gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;
	pe_resource_t *remote = NULL;

	// We are only interested in resource actions
	if (action->rsc == NULL) {
	continue;
	}

	/* Special case: If we are clearing the failcount of an actual
	* remote connection resource, then make sure this happens before
	* any start of the resource in this transition.
	*/
	if (action->rsc->is_remote_node &&
	pcmk__str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT, pcmk__str_casei)) {

	custom_action_order(action->rsc,
	NULL,
	action,
	action->rsc,
	pcmk__op_key(action->rsc->id, RSC_START, 0),
	NULL,
	pe_order_optional,
	data_set);

	continue;
	}

	// We are only interested in actions allocated to a node
	if (action->node == NULL) {
	continue;
	}

	if (!pe__is_guest_or_remote_node(action->node)) {
	continue;
	}

	/* We are only interested in real actions.
	*
	* @TODO This is probably wrong; pseudo-actions might be converted to
	* real actions and vice versa later in update_actions() at the end of
	* stage7().
	*/
	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	continue;
	}

	remote = action->node->details->remote_rsc;
	if (remote == NULL) {
	// Orphaned
	continue;
	}

	/* Another special case: if a resource is moving to a Pacemaker Remote
	* node, order the stop on the original node after any start of the
	* remote connection. This ensures that if the connection fails to
	* start, we leave the resource running on the original node.
	*/
	if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
	for (GList *item = action->rsc->actions; item != NULL;
	item = item->next) {
	pe_action_t *rsc_action = item->data;

	if ((rsc_action->node->details != action->node->details)
	&& pcmk__str_eq(rsc_action->task, RSC_STOP, pcmk__str_casei)) {
	custom_action_order(remote, start_key(remote), NULL,
	action->rsc, NULL, rsc_action,
	pe_order_optional, data_set);
	}
	}
	}

	/* The action occurs across a remote connection, so create
	* ordering constraints that guarantee the action occurs while the node
	* is active (after start, before stop ... things like that).
	*
	* This is somewhat brittle in that we need to make sure the results of
	* this ordering are compatible with the result of get_router_node().
	* It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
	* of this logic rather than action2xml().
	*/
	if (remote->container) {
	crm_trace("Container ordering for %s", action->uuid);
	apply_container_ordering(action, data_set);

	} else {
	crm_trace("Remote ordering for %s", action->uuid);
	apply_remote_ordering(action, data_set);
	}
	}
	}

	static gboolean
	order_first_probe_unneeded(pe_action_t * probe, pe_action_t * rh_action)
	{
	/* No need to probe the resource on the node that is being
	* unfenced. Otherwise it might introduce transition loop
	* since probe will be performed after the node is
	* unfenced.
	*/
	if (pcmk__str_eq(rh_action->task, CRM_OP_FENCE, pcmk__str_casei)
	&& probe->node && rh_action->node
	&& probe->node->details == rh_action->node->details) {
	const char *op = g_hash_table_lookup(rh_action->meta, "stonith_action");

	if (pcmk__str_eq(op, "on", pcmk__str_casei)) {
	return TRUE;
	}
	}

	// Shutdown waits for probe to complete only if it's on the same node
	if ((pcmk__str_eq(rh_action->task, CRM_OP_SHUTDOWN, pcmk__str_casei))
	&& probe->node && rh_action->node
	&& probe->node->details != rh_action->node->details) {
	return TRUE;
	}
	return FALSE;
	}

	static void
	order_first_probes_imply_stops(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
	pe__ordering_t *order = gIter->data;
	enum pe_ordering order_type = pe_order_optional;

	pe_resource_t *lh_rsc = order->lh_rsc;
	pe_resource_t *rh_rsc = order->rh_rsc;
	pe_action_t *lh_action = order->lh_action;
	pe_action_t *rh_action = order->rh_action;
	const char *lh_action_task = order->lh_action_task;
	const char *rh_action_task = order->rh_action_task;

	GList *probes = NULL;
	GList *rh_actions = NULL;

	GList *pIter = NULL;

	if (lh_rsc == NULL) {
	continue;

	} else if (rh_rsc && lh_rsc == rh_rsc) {
	continue;
	}

	if (lh_action == NULL && lh_action_task == NULL) {
	continue;
	}

	if (rh_action == NULL && rh_action_task == NULL) {
	continue;
	}

	/* Technically probe is expected to return "not running", which could be
	* the alternative of stop action if the status of the resource is
	* unknown yet.
	*/
	if (lh_action && !pcmk__str_eq(lh_action->task, RSC_STOP, pcmk__str_casei)) {
	continue;

	} else if (lh_action == NULL
	&& lh_action_task
	&& !pcmk__ends_with(lh_action_task, "_" RSC_STOP "_0")) {
	continue;
	}

	/* Do not probe the resource inside of a stopping container. Otherwise
	* it might introduce transition loop since probe will be performed
	* after the container starts again.
	*/
	if (rh_rsc && lh_rsc->container == rh_rsc) {
	if (rh_action && pcmk__str_eq(rh_action->task, RSC_STOP, pcmk__str_casei)) {
	continue;

	} else if (rh_action == NULL && rh_action_task
	&& pcmk__ends_with(rh_action_task,"_" RSC_STOP "_0")) {
	continue;
	}
	}

	if (order->type == pe_order_none) {
	continue;
	}

	// Preserve the order options for future filtering
	if (pcmk_is_set(order->type, pe_order_apply_first_non_migratable)) {
	pe__set_order_flags(order_type,
	pe_order_apply_first_non_migratable);
	}

	if (pcmk_is_set(order->type, pe_order_same_node)) {
	pe__set_order_flags(order_type, pe_order_same_node);
	}

	// Keep the order types for future filtering
	if (order->type == pe_order_anti_colocation
	\|\| order->type == pe_order_load) {
	order_type = order->type;
	}

	probes = pe__resource_actions(lh_rsc, NULL, RSC_STATUS, FALSE);
	if (probes == NULL) {
	continue;
	}

	if (rh_action) {
	rh_actions = g_list_prepend(rh_actions, rh_action);

	} else if (rh_rsc && rh_action_task) {
	rh_actions = find_actions(rh_rsc->actions, rh_action_task, NULL);
	}

	if (rh_actions == NULL) {
	g_list_free(probes);
	continue;
	}

	crm_trace("Processing for LH probe based on ordering constraint %s -> %s"
	" (id=%d, type=%.6x)",
	lh_action ? lh_action->uuid : lh_action_task,
	rh_action ? rh_action->uuid : rh_action_task,
	order->id, order->type);

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	pe_action_t probe = (pe_action_t ) pIter->data;
	GList *rIter = NULL;

	for (rIter = rh_actions; rIter != NULL; rIter = rIter->next) {
	pe_action_t rh_action_iter = (pe_action_t ) rIter->data;

	if (order_first_probe_unneeded(probe, rh_action_iter)) {
	continue;
	}
	order_actions(probe, rh_action_iter, order_type);
	}
	}

	g_list_free(rh_actions);
	g_list_free(probes);
	}
	}

	static void
	order_first_probe_then_restart_repromote(pe_action_t * probe,
	pe_action_t * after,
	pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	bool interleave = FALSE;
	pe_resource_t *compatible_rsc = NULL;

	if (probe == NULL
	\|\| probe->rsc == NULL
	\|\| probe->rsc->variant != pe_native) {
	return;
	}

	if (after == NULL
	// Avoid running into any possible loop
	\|\| pcmk_is_set(after->flags, pe_action_tracking)) {
	return;
	}

	if (!pcmk__str_eq(probe->task, RSC_STATUS, pcmk__str_casei)) {
	return;
	}

	pe__set_action_flags(after, pe_action_tracking);

	crm_trace("Processing based on %s %s -> %s %s",
	probe->uuid,
	probe->node ? probe->node->details->uname: "",
	after->uuid,
	after->node ? after->node->details->uname : "");

	if (after->rsc
	/* Better not build a dependency directly with a clone/group.
	* We are going to proceed through the ordering chain and build
	* dependencies with its children.
	*/
	&& after->rsc->variant == pe_native
	&& probe->rsc != after->rsc) {

	GList *then_actions = NULL;
	enum pe_ordering probe_order_type = pe_order_optional;

	if (pcmk__str_eq(after->task, RSC_START, pcmk__str_casei)) {
	then_actions = pe__resource_actions(after->rsc, NULL, RSC_STOP, FALSE);

	} else if (pcmk__str_eq(after->task, RSC_PROMOTE, pcmk__str_casei)) {
	then_actions = pe__resource_actions(after->rsc, NULL, RSC_DEMOTE, FALSE);
	}

	for (gIter = then_actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t then = (pe_action_t ) gIter->data;

	// Skip any pseudo action which for example is implied by fencing
	if (pcmk_is_set(then->flags, pe_action_pseudo)) {
	continue;
	}

	order_actions(probe, then, probe_order_type);
	}
	g_list_free(then_actions);
	}

	if (after->rsc
	&& after->rsc->variant > pe_group) {
	const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
	XML_RSC_ATTR_INTERLEAVE);

	interleave = crm_is_true(interleave_s);

	if (interleave) {
	/* For an interleaved clone, we should build a dependency only
	* with the relevant clone child.
	*/
	compatible_rsc = find_compatible_child(probe->rsc,
	after->rsc,
	RSC_ROLE_UNKNOWN,
	FALSE, data_set);
	}
	}

	for (gIter = after->actions_after; gIter != NULL; gIter = gIter->next) {
	pe_action_wrapper_t after_wrapper = (pe_action_wrapper_t ) gIter->data;
	/* pe_order_implies_then is the reason why a required A.start
	* implies/enforces B.start to be required too, which is the cause of
	* B.restart/re-promote.
	*
	* Not sure about pe_order_implies_then_on_node though. It's now only
	* used for unfencing case, which tends to introduce transition
	* loops...
	*/

	if (!pcmk_is_set(after_wrapper->type, pe_order_implies_then)) {
	/* The order type between a group/clone and its child such as
	* B.start-> B_child.start is:
	* pe_order_implies_first_printed \| pe_order_runnable_left
	*
	* Proceed through the ordering chain and build dependencies with
	* its children.
	*/
	if (after->rsc == NULL
	\|\| after->rsc->variant < pe_group
	\|\| probe->rsc->parent == after->rsc
	\|\| after_wrapper->action->rsc == NULL
	\|\| after_wrapper->action->rsc->variant > pe_group
	\|\| after->rsc != after_wrapper->action->rsc->parent) {
	continue;
	}

	/* Proceed to the children of a group or a non-interleaved clone.
	* For an interleaved clone, proceed only to the relevant child.
	*/
	if (after->rsc->variant > pe_group
	&& interleave == TRUE
	&& (compatible_rsc == NULL
	\|\| compatible_rsc != after_wrapper->action->rsc)) {
	continue;
	}
	}

	crm_trace("Proceeding through %s %s -> %s %s (type=0x%.6x)",
	after->uuid,
	after->node ? after->node->details->uname: "",
	after_wrapper->action->uuid,
	after_wrapper->action->node ? after_wrapper->action->node->details->uname : "",
	after_wrapper->type);

	order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
	}
	}

	static void clear_actions_tracking_flag(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (pcmk_is_set(action->flags, pe_action_tracking)) {
	pe__clear_action_flags(action, pe_action_tracking);
	}
	}
	}

	static void
	order_first_rsc_probes(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	GList *probes = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t * child = (pe_resource_t *) gIter->data;

	order_first_rsc_probes(child, data_set);
	}

	if (rsc->variant != pe_native) {
	return;
	}

	probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);

	for (gIter = probes; gIter != NULL; gIter= gIter->next) {
	pe_action_t probe = (pe_action_t ) gIter->data;
	GList *aIter = NULL;

	for (aIter = probe->actions_after; aIter != NULL; aIter = aIter->next) {
	pe_action_wrapper_t after_wrapper = (pe_action_wrapper_t ) aIter->data;

	order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
	clear_actions_tracking_flag(data_set);
	}
	}

	g_list_free(probes);
	}

	static void
	order_first_probes(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	order_first_rsc_probes(rsc, data_set);
	}

	order_first_probes_imply_stops(data_set);
	}

	static void
	order_then_probes(pe_working_set_t * data_set)
	{
	#if 0
	GList *gIter = NULL;

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* Given "A then B", we would prefer to wait for A to be
	* started before probing B.
	*
	* If A was a filesystem on which the binaries and data for B
	* lived, it would have been useful if the author of B's agent
	* could assume that A is running before B.monitor will be
	* called.
	*
	* However we can't _only_ probe once A is running, otherwise
	* we'd not detect the state of B if A could not be started
	* for some reason.
	*
	* In practice however, we cannot even do an opportunistic
	* version of this because B may be moving:
	*
	* B.probe -> B.start
	* B.probe -> B.stop
	* B.stop -> B.start
	* A.stop -> A.start
	* A.start -> B.probe
	*
	* So far so good, but if we add the result of this code:
	*
	* B.stop -> A.stop
	*
	* Then we get a loop:
	*
	* B.probe -> B.stop -> A.stop -> A.start -> B.probe
	*
	* We could kill the 'B.probe -> B.stop' dependency, but that
	* could mean stopping B "too" soon, because B.start must wait
	* for the probes to complete.
	*
	* Another option is to allow it only if A is a non-unique
	* clone with clone-max == node-max (since we'll never be
	* moving it). However, we could still be stopping one
	* instance at the same time as starting another.

	* The complexity of checking for allowed conditions combined
	* with the ever narrowing usecase suggests that this code
	* should remain disabled until someone gets smarter.
	*/
	pe_action_t *start = NULL;
	GList *actions = NULL;
	GList *probes = NULL;

	actions = pe__resource_actions(rsc, NULL, RSC_START, FALSE);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if(start == NULL) {
	crm_err("No start action for %s", rsc->id);
	continue;
	}

	probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);

	for (actions = start->actions_before; actions != NULL; actions = actions->next) {
	pe_action_wrapper_t before = (pe_action_wrapper_t ) actions->data;

	GList *pIter = NULL;
	pe_action_t *first = before->action;
	pe_resource_t *first_rsc = first->rsc;

	if(first->required_runnable_before) {
	GList *clone_actions = NULL;
	for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
	before = (pe_action_wrapper_t *) clone_actions->data;

	crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);

	CRM_ASSERT(before->action->rsc);
	first_rsc = before->action->rsc;
	break;
	}

	} else if(!pcmk__str_eq(first->task, RSC_START, pcmk__str_casei)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if(first_rsc == NULL) {
	continue;

	} else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	crm_err("Applying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	pe_action_t probe = (pe_action_t ) pIter->data;

	crm_err("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pe_order_optional);
	}
	}
	}
	#endif
	}

	static void
	order_probes(pe_working_set_t * data_set)
	{
	order_first_probes(data_set);
	order_then_probes(data_set);
	}

	gboolean
	stage7(pe_working_set_t * data_set)
	{
	pcmk__output_t *prev_out = data_set->priv;
	pcmk__output_t *out = NULL;
	GList *gIter = NULL;

	crm_trace("Applying ordering constraints");

	/* Don't ask me why, but apparently they need to be processed in
	* the order they were created in... go figure
	*
	* Also g_list_append() has horrendous performance characteristics
	* So we need to use g_list_prepend() and then reverse the list here
	*/
	data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);

	for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
	pe__ordering_t *order = gIter->data;
	pe_resource_t *rsc = order->lh_rsc;

	crm_trace("Applying ordering constraint: %d", order->id);

	if (rsc != NULL) {
	crm_trace("rsc_action-to-*");
	rsc_order_first(rsc, order, data_set);
	continue;
	}

	rsc = order->rh_rsc;
	if (rsc != NULL) {
	crm_trace("action-to-rsc_action");
	rsc_order_then(order->lh_action, rsc, order);

	} else {
	crm_trace("action-to-action");
	order_actions(order->lh_action, order->rh_action, order->type);
	}
	}

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	update_colo_start_chain(action, data_set);
	}

	crm_trace("Ordering probes");
	order_probes(data_set);

	crm_trace("Updating %d actions", g_list_length(data_set->actions));
	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	update_action(action, data_set);
	}

	// Check for invalid orderings
	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;
	pe_action_wrapper_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = (pe_action_wrapper_t *) input_iter->data;
	if (pcmk__ordering_is_invalid(action, input)) {
	input->type = pe_order_none;
	}
	}
	}

	/* stage7 only ever outputs to the log, so ignore whatever output object was
	* previously set and just log instead.
	*/
	out = pcmk__new_logger();
	if (out == NULL) {
	return FALSE;
	}

	pcmk__output_set_log_level(out, LOG_NOTICE);
	data_set->priv = out;

	out->begin_list(out, NULL, NULL, "Actions");
	LogNodeActions(data_set);

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	LogActions(rsc, data_set);
	}

	out->end_list(out);
	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);

	data_set->priv = prev_out;
	return TRUE;
	}

	static int transition_id = -1;

	/*!
	* \internal
	* \brief Log a message after calculating a transition
	*
	* \param[in] filename Where transition input is stored
	*/
	void
	pcmk__log_transition_summary(const char *filename)
	{
	if (was_processing_error) {
	crm_err("Calculated transition %d (with errors)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else if (was_processing_warning) {
	crm_warn("Calculated transition %d (with warnings)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else {
	crm_notice("Calculated transition %d%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);
	}
	if (crm_config_error) {
	crm_notice("Configuration errors found during scheduler processing,"
	" please run \"crm_verify -L\" to identify issues");
	}
	}

	/*
	* Create a dependency graph to send to the transitioner (via the controller)
	*/
	gboolean
	stage8(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	const char *value = NULL;
	long long limit = 0LL;

	transition_id++;
	crm_trace("Creating transition graph %d.", transition_id);

	data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);

	value = pe_pref(data_set->config_hash, "cluster-delay");
	crm_xml_add(data_set->graph, "cluster-delay", value);

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	crm_xml_add(data_set->graph, "stonith-timeout", value);

	crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");

	if (pcmk_is_set(data_set->flags, pe_flag_start_failure_fatal)) {
	crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
	} else {
	crm_xml_add(data_set->graph, "failed-start-offset", "1");
	}

	value = pe_pref(data_set->config_hash, "batch-limit");
	crm_xml_add(data_set->graph, "batch-limit", value);

	crm_xml_add_int(data_set->graph, "transition_id", transition_id);

	value = pe_pref(data_set->config_hash, "migration-limit");
	if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
	crm_xml_add(data_set->graph, "migration-limit", value);
	}

	if (data_set->recheck_by > 0) {
	char *recheck_epoch = NULL;

	recheck_epoch = crm_strdup_printf("%llu",
	(long long) data_set->recheck_by);
	crm_xml_add(data_set->graph, "recheck-by", recheck_epoch);
	free(recheck_epoch);
	}

	/* errors...
	slist_iter(action, pe_action_t, action_list, lpc,
	if(action->optional == FALSE && action->runnable == FALSE) {
	print_action("Ignoring", action, TRUE);
	}
	);
	*/

	/* The following code will de-duplicate action inputs, so nothing past this
	* should rely on the action input type flags retaining their original
	* values.
	*/

	gIter = data_set->resources;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
	rsc->cmds->expand(rsc, data_set);
	}

	crm_log_xml_trace(data_set->graph, "created resource-driven action list");

	/* pseudo action to distribute list of nodes with maintenance state update */
	add_maintenance_update(data_set);

	/* catch any non-resource specific actions */
	crm_trace("processing non-resource actions");

	gIter = data_set->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (action->rsc
	&& action->node
	&& action->node->details->shutdown
	&& !pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)
	&& !pcmk_any_flags_set(action->flags,
	pe_action_optional\|pe_action_runnable)
	&& pcmk__str_eq(action->task, RSC_STOP, pcmk__str_none)
	) {
	/* Eventually we should just ignore the 'fence' case
	* But for now it's the best way to detect (in CTS) when
	* CIB resource updates are being lost
	*/
	if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)
	\|\| data_set->no_quorum_policy == no_quorum_ignore) {
	crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)",
	action->node->details->unclean ? "fence" : "shut down",
	action->node->details->uname, action->rsc->id,
	pcmk_is_set(action->rsc->flags, pe_rsc_managed)? " blocked" : " unmanaged",
	pcmk_is_set(action->rsc->flags, pe_rsc_failed)? " failed" : "",
	action->uuid);
	}
	}

	graph_element_from_action(action, data_set);
	}

	crm_log_xml_trace(data_set->graph, "created generic action list");
	crm_trace("Created transition graph %d.", transition_id);

	return TRUE;
	}

	void
	LogNodeActions(pe_working_set_t * data_set)
	{
	pcmk__output_t *out = data_set->priv;
	GList *gIter = NULL;

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	char *node_name = NULL;
	char *task = NULL;
	pe_action_t action = (pe_action_t ) gIter->data;

	if (action->rsc != NULL) {
	continue;
	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	continue;
	}

	if (pe__is_guest_node(action->node)) {
	node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
	} else if(action->node) {
	node_name = crm_strdup_printf("%s", action->node->details->uname);
	}


	if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	task = strdup("Shutdown");
	} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
	const char *op = g_hash_table_lookup(action->meta, "stonith_action");
	task = crm_strdup_printf("Fence (%s)", op);
	}

	out->message(out, "node-action", task, node_name, action->reason);

	free(node_name);
	free(task);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_graph.c b/lib/pacemaker/pcmk_sched_graph.c
	index 2d749b21cd..76d5365041 100644
	--- a/lib/pacemaker/pcmk_sched_graph.c
	+++ b/lib/pacemaker/pcmk_sched_graph.c
	@@ -1,1902 +1,1902 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <pacemaker-internal.h>

	void update_colo_start_chain(pe_action_t action, pe_working_set_t data_set);
	gboolean rsc_update_action(pe_action_t * first, pe_action_t * then, enum pe_ordering type);

	static enum pe_action_flags
	get_action_flags(pe_action_t * action, pe_node_t * node)
	{
	enum pe_action_flags flags = action->flags;

	if (action->rsc) {
	flags = action->rsc->cmds->action_flags(action, NULL);

	if (pe_rsc_is_clone(action->rsc) && node) {

	/* We only care about activity on $node */
	enum pe_action_flags clone_flags = action->rsc->cmds->action_flags(action, node);

	/* Go to great lengths to ensure the correct value for pe_action_runnable...
	*
	* If we are a clone, then for _ordering_ constraints, it's only relevant
	* if we are runnable _anywhere_.
	*
	* This only applies to _runnable_ though, and only for ordering constraints.
	* If this function is ever used during colocation, then we'll need additional logic
	*
	* Not very satisfying, but it's logical and appears to work well.
	*/
	if (!pcmk_is_set(clone_flags, pe_action_runnable)
	&& pcmk_is_set(flags, pe_action_runnable)) {

	pe__set_raw_action_flags(clone_flags, action->rsc->id,
	pe_action_runnable);
	}
	flags = clone_flags;
	}
	}
	return flags;
	}

	static char *
	convert_non_atomic_uuid(char old_uuid, pe_resource_t rsc, gboolean allow_notify,
	gboolean free_original)
	{
	guint interval_ms = 0;
	char *uuid = NULL;
	char *rid = NULL;
	char *raw_task = NULL;
	int task = no_action;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Processing %s", old_uuid);
	if (old_uuid == NULL) {
	return NULL;

	} else if (strstr(old_uuid, "notify") != NULL) {
	goto done; /* no conversion */

	} else if (rsc->variant < pe_group) {
	goto done; /* no conversion */
	}

	CRM_ASSERT(parse_op_key(old_uuid, &rid, &raw_task, &interval_ms));
	if (interval_ms > 0) {
	goto done; /* no conversion */
	}

	task = text2task(raw_task);
	switch (task) {
	case stop_rsc:
	case start_rsc:
	case action_notify:
	case action_promote:
	case action_demote:
	break;
	case stopped_rsc:
	case started_rsc:
	case action_notified:
	case action_promoted:
	case action_demoted:
	task--;
	break;
	case monitor_rsc:
	case shutdown_crm:
	case stonith_node:
	task = no_action;
	break;
	default:
	crm_err("Unknown action: %s", raw_task);
	task = no_action;
	break;
	}

	if (task != no_action) {
	if (pcmk_is_set(rsc->flags, pe_rsc_notify) && allow_notify) {
	uuid = pcmk__notify_key(rid, "confirmed-post", task2text(task + 1));

	} else {
	uuid = pcmk__op_key(rid, task2text(task + 1), 0);
	}
	pe_rsc_trace(rsc, "Converted %s -> %s", old_uuid, uuid);
	}

	done:
	if (uuid == NULL) {
	uuid = strdup(old_uuid);
	}

	if (free_original) {
	free(old_uuid);
	}

	free(raw_task);
	free(rid);
	return uuid;
	}

	static pe_action_t *
	rsc_expand_action(pe_action_t * action)
	{
	gboolean notify = FALSE;
	pe_action_t *result = action;
	pe_resource_t *rsc = action->rsc;

	if (rsc == NULL) {
	return action;
	}

	if ((rsc->parent == NULL)
	\|\| (pe_rsc_is_clone(rsc) && (rsc->parent->variant == pe_container))) {
	/* Only outermost resources have notification actions.
	* The exception is those in bundles.
	*/
	notify = pcmk_is_set(rsc->flags, pe_rsc_notify);
	}

	if (rsc->variant >= pe_group) {
	/* Expand 'start' -> 'started' */
	char *uuid = NULL;

	uuid = convert_non_atomic_uuid(action->uuid, rsc, notify, FALSE);
	if (uuid) {
	pe_rsc_trace(rsc, "Converting %s to %s %d", action->uuid, uuid,
	pcmk_is_set(rsc->flags, pe_rsc_notify));
	result = find_first_action(rsc->actions, uuid, NULL, NULL);
	if (result == NULL) {
	crm_err("Couldn't expand %s to %s in %s", action->uuid, uuid, rsc->id);
	result = action;
	}
	free(uuid);
	}
	}
	return result;
	}

	static enum pe_graph_flags
	graph_update_action(pe_action_t * first, pe_action_t * then, pe_node_t * node,
	enum pe_action_flags first_flags, enum pe_action_flags then_flags,
	pe_action_wrapper_t order, pe_working_set_t data_set)
	{
	enum pe_graph_flags changed = pe_graph_none;
	enum pe_ordering type = order->type;
	gboolean processed = FALSE;

	/* TODO: Do as many of these in parallel as possible */

	if (pcmk_is_set(type, pe_order_implies_then_on_node)) {
	/* Normally we want the _whole_ 'then' clone to
	* restart if 'first' is restarted, so then->node is
	* needed.
	*
	* However for unfencing, we want to limit this to
	* instances on the same node as 'first' (the
	* unfencing operation), so first->node is supplied.
	*
	* Swap the node, from then on we can can treat it
	* like any other 'pe_order_implies_then'
	*/

	pe__clear_order_flags(type, pe_order_implies_then_on_node);
	pe__set_order_flags(type, pe_order_implies_then);
	node = first->node;
	}

	pe__clear_raw_action_flags(first_flags, "first action update",
	pe_action_pseudo);

	if (type & pe_order_implies_then) {
	processed = TRUE;
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags & pe_action_optional, pe_action_optional,
	pe_order_implies_then, data_set);

	} else if (!pcmk_is_set(first_flags, pe_action_optional)) {
	if (update_action_flags(then, pe_action_optional \| pe_action_clear, __func__, __LINE__)) {
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	}
	}
	if (changed) {
	pe_rsc_trace(then->rsc, "implies right: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("implies right: %s then %s %p", first->uuid, then->uuid, then->rsc);
	}
	}

	if ((type & pe_order_restart) && then->rsc) {
	enum pe_action_flags restart = (pe_action_optional \| pe_action_runnable);

	processed = TRUE;
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, restart,
	pe_order_restart, data_set);
	if (changed) {
	pe_rsc_trace(then->rsc, "restart: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("restart: %s then %s", first->uuid, then->uuid);
	}
	}

	if (type & pe_order_implies_first) {
	processed = TRUE;
	if (first->rsc) {
	changed \|= first->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_optional, pe_order_implies_first,
	data_set);

	} else if (!pcmk_is_set(first_flags, pe_action_optional)) {
	pe_rsc_trace(first->rsc, "first unrunnable: %s (%d) then %s (%d)",
	first->uuid, pcmk_is_set(first_flags, pe_action_optional),
	then->uuid, pcmk_is_set(then_flags, pe_action_optional));
	if (update_action_flags(first, pe_action_runnable \| pe_action_clear, __func__, __LINE__)) {
	pe__set_graph_flags(changed, first, pe_graph_updated_first);
	}
	}

	if (changed) {
	pe_rsc_trace(then->rsc, "implies left: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("implies left: %s (%d) then %s (%d)",
	first->uuid, pcmk_is_set(first_flags, pe_action_optional),
	then->uuid, pcmk_is_set(then_flags, pe_action_optional));
	}
	}

	if (type & pe_order_promoted_implies_first) {
	processed = TRUE;
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags & pe_action_optional, pe_action_optional,
	pe_order_promoted_implies_first, data_set);
	}

	if (changed) {
	pe_rsc_trace(then->rsc,
	"implies left when right resource is promoted: "
	"%s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("implies left when right resource is promoted: "
	"%s then %s", first->uuid, then->uuid);
	}
	}

	if (type & pe_order_one_or_more) {
	processed = TRUE;
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_one_or_more,
	data_set);

	} else if (pcmk_is_set(first_flags, pe_action_runnable)) {
	/* alright. a "first" action is considered runnable, incremente
	* the 'runnable_before' counter */
	then->runnable_before++;

	/* if the runnable before count for then exceeds the required number
	* of "before" runnable actions... mark then as runnable */
	if (then->runnable_before >= then->required_runnable_before) {
	if (update_action_flags(then, pe_action_runnable, __func__, __LINE__)) {
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	}
	}
	}
	if (changed) {
	pe_rsc_trace(then->rsc, "runnable_one_or_more: %s then %s: changed", first->uuid,
	then->uuid);
	} else {
	crm_trace("runnable_one_or_more: %s then %s", first->uuid, then->uuid);
	}
	}

	if (then->rsc && pcmk_is_set(type, pe_order_probe)) {
	processed = TRUE;

	if (!pcmk_is_set(first_flags, pe_action_runnable)
	&& (first->rsc->running_on != NULL)) {

	pe_rsc_trace(then->rsc, "Ignoring %s then %s - %s is about to be stopped",
	first->uuid, then->uuid, first->rsc->id);
	type = pe_order_none;
	order->type = pe_order_none;

	} else {
	pe_rsc_trace(then->rsc, "Enforcing %s then %s", first->uuid, then->uuid);
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_runnable_left,
	data_set);
	}

	if (changed) {
	pe_rsc_trace(then->rsc, "runnable: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("runnable: %s then %s", first->uuid, then->uuid);
	}
	}

	if (type & pe_order_runnable_left) {
	processed = TRUE;
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_runnable_left,
	data_set);

	} else if (!pcmk_is_set(first_flags, pe_action_runnable)) {
	pe_rsc_trace(then->rsc, "then unrunnable: %s then %s", first->uuid, then->uuid);
	if (update_action_flags(then, pe_action_runnable \| pe_action_clear, __func__, __LINE__)) {
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	}
	}
	if (changed) {
	pe_rsc_trace(then->rsc, "runnable: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("runnable: %s then %s", first->uuid, then->uuid);
	}
	}

	if (type & pe_order_implies_first_migratable) {
	processed = TRUE;
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_optional,
	pe_order_implies_first_migratable, data_set);
	}
	if (changed) {
	pe_rsc_trace(then->rsc, "optional: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("optional: %s then %s", first->uuid, then->uuid);
	}
	}

	if (type & pe_order_pseudo_left) {
	processed = TRUE;
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_optional, pe_order_pseudo_left,
	data_set);
	}
	if (changed) {
	pe_rsc_trace(then->rsc, "optional: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("optional: %s then %s", first->uuid, then->uuid);
	}
	}

	if (type & pe_order_optional) {
	processed = TRUE;
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_optional, data_set);
	}
	if (changed) {
	pe_rsc_trace(then->rsc, "optional: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("optional: %s then %s", first->uuid, then->uuid);
	}
	}

	if (type & pe_order_asymmetrical) {
	processed = TRUE;
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_asymmetrical,
	data_set);
	}

	if (changed) {
	pe_rsc_trace(then->rsc, "asymmetrical: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("asymmetrical: %s then %s", first->uuid, then->uuid);
	}

	}

	if ((first->flags & pe_action_runnable) && (type & pe_order_implies_then_printed)
	&& (first_flags & pe_action_optional) == 0) {
	processed = TRUE;
	crm_trace("%s implies %s printed", first->uuid, then->uuid);
	update_action_flags(then, pe_action_print_always, __func__, __LINE__); /* don't care about changed */
	}

	if (pcmk_is_set(type, pe_order_implies_first_printed)
	&& !pcmk_is_set(then_flags, pe_action_optional)) {

	processed = TRUE;
	crm_trace("%s implies %s printed", then->uuid, first->uuid);
	update_action_flags(first, pe_action_print_always, __func__, __LINE__); /* don't care about changed */
	}

	if ((type & pe_order_implies_then
	\|\| type & pe_order_implies_first
	\|\| type & pe_order_restart)
	&& first->rsc
	&& pcmk__str_eq(first->task, RSC_STOP, pcmk__str_casei)
	&& !pcmk_is_set(first->rsc->flags, pe_rsc_managed)
	&& pcmk_is_set(first->rsc->flags, pe_rsc_block)
	&& !pcmk_is_set(first->flags, pe_action_runnable)) {

	if (update_action_flags(then, pe_action_runnable \| pe_action_clear, __func__, __LINE__)) {
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	}

	if (changed) {
	pe_rsc_trace(then->rsc, "unmanaged left: %s then %s: changed", first->uuid, then->uuid);
	} else {
	crm_trace("unmanaged left: %s then %s", first->uuid, then->uuid);
	}
	}

	if (processed == FALSE) {
	crm_trace("Constraint 0x%.6x not applicable", type);
	}

	return changed;
	}

	static void
	mark_start_blocked(pe_resource_t rsc, pe_resource_t reason,
	pe_working_set_t *data_set)
	{
	GList *gIter = rsc->actions;
	char *reason_text = crm_strdup_printf("colocation with %s", reason->id);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (!pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
	continue;
	}
	if (pcmk_is_set(action->flags, pe_action_runnable)) {
	pe_action_set_flag_reason(__func__, __LINE__, action, NULL,
	reason_text, pe_action_runnable, FALSE);
	update_colo_start_chain(action, data_set);
	update_action(action, data_set);
	}
	}
	free(reason_text);
	}

	void
	update_colo_start_chain(pe_action_t action, pe_working_set_t data_set)
	{
	GList *gIter = NULL;
	pe_resource_t *rsc = NULL;

	if (!pcmk_is_set(action->flags, pe_action_runnable)
	&& pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {

	rsc = uber_parent(action->rsc);
	if (rsc->parent) {
	/* For bundles, uber_parent() returns the clone, not the bundle, so
	* the existence of rsc->parent implies this is a bundle.
	* In this case, we need the bundle resource, so that we can check
	* if all containers are stopped/stopping.
	*/
	rsc = rsc->parent;
	}
	}

	if (rsc == NULL \|\| rsc->rsc_cons_lhs == NULL) {
	return;
	}

	/* if rsc has children, all the children need to have start set to
	* unrunnable before we follow the colo chain for the parent. */
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t )gIter->data;
	pe_action_t *start = find_first_action(child->actions, NULL, RSC_START, NULL);
	if ((start == NULL) \|\| pcmk_is_set(start->flags, pe_action_runnable)) {
	return;
	}
	}

	for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t colocate_with = (pcmk__colocation_t ) gIter->data;

	if (colocate_with->score == INFINITY) {
	mark_start_blocked(colocate_with->rsc_lh, action->rsc, data_set);
	}
	}
	}

	gboolean
	update_action(pe_action_t then, pe_working_set_t data_set)
	{
	GList *lpc = NULL;
	enum pe_graph_flags changed = pe_graph_none;
	int last_flags = then->flags;

	crm_trace("Processing %s (%s %s %s)",
	then->uuid,
	pcmk_is_set(then->flags, pe_action_optional)? "optional" : "required",
	pcmk_is_set(then->flags, pe_action_runnable)? "runnable" : "unrunnable",
	pcmk_is_set(then->flags, pe_action_pseudo)? "pseudo"
	: (then->node? then->node->details->uname : ""));

	if (pcmk_is_set(then->flags, pe_action_requires_any)) {
	/* initialize current known runnable before actions to 0
	* from here as graph_update_action is called for each of
	* then's before actions, this number will increment as
	* runnable 'first' actions are encountered */
	then->runnable_before = 0;

	/* for backwards compatibility with previous options that use
	* the 'requires_any' flag, initialize required to 1 if it is
	* not set. */
	if (then->required_runnable_before == 0) {
	then->required_runnable_before = 1;
	}
	pe__clear_action_flags(then, pe_action_runnable);
	/* We are relying on the pe_order_one_or_more clause of
	* graph_update_action(), called as part of the:
	*
	* 'if (first == other->action)'
	*
	* block below, to set this back if appropriate
	*/
	}

	for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;
	pe_action_t *first = other->action;

	pe_node_t *then_node = then->node;
	pe_node_t *first_node = first->node;

	enum pe_action_flags then_flags = 0;
	enum pe_action_flags first_flags = 0;

	if (first->rsc && first->rsc->variant == pe_group && pcmk__str_eq(first->task, RSC_START, pcmk__str_casei)) {
	first_node = first->rsc->fns->location(first->rsc, NULL, FALSE);
	if (first_node) {
	crm_trace("First: Found node %s for %s", first_node->details->uname, first->uuid);
	}
	}

	if (then->rsc && then->rsc->variant == pe_group && pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)) {
	then_node = then->rsc->fns->location(then->rsc, NULL, FALSE);
	if (then_node) {
	crm_trace("Then: Found node %s for %s", then_node->details->uname, then->uuid);
	}
	}
	/* Disable constraint if it only applies when on same node, but isn't */
	if (pcmk_is_set(other->type, pe_order_same_node)
	&& (first_node != NULL) && (then_node != NULL)
	&& (first_node->details != then_node->details)) {

	crm_trace("Disabled constraint %s on %s -> %s on %s",
	other->action->uuid, first_node->details->uname,
	then->uuid, then_node->details->uname);
	other->type = pe_order_none;
	continue;
	}

	pe__clear_graph_flags(changed, then, pe_graph_updated_first);

	if (first->rsc && pcmk_is_set(other->type, pe_order_then_cancels_first)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {

	/* 'then' is required, so we must abandon 'first'
	- * (e.g. a required stop cancels any reload).
	+ * (e.g. a required stop cancels any agent reload).
	*/
	pe__set_action_flags(other->action, pe_action_optional);
	- if (!strcmp(first->task, CRMD_ACTION_RELOAD)) {
	+ if (!strcmp(first->task, CRMD_ACTION_RELOAD_AGENT)) {
	pe__clear_resource_flags(first->rsc, pe_rsc_reload);
	}
	}

	if (first->rsc && then->rsc && (first->rsc != then->rsc)
	&& (is_parent(then->rsc, first->rsc) == FALSE)) {
	first = rsc_expand_action(first);
	}
	if (first != other->action) {
	crm_trace("Ordering %s after %s instead of %s", then->uuid, first->uuid,
	other->action->uuid);
	}

	first_flags = get_action_flags(first, then_node);
	then_flags = get_action_flags(then, first_node);

	crm_trace("Checking %s (%s %s %s) against %s (%s %s %s) filter=0x%.6x type=0x%.6x",
	then->uuid,
	pcmk_is_set(then_flags, pe_action_optional)? "optional" : "required",
	pcmk_is_set(then_flags, pe_action_runnable)? "runnable" : "unrunnable",
	pcmk_is_set(then_flags, pe_action_pseudo)? "pseudo"
	: (then->node? then->node->details->uname : ""),
	first->uuid,
	pcmk_is_set(first_flags, pe_action_optional)? "optional" : "required",
	pcmk_is_set(first_flags, pe_action_runnable)? "runnable" : "unrunnable",
	pcmk_is_set(first_flags, pe_action_pseudo)? "pseudo"
	: (first->node? first->node->details->uname : ""),
	first_flags, other->type);

	if (first == other->action) {
	/*
	* 'first' was not expanded (e.g. from 'start' to 'running'), which could mean it:
	* - has no associated resource,
	* - was a primitive,
	* - was pre-expanded (e.g. 'running' instead of 'start')
	*
	* The third argument here to graph_update_action() is a node which is used under two conditions:
	* - Interleaving, in which case first->node and
	* then->node are equal (and NULL)
	* - If 'then' is a clone, to limit the scope of the
	* constraint to instances on the supplied node
	*
	*/
	pe_node_t *node = then->node;
	changed \|= graph_update_action(first, then, node, first_flags,
	then_flags, other, data_set);

	/* 'first' was for a complex resource (clone, group, etc),
	* create a new dependency if necessary
	*/
	} else if (order_actions(first, then, other->type)) {
	/* This was the first time 'first' and 'then' were associated,
	* start again to get the new actions_before list
	*/
	pe__set_graph_flags(changed, then,
	pe_graph_updated_then\|pe_graph_disable);
	}

	if (changed & pe_graph_disable) {
	crm_trace("Disabled constraint %s -> %s in favor of %s -> %s",
	other->action->uuid, then->uuid, first->uuid, then->uuid);
	pe__clear_graph_flags(changed, then, pe_graph_disable);
	other->type = pe_order_none;
	}

	if (changed & pe_graph_updated_first) {
	GList *lpc2 = NULL;

	crm_trace("Updated %s (first %s %s %s), processing dependents ",
	first->uuid,
	pcmk_is_set(first->flags, pe_action_optional)? "optional" : "required",
	pcmk_is_set(first->flags, pe_action_runnable)? "runnable" : "unrunnable",
	pcmk_is_set(first->flags, pe_action_pseudo)? "pseudo"
	: (first->node? first->node->details->uname : ""));
	for (lpc2 = first->actions_after; lpc2 != NULL; lpc2 = lpc2->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc2->data;

	update_action(other->action, data_set);
	}
	update_action(first, data_set);
	}
	}

	if (pcmk_is_set(then->flags, pe_action_requires_any)) {
	if (last_flags != then->flags) {
	pe__set_graph_flags(changed, then, pe_graph_updated_then);
	} else {
	pe__clear_graph_flags(changed, then, pe_graph_updated_then);
	}
	}

	if (changed & pe_graph_updated_then) {
	crm_trace("Updated %s (then %s %s %s), processing dependents ",
	then->uuid,
	pcmk_is_set(then->flags, pe_action_optional)? "optional" : "required",
	pcmk_is_set(then->flags, pe_action_runnable)? "runnable" : "unrunnable",
	pcmk_is_set(then->flags, pe_action_pseudo)? "pseudo"
	: (then->node? then->node->details-> uname : ""));

	if (pcmk_is_set(last_flags, pe_action_runnable)
	&& !pcmk_is_set(then->flags, pe_action_runnable)) {
	update_colo_start_chain(then, data_set);
	}
	update_action(then, data_set);
	for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;

	update_action(other->action, data_set);
	}
	}

	return FALSE;
	}

	gboolean
	shutdown_constraints(pe_node_t * node, pe_action_t * shutdown_op, pe_working_set_t * data_set)
	{
	/* add the stop to the before lists so it counts as a pre-req
	* for the shutdown
	*/
	GList *lpc = NULL;

	for (lpc = data_set->actions; lpc != NULL; lpc = lpc->next) {
	pe_action_t action = (pe_action_t ) lpc->data;

	if (action->rsc == NULL \|\| action->node == NULL) {
	continue;
	} else if (action->node->details != node->details) {
	continue;
	} else if (pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)) {
	pe_rsc_trace(action->rsc, "Skipping %s: maintenance mode", action->uuid);
	continue;
	} else if (node->details->maintenance) {
	pe_rsc_trace(action->rsc, "Skipping %s: node %s is in maintenance mode",
	action->uuid, node->details->uname);
	continue;
	} else if (!pcmk__str_eq(action->task, RSC_STOP, pcmk__str_casei)) {
	continue;
	} else if (!pcmk_any_flags_set(action->rsc->flags,
	pe_rsc_managed\|pe_rsc_block)) {
	/*
	* If another action depends on this one, we may still end up blocking
	*/
	pe_rsc_trace(action->rsc, "Skipping %s: unmanaged", action->uuid);
	continue;
	}

	pe_rsc_trace(action->rsc, "Ordering %s before shutdown on %s", action->uuid,
	node->details->uname);
	pe__clear_action_flags(action, pe_action_optional);
	custom_action_order(action->rsc, NULL, action,
	NULL, strdup(CRM_OP_SHUTDOWN), shutdown_op,
	pe_order_optional \| pe_order_runnable_left, data_set);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Order all actions appropriately relative to a fencing operation
	*
	* Ensure start operations of affected resources are ordered after fencing,
	* imply stop and demote operations of affected resources by marking them as
	* pseudo-actions, etc.
	*
	* \param[in] stonith_op Fencing operation
	* \param[in,out] data_set Working set of cluster
	*/
	void
	pcmk__order_vs_fence(pe_action_t stonith_op, pe_working_set_t data_set)
	{
	CRM_CHECK(stonith_op && data_set, return);
	for (GList *r = data_set->resources; r != NULL; r = r->next) {
	rsc_stonith_ordering((pe_resource_t *) r->data, stonith_op, data_set);
	}
	}

	static pe_node_t *
	get_router_node(pe_action_t *action)
	{
	pe_node_t *began_on = NULL;
	pe_node_t *ended_on = NULL;
	bool partial_migration = FALSE;
	const char *task = action->task;

	if (pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)
	\|\| !pe__is_guest_or_remote_node(action->node)) {
	return NULL;
	}

	CRM_ASSERT(action->node->details->remote_rsc != NULL);

	began_on = pe__current_node(action->node->details->remote_rsc);
	ended_on = action->node->details->remote_rsc->allocated_to;
	if (action->node->details->remote_rsc
	&& (action->node->details->remote_rsc->container == NULL)
	&& action->node->details->remote_rsc->partial_migration_target) {
	partial_migration = TRUE;
	}

	if (began_on == NULL) {
	crm_trace("Routing %s for %s through remote connection's "
	"next node %s (starting)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(ended_on? ended_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return ended_on;
	}

	if (ended_on == NULL) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (stopping)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	if (began_on->details == ended_on->details) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (not moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	/* If we get here, the remote connection is moving during this transition.
	* This means some actions for resources behind the connection will get
	* routed through the cluster node the connection reource is currently on,
	* and others are routed through the cluster node the connection will end up
	* on.
	*/

	if (pcmk__str_eq(task, "notify", pcmk__str_casei)) {
	task = g_hash_table_lookup(action->meta, "notify_operation");
	}

	/*
	* Stop, demote, and migration actions must occur before the connection can
	* move (these actions are required before the remote resource can stop). In
	* this case, we know these actions have to be routed through the initial
	* cluster node the connection resource lived on before the move takes
	* place.
	*
	* The exception is a partial migration of a (non-guest) remote connection
	* resource; in that case, all actions (even these) will be ordered after
	* the connection's pseudo-start on the migration target, so the target is
	* the router node.
	*/
	if (pcmk__strcase_any_of(task, "cancel", "stop", "demote", "migrate_from",
	"migrate_to", NULL) && !partial_migration) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	/* Everything else (start, promote, monitor, probe, refresh,
	* clear failcount, delete, ...) must occur after the connection starts on
	* the node it is moving to.
	*/
	crm_trace("Routing %s for %s through remote connection's "
	"next node %s (moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(ended_on? ended_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return ended_on;
	}

	/*!
	* \internal
	* \brief Add an XML node tag for a specified ID
	*
	* \param[in] id Node UUID to add
	* \param[in,out] xml Parent XML tag to add to
	*/
	static xmlNode*
	add_node_to_xml_by_id(const char id, xmlNode xml)
	{
	xmlNode *node_xml;

	node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
	crm_xml_add(node_xml, XML_ATTR_UUID, id);

	return node_xml;
	}

	/*!
	* \internal
	* \brief Add an XML node tag for a specified node
	*
	* \param[in] node Node to add
	* \param[in,out] xml XML to add node to
	*/
	static void
	add_node_to_xml(const pe_node_t node, void xml)
	{
	add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
	}

	/*!
	* \internal
	* \brief Add XML with nodes that need an update of their maintenance state
	*
	* \param[in,out] xml Parent XML tag to add to
	* \param[in] data_set Working set for cluster
	*/
	static int
	add_maintenance_nodes(xmlNode xml, const pe_working_set_t data_set)
	{
	GList *gIter = NULL;
	xmlNode *maintenance =
	xml?create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE):NULL;
	int count = 0;

	for (gIter = data_set->nodes; gIter != NULL;
	gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;
	struct pe_node_shared_s *details = node->details;

	if (!pe__is_guest_or_remote_node(node)) {
	continue; /* just remote nodes need to know atm */
	}

	if (details->maintenance != details->remote_maintenance) {
	if (maintenance) {
	crm_xml_add(
	add_node_to_xml_by_id(node->details->id, maintenance),
	XML_NODE_IS_MAINTENANCE, details->maintenance?"1":"0");
	}
	count++;
	}
	}
	crm_trace("%s %d nodes to adjust maintenance-mode "
	"to transition", maintenance?"Added":"Counted", count);
	return count;
	}

	/*!
	* \internal
	* \brief Add pseudo action with nodes needing maintenance state update
	*
	* \param[in,out] data_set Working set for cluster
	*/
	void
	add_maintenance_update(pe_working_set_t *data_set)
	{
	pe_action_t *action = NULL;

	if (add_maintenance_nodes(NULL, data_set)) {
	crm_trace("adding maintenance state update pseudo action");
	action = get_pseudo_op(CRM_OP_MAINTENANCE_NODES, data_set);
	pe__set_action_flags(action, pe_action_print_always);
	}
	}

	/*!
	* \internal
	* \brief Add XML with nodes that an action is expected to bring down
	*
	* If a specified action is expected to bring any nodes down, add an XML block
	* with their UUIDs. When a node is lost, this allows the controller to
	* determine whether it was expected.
	*
	* \param[in,out] xml Parent XML tag to add to
	* \param[in] action Action to check for downed nodes
	* \param[in] data_set Working set for cluster
	*/
	static void
	add_downed_nodes(xmlNode xml, const pe_action_t action,
	const pe_working_set_t *data_set)
	{
	CRM_CHECK(xml && action && action->node && data_set, return);

	if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {

	/* Shutdown makes the action's node down */
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);

	} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {

	/* Fencing makes the action's node and any hosted guest nodes down */
	const char *fence = g_hash_table_lookup(action->meta, "stonith_action");

	if (pcmk__strcase_any_of(fence, "off", "reboot", NULL)) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);
	pe_foreach_guest_node(data_set, action->node, add_node_to_xml, downed);
	}

	} else if (action->rsc && action->rsc->is_remote_node
	&& pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)) {

	/* Stopping a remote connection resource makes connected node down,
	* unless it's part of a migration
	*/
	GList *iter;
	pe_action_t *input;
	gboolean migrating = FALSE;

	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	input = ((pe_action_wrapper_t *) iter->data)->action;
	if (input->rsc && pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_casei)
	&& pcmk__str_eq(input->task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
	migrating = TRUE;
	break;
	}
	}
	if (!migrating) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->rsc->id, downed);
	}
	}
	}

	static bool
	should_lock_action(pe_action_t *action)
	{
	// Only actions taking place on resource's lock node are locked
	if ((action->rsc->lock_node == NULL) \|\| (action->node == NULL)
	\|\| (action->node->details != action->rsc->lock_node->details)) {
	return false;
	}

	/* During shutdown, only stops are locked (otherwise, another action such as
	* a demote would cause the controller to clear the lock)
	*/
	if (action->node->details->shutdown && action->task
	&& strcmp(action->task, RSC_STOP)) {
	return false;
	}

	return true;
	}

	static xmlNode *
	action2xml(pe_action_t * action, gboolean as_input, pe_working_set_t *data_set)
	{
	gboolean needs_node_info = TRUE;
	gboolean needs_maintenance_info = FALSE;
	xmlNode *action_xml = NULL;
	xmlNode *args_xml = NULL;
	#if ENABLE_VERSIONED_ATTRS
	pe_rsc_action_details_t *rsc_details = NULL;
	#endif

	if (action == NULL) {
	return NULL;
	}

	if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
	/* All fences need node info; guest node fences are pseudo-events */
	action_xml = create_xml_node(NULL,
	pcmk_is_set(action->flags, pe_action_pseudo)?
	XML_GRAPH_TAG_PSEUDO_EVENT :
	XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT, pcmk__str_casei)) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, CRM_OP_LRM_REFRESH, pcmk__str_casei)) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	// CIB-only clean-up for shutdown locks
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
	crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);

	/* } else if(pcmk__str_eq(action->task, RSC_PROBED, pcmk__str_casei)) { */
	/* action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); */

	} else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	if (pcmk__str_eq(action->task, CRM_OP_MAINTENANCE_NODES, pcmk__str_casei)) {
	needs_maintenance_info = TRUE;
	}
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_PSEUDO_EVENT);
	needs_node_info = FALSE;

	} else {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);

	#if ENABLE_VERSIONED_ATTRS
	rsc_details = pe_rsc_action_details(action);
	#endif
	}

	crm_xml_add_int(action_xml, XML_ATTR_ID, action->id);
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);
	if (action->rsc != NULL && action->rsc->clone_name != NULL) {
	char *clone_key = NULL;
	guint interval_ms;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}

	if (pcmk__str_eq(action->task, RSC_NOTIFY, pcmk__str_casei)) {
	const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
	const char *n_task = g_hash_table_lookup(action->meta, "notify_operation");

	CRM_CHECK(n_type != NULL, crm_err("No notify type value found for %s", action->uuid));
	CRM_CHECK(n_task != NULL,
	crm_err("No notify operation value found for %s", action->uuid));
	clone_key = pcmk__notify_key(action->rsc->clone_name,
	n_type, n_task);

	} else if(action->cancel_task) {
	clone_key = pcmk__op_key(action->rsc->clone_name,
	action->cancel_task, interval_ms);
	} else {
	clone_key = pcmk__op_key(action->rsc->clone_name,
	action->task, interval_ms);
	}

	CRM_CHECK(clone_key != NULL, crm_err("Could not generate a key for %s", action->uuid));
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
	crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY, action->uuid);
	free(clone_key);

	} else {
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
	}

	if (needs_node_info && action->node != NULL) {
	pe_node_t *router_node = get_router_node(action);

	crm_xml_add(action_xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
	crm_xml_add(action_xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
	if (router_node) {
	crm_xml_add(action_xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
	}

	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET), strdup(action->node->details->uname));
	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID), strdup(action->node->details->id));
	}

	/* No details if this action is only being listed in the inputs section */
	if (as_input) {
	return action_xml;
	}

	if (action->rsc && !pcmk_is_set(action->flags, pe_action_pseudo)) {
	int lpc = 0;
	xmlNode *rsc_xml = NULL;
	const char *attr_list[] = {
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER,
	XML_ATTR_TYPE
	};

	/* If a resource is locked to a node via shutdown-lock, mark its actions
	* so the controller can preserve the lock when the action completes.
	*/
	if (should_lock_action(action)) {
	crm_xml_add_ll(action_xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	(long long) action->rsc->lock_time);
	}

	// List affected resource

	rsc_xml = create_xml_node(action_xml,
	crm_element_name(action->rsc->xml));
	if (pcmk_is_set(action->rsc->flags, pe_rsc_orphan)
	&& action->rsc->clone_name) {
	/* Do not use the 'instance free' name here as that
	* might interfere with the instance we plan to keep.
	* Ie. if there are more than two named /anonymous/
	* instances on a given node, we need to make sure the
	* command goes to the right one.
	*
	* Keep this block, even when everyone is using
	* 'instance free' anonymous clone names - it means
	* we'll do the right thing if anyone toggles the
	* unique flag to 'off'
	*/
	crm_debug("Using orphan clone name %s instead of %s", action->rsc->id,
	action->rsc->clone_name);
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name);
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);

	} else if (!pcmk_is_set(action->rsc->flags, pe_rsc_unique)) {
	const char *xml_id = ID(action->rsc->xml);

	crm_debug("Using anonymous clone name %s for %s (aka. %s)", xml_id, action->rsc->id,
	action->rsc->clone_name);

	/* ID is what we'd like client to use
	* ID_LONG is what they might know it as instead
	*
	* ID_LONG is only strictly needed /here/ during the
	* transition period until all nodes in the cluster
	* are running the new software /and/ have rebooted
	* once (meaning that they've only ever spoken to a DC
	* supporting this feature).
	*
	* If anyone toggles the unique flag to 'on', the
	* 'instance free' name will correspond to an orphan
	* and fall into the clause above instead
	*/
	crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id);
	if (action->rsc->clone_name && !pcmk__str_eq(xml_id, action->rsc->clone_name, pcmk__str_casei)) {
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name);
	} else {
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
	}

	} else {
	CRM_ASSERT(action->rsc->clone_name == NULL);
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id);
	}

	for (lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
	crm_xml_add(rsc_xml, attr_list[lpc],
	g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
	}
	}

	/* List any attributes in effect */
	args_xml = create_xml_node(NULL, XML_TAG_ATTRS);
	crm_xml_add(args_xml, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);

	g_hash_table_foreach(action->extra, hash2field, args_xml);
	if (action->rsc != NULL && action->node) {
	// Get the resource instance attributes, evaluated properly for node
	GHashTable *params = pe_rsc_params(action->rsc, action->node, data_set);

	/* REMOTE_CONTAINER_HACK: If this is a remote connection resource with
	* addr="#uname", pull the actual value from the parameters evaluated
	* without a node (which was put there earlier in stage8() when the
	* bundle's expand() method was called).
	*/
	const char *remote_addr = g_hash_table_lookup(params,
	XML_RSC_ATTR_REMOTE_RA_ADDR);

	if (pcmk__str_eq(remote_addr, "#uname", pcmk__str_none)) {
	GHashTable *base = pe_rsc_params(action->rsc, NULL, data_set);

	remote_addr = g_hash_table_lookup(base,
	XML_RSC_ATTR_REMOTE_RA_ADDR);
	if (remote_addr != NULL) {
	g_hash_table_insert(params, strdup(XML_RSC_ATTR_REMOTE_RA_ADDR),
	strdup(remote_addr));
	}
	}

	g_hash_table_foreach(params, hash2smartfield, args_xml);

	#if ENABLE_VERSIONED_ATTRS
	{
	xmlNode *versioned_parameters = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);

	pe_get_versioned_attributes(versioned_parameters, action->rsc,
	action->node, data_set);
	if (xml_has_children(versioned_parameters)) {
	add_node_copy(action_xml, versioned_parameters);
	}
	free_xml(versioned_parameters);
	}
	#endif

	} else if(action->rsc && action->rsc->variant <= pe_native) {
	GHashTable *params = pe_rsc_params(action->rsc, NULL, data_set);

	g_hash_table_foreach(params, hash2smartfield, args_xml);

	#if ENABLE_VERSIONED_ATTRS
	if (xml_has_children(action->rsc->versioned_parameters)) {
	add_node_copy(action_xml, action->rsc->versioned_parameters);
	}
	#endif
	}

	#if ENABLE_VERSIONED_ATTRS
	if (rsc_details) {
	if (xml_has_children(rsc_details->versioned_parameters)) {
	add_node_copy(action_xml, rsc_details->versioned_parameters);
	}

	if (xml_has_children(rsc_details->versioned_meta)) {
	add_node_copy(action_xml, rsc_details->versioned_meta);
	}
	}
	#endif

	g_hash_table_foreach(action->meta, hash2metafield, args_xml);
	if (action->rsc != NULL) {
	const char *value = g_hash_table_lookup(action->rsc->meta, "external-ip");
	pe_resource_t *parent = action->rsc;

	while (parent != NULL) {
	parent->cmds->append_meta(parent, args_xml);
	parent = parent->parent;
	}

	if(value) {
	hash2smartfield((gpointer)"pcmk_external_ip", (gpointer)value, (gpointer)args_xml);
	}

	if (action->node && /* make clang analyzer happy */
	pe__is_guest_node(action->node)) {
	pe_node_t *host = NULL;
	enum action_tasks task = text2task(action->task);

	if(task == action_notify \|\| task == action_notified) {
	const char *n_task = g_hash_table_lookup(action->meta, "notify_operation");
	task = text2task(n_task);
	}

	// Differentiate between up and down actions
	switch (task) {
	case stop_rsc:
	case stopped_rsc:
	case action_demote:
	case action_demoted:
	host = pe__current_node(action->node->details->remote_rsc->container);
	break;
	case start_rsc:
	case started_rsc:
	case monitor_rsc:
	case action_promote:
	case action_promoted:
	host = action->node->details->remote_rsc->container->allocated_to;
	break;
	default:
	break;
	}

	if(host) {
	hash2metafield((gpointer)XML_RSC_ATTR_TARGET,
	(gpointer)g_hash_table_lookup(action->rsc->meta, XML_RSC_ATTR_TARGET), (gpointer)args_xml);
	hash2metafield((gpointer) PCMK__ENV_PHYSICAL_HOST,
	(gpointer)host->details->uname,
	(gpointer)args_xml);
	}
	}

	} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei) && action->node) {
	/* Pass the node's attributes as meta-attributes.
	*
	* @TODO: Determine whether it is still necessary to do this. It was
	* added in 33d99707, probably for the libfence-based implementation in
	* c9a90bd, which is no longer used.
	*/
	g_hash_table_foreach(action->node->details->attrs, hash2metafield, args_xml);
	}

	sorted_xml(args_xml, action_xml, FALSE);
	free_xml(args_xml);

	/* List any nodes this action is expected to make down */
	if (needs_node_info && (action->node != NULL)) {
	add_downed_nodes(action_xml, action, data_set);
	}

	if (needs_maintenance_info) {
	add_maintenance_nodes(action_xml, data_set);
	}

	crm_log_xml_trace(action_xml, "dumped action");
	return action_xml;
	}

	static bool
	should_dump_action(pe_action_t *action)
	{
	CRM_CHECK(action != NULL, return false);

	if (pcmk_is_set(action->flags, pe_action_dumped)) {
	crm_trace("Action %s (%d) already dumped", action->uuid, action->id);
	return false;

	} else if (pcmk_is_set(action->flags, pe_action_pseudo)
	&& pcmk__str_eq(action->task, CRM_OP_PROBED, pcmk__str_casei)) {
	GList *lpc = NULL;

	/* This is a horrible but convenient hack
	*
	* It mimimizes the number of actions with unsatisfied inputs
	* (i.e. not included in the graph)
	*
	* This in turn, means we can be more concise when printing
	* aborted/incomplete graphs.
	*
	* It also makes it obvious which node is preventing
	* probe_complete from running (presumably because it is only
	* partially up)
	*
	* For these reasons we tolerate such perversions
	*/

	for (lpc = action->actions_after; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t wrapper = (pe_action_wrapper_t ) lpc->data;

	if (!pcmk_is_set(wrapper->action->flags, pe_action_runnable)) {
	/* Only interested in runnable operations */
	} else if (!pcmk__str_eq(wrapper->action->task, RSC_START, pcmk__str_casei)) {
	/* Only interested in start operations */
	} else if (pcmk_is_set(wrapper->action->flags, pe_action_dumped)
	\|\| should_dump_action(wrapper->action)) {
	crm_trace("Action %s (%d) should be dumped: "
	"dependency of %s (%d)",
	action->uuid, action->id,
	wrapper->action->uuid, wrapper->action->id);
	return true;
	}
	}
	}

	if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	crm_trace("Ignoring action %s (%d): unrunnable",
	action->uuid, action->id);
	return false;

	} else if (pcmk_is_set(action->flags, pe_action_optional)
	&& !pcmk_is_set(action->flags, pe_action_print_always)) {
	crm_trace("Ignoring action %s (%d): optional",
	action->uuid, action->id);
	return false;

	// Monitors should be dumped even for unmanaged resources
	} else if (action->rsc && !pcmk_is_set(action->rsc->flags, pe_rsc_managed)
	&& !pcmk__str_eq(action->task, RSC_STATUS, pcmk__str_casei)) {

	const char *interval_ms_s = g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS);

	// Cancellation of recurring monitors should still be dumped
	if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
	crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
	action->uuid, action->id, action->rsc->id);
	return false;
	}
	}

	if (pcmk_is_set(action->flags, pe_action_pseudo) \|\|
	pcmk__strcase_any_of(action->task, CRM_OP_FENCE, CRM_OP_SHUTDOWN, NULL)) {
	/* skip the next checks */
	return true;
	}

	if (action->node == NULL) {
	pe_err("Skipping action %s (%d) "
	"because it was not allocated to a node (bug?)",
	action->uuid, action->id);
	log_action(LOG_DEBUG, "Unallocated action", action, false);
	return false;

	} else if (pcmk_is_set(action->flags, pe_action_dc)) {
	crm_trace("Action %s (%d) should be dumped: "
	"can run on DC instead of %s",
	action->uuid, action->id, action->node->details->uname);

	} else if (pe__is_guest_node(action->node)
	&& !action->node->details->remote_requires_reset) {
	crm_trace("Action %s (%d) should be dumped: "
	"assuming will be runnable on guest node %s",
	action->uuid, action->id, action->node->details->uname);

	} else if (action->node->details->online == false) {
	pe_err("Skipping action %s (%d) "
	"because it was scheduled for offline node (bug?)",
	action->uuid, action->id);
	log_action(LOG_DEBUG, "Action for offline node", action, FALSE);
	return false;
	#if 0
	/* but this would also affect resources that can be safely
	* migrated before a fencing op
	*/
	} else if (action->node->details->unclean == false) {
	pe_err("Skipping action %s (%d) "
	"because it was scheduled for unclean node (bug?)",
	action->uuid, action->id);
	log_action(LOG_DEBUG, "Action for unclean node", action, false);
	return false;
	#endif
	}
	return true;
	}

	/* lowest to highest */
	static gint
	sort_action_id(gconstpointer a, gconstpointer b)
	{
	const pe_action_wrapper_t action_wrapper2 = (const pe_action_wrapper_t )a;
	const pe_action_wrapper_t action_wrapper1 = (const pe_action_wrapper_t )b;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (action_wrapper1->action->id > action_wrapper2->action->id) {
	return -1;
	}

	if (action_wrapper1->action->id < action_wrapper2->action->id) {
	return 1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Check whether an action input should be in the transition graph
	*
	* \param[in] action Action to check
	* \param[in,out] input Action input to check
	*
	* \return true if input should be in graph, false otherwise
	* \note This function may not only check an input, but disable it under certian
	* circumstances (load or anti-colocation orderings that are not needed).
	*/
	static bool
	check_dump_input(pe_action_t action, pe_action_wrapper_t input)
	{
	int type = input->type;

	if (input->state == pe_link_dumped) {
	return true;
	}

	pe__clear_order_flags(type, pe_order_implies_first_printed
	\|pe_order_implies_then_printed
	\|pe_order_optional);

	if (input->type == pe_order_none) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"ordering disabled",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
	&& (type == pe_order_none)
	&& !pcmk__str_eq(input->action->uuid, CRM_OP_PROBED, pcmk__str_casei)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional and input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
	&& pcmk_is_set(input->type, pe_order_one_or_more)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"one-or-more and input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(action->flags, pe_action_pseudo)
	&& pcmk_is_set(input->type, pe_order_stonith_stop)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"stonith stop but action is pseudo",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pe_order_implies_first_migratable)
	&& !pcmk_is_set(input->action->flags, pe_action_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"implies input migratable but input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pe_order_apply_first_non_migratable)
	&& pcmk_is_set(input->action->flags, pe_action_migrate_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"only if input unmigratable but input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if ((input->type == pe_order_optional)
	&& pcmk_is_set(input->action->flags, pe_action_migrate_runnable)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional but stop in migration",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (input->type == pe_order_load) {
	pe_node_t *input_node = input->action->node;

	// load orderings are relevant only if actions are for same node

	if (action->rsc && pcmk__str_eq(action->task, RSC_MIGRATE, pcmk__str_casei)) {
	pe_node_t *allocated = action->rsc->allocated_to;

	/* For load_stopped -> migrate_to orderings, we care about where it
	* has been allocated to, not where it will be executed.
	*/
	if ((input_node == NULL) \|\| (allocated == NULL)
	\|\| (input_node->details != allocated->details)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"load ordering node mismatch %s vs %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(allocated? allocated->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = pe_order_none;
	return false;
	}

	} else if ((input_node == NULL) \|\| (action->node == NULL)
	\|\| (input_node->details != action->node->details)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"load ordering node mismatch %s vs %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(action->node? action->node->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = pe_order_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"load ordering input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = pe_order_none;
	return false;
	}

	} else if (input->type == pe_order_anti_colocation) {
	if (input->action->node && action->node
	&& (input->action->node->details != action->node->details)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"anti-colocation node mismatch %s vs %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	action->node->details->uname,
	input->action->node->details->uname);
	input->type = pe_order_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"anti-colocation input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = pe_order_none;
	return false;
	}

	} else if (input->action->rsc
	&& input->action->rsc != action->rsc
	&& pcmk_is_set(input->action->rsc->flags, pe_rsc_failed)
	&& !pcmk_is_set(input->action->rsc->flags, pe_rsc_managed)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")
	&& action->rsc && pe_rsc_is_clone(action->rsc)) {
	crm_warn("Ignoring requirement that %s complete before %s:"
	" unmanaged failed resources cannot prevent clone shutdown",
	input->action->uuid, action->uuid);
	return false;

	} else if (pcmk_is_set(input->action->flags, pe_action_optional)
	&& !pcmk_any_flags_set(input->action->flags,
	pe_action_print_always\|pe_action_dumped)
	&& !should_dump_action(input->action)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;
	}

	crm_trace("%s (%d) input %s (%d) @ %s should be dumped: %s, %s, %s, 0x%.6x",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	input->action->node? input->action->node->details->uname : "no node",
	pcmk_is_set(input->action->flags, pe_action_pseudo)? "pseudo" : "real",
	pcmk_is_set(input->action->flags, pe_action_runnable)? "runnable" : "unrunnable",
	pcmk_is_set(input->action->flags, pe_action_optional)? "optional" : "required",
	input->type);
	return true;
	}

	static bool
	graph_has_loop(pe_action_t init_action, pe_action_t action,
	pe_action_wrapper_t *input)
	{
	bool has_loop = false;

	if (pcmk_is_set(input->action->flags, pe_action_tracking)) {
	crm_trace("Breaking tracking loop: %s@%s -> %s@%s (0x%.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	return false;
	}

	// Don't need to check inputs that won't be used
	if (!check_dump_input(action, input)) {
	return false;
	}

	if (input->action == init_action) {
	crm_debug("Input loop found in %s@%s ->...-> %s@%s",
	action->uuid,
	action->node? action->node->details->uname : "",
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");
	return true;
	}

	pe__set_action_flags(input->action, pe_action_tracking);

	crm_trace("Checking inputs of action %s@%s input %s@%s (0x%.6x)"
	"for graph loop with %s@%s ",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	input->type,
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");

	// Recursively check input itself for loops
	for (GList *iter = input->action->actions_before;
	iter != NULL; iter = iter->next) {

	if (graph_has_loop(init_action, input->action,
	(pe_action_wrapper_t *) iter->data)) {
	// Recursive call already logged a debug message
	has_loop = true;
	goto done;
	}
	}

	done:
	pe__clear_action_flags(input->action, pe_action_tracking);

	if (!has_loop) {
	crm_trace("No input loop found in %s@%s -> %s@%s (0x%.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	}
	return has_loop;
	}

	bool
	pcmk__ordering_is_invalid(pe_action_t action, pe_action_wrapper_t input)
	{
	/* Prevent user-defined ordering constraints between resources
	* running in a guest node and the resource that defines that node.
	*/
	if (!pcmk_is_set(input->type, pe_order_preserve)
	&& action->rsc && action->rsc->fillers
	&& input->action->rsc && input->action->node
	&& input->action->node->details->remote_rsc
	&& (input->action->node->details->remote_rsc->container == action->rsc)) {
	crm_warn("Invalid ordering constraint between %s and %s",
	input->action->rsc->id, action->rsc->id);
	return true;
	}

	/* If there's an order like
	* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
	*
	* then rscA is being migrated from node1 to node2, while rscB is being
	* migrated from node2 to node1. If there would be a graph loop,
	* break the order "load_stopped_node2" -> "rscA_migrate_to node1".
	*/
	if ((input->type == pe_order_load) && action->rsc
	&& pcmk__str_eq(action->task, RSC_MIGRATE, pcmk__str_casei)
	&& graph_has_loop(action, action, input)) {
	return true;
	}

	return false;
	}

	// Remove duplicate inputs (regardless of flags)
	static void
	deduplicate_inputs(pe_action_t *action)
	{
	GList *item = NULL;
	GList *next = NULL;
	pe_action_wrapper_t *last_input = NULL;

	action->actions_before = g_list_sort(action->actions_before,
	sort_action_id);
	for (item = action->actions_before; item != NULL; item = next) {
	pe_action_wrapper_t input = (pe_action_wrapper_t ) item->data;

	next = item->next;
	if (last_input && (input->action->id == last_input->action->id)) {
	crm_trace("Input %s (%d) duplicate skipped for action %s (%d)",
	input->action->uuid, input->action->id,
	action->uuid, action->id);

	/* For the purposes of scheduling, the ordering flags no longer
	* matter, but crm_simulate looks at certain ones when creating a
	* dot graph. Combining the flags is sufficient for that purpose.
	*/
	last_input->type \|= input->type;
	if (input->state == pe_link_dumped) {
	last_input->state = pe_link_dumped;
	}

	free(item->data);
	action->actions_before = g_list_delete_link(action->actions_before,
	item);
	} else {
	last_input = input;
	input->state = pe_link_not_dumped;
	}
	}
	}

	/*!
	* \internal
	* \brief Add an action to the transition graph XML if appropriate
	*
	* \param[in] action Action to possibly add
	* \param[in] data_set Cluster working set
	*
	* \note This will de-duplicate the action inputs, meaning that the
	* pe_action_wrapper_t:type flags can no longer be relied on to retain
	* their original settings. That means this MUST be called after stage7()
	* is complete, and nothing after this should rely on those type flags.
	* (For example, some code looks for type equal to some flag rather than
	* whether the flag is set, and some code looks for particular
	* combinations of flags -- such code must be done before stage8().)
	*/
	void
	graph_element_from_action(pe_action_t action, pe_working_set_t data_set)
	{
	GList *lpc = NULL;
	int synapse_priority = 0;
	xmlNode *syn = NULL;
	xmlNode *set = NULL;
	xmlNode *in = NULL;
	xmlNode *xml_action = NULL;
	pe_action_wrapper_t *input = NULL;

	/* If we haven't already, de-duplicate inputs -- even if we won't be dumping
	* the action, so that crm_simulate dot graphs don't have duplicates.
	*/
	if (!pcmk_is_set(action->flags, pe_action_dedup)) {
	deduplicate_inputs(action);
	pe__set_action_flags(action, pe_action_dedup);
	}

	if (should_dump_action(action) == FALSE) {
	return;
	}

	pe__set_action_flags(action, pe_action_dumped);

	syn = create_xml_node(data_set->graph, "synapse");
	set = create_xml_node(syn, "action_set");
	in = create_xml_node(syn, "inputs");

	crm_xml_add_int(syn, XML_ATTR_ID, data_set->num_synapse);
	data_set->num_synapse++;

	if (action->rsc != NULL) {
	synapse_priority = action->rsc->priority;
	}
	if (action->priority > synapse_priority) {
	synapse_priority = action->priority;
	}
	if (synapse_priority > 0) {
	crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
	}

	xml_action = action2xml(action, FALSE, data_set);
	add_node_nocopy(set, crm_element_name(xml_action), xml_action);

	for (lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
	input = (pe_action_wrapper_t *) lpc->data;
	if (check_dump_input(action, input)) {
	xmlNode *input_xml = create_xml_node(in, "trigger");

	input->state = pe_link_dumped;
	xml_action = action2xml(input->action, TRUE, data_set);
	add_node_nocopy(input_xml, crm_element_name(xml_action), xml_action);
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
	index 9c7fa6b935..88a4a0374b 100644
	--- a/lib/pacemaker/pcmk_sched_native.c
	+++ b/lib/pacemaker/pcmk_sched_native.c
	@@ -1,3221 +1,3223 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/pengine/rules.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>
	#include <pacemaker-internal.h>
	#include <crm/services.h>

	// The controller removes the resource from the CIB, making this redundant
	// #define DELETE_THEN_REFRESH 1

	#define INFINITY_HACK (INFINITY * -100)

	#define VARIANT_NATIVE 1
	#include <lib/pengine/variant.h>

	extern bool pcmk__is_daemon;

	static void Recurring(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	pe_working_set_t *data_set);
	static void RecurringOp(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	xmlNode operation, pe_working_set_t data_set);
	static void Recurring_Stopped(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	pe_working_set_t *data_set);
	static void RecurringOp_Stopped(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	xmlNode operation, pe_working_set_t data_set);

	void ReloadRsc(pe_resource_t * rsc, pe_node_t node, pe_working_set_t data_set);
	gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
	gboolean StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional,
	pe_working_set_t * data_set);
	gboolean RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);

	/* This array says what the next role should be when transitioning from one
	* role to another. For example going from Stopped to Promoted, the next role is
	* RSC_ROLE_UNPROMOTED, because the resource must be started before being promoted.
	* The current state then becomes Started, which is fed into this array again,
	* giving a next role of RSC_ROLE_PROMOTED.
	*/
	static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* Current state Next state*/
	/* Unknown Stopped Started Unpromoted Promoted */
	/* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED },
	/* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_UNPROMOTED },
	/* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_PROMOTED },
	/* Unpromoted */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_UNPROMOTED, RSC_ROLE_PROMOTED },
	/* Promoted */ { RSC_ROLE_STOPPED, RSC_ROLE_UNPROMOTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_PROMOTED },
	};

	typedef gboolean (rsc_transition_fn)(pe_resource_t rsc, pe_node_t *next,
	gboolean optional,
	pe_working_set_t *data_set);

	// This array picks the function needed to transition from one role to another
	static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* Current state Next state */
	/* Unknown Stopped Started Unpromoted Promoted */
	/* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
	/* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
	/* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
	/* Unpromoted */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, },
	/* Promoted */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp, },
	};

	#define clear_node_weights_flags(nw_flags, nw_rsc, flags_to_clear) do { \
	flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Node weight", (nw_rsc)->id, (flags), \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	static gboolean
	native_choose_node(pe_resource_t * rsc, pe_node_t * prefer, pe_working_set_t * data_set)
	{
	GList *nodes = NULL;
	pe_node_t *chosen = NULL;
	pe_node_t *best = NULL;
	int multiple = 1;
	int length = 0;
	gboolean result = FALSE;

	process_utilization(rsc, &prefer, data_set);

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to ? TRUE : FALSE;
	}

	// Sort allowed nodes by weight
	if (rsc->allowed_nodes) {
	length = g_hash_table_size(rsc->allowed_nodes);
	}
	if (length > 0) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	nodes = sort_nodes_by_weight(nodes, pe__current_node(rsc), data_set);

	// First node in sorted list has the best score
	best = g_list_nth_data(nodes, 0);
	}

	if (prefer && nodes) {
	chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);

	if (chosen == NULL) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
	prefer->details->uname, rsc->id);

	/* Favor the preferred node as long as its weight is at least as good as
	* the best allowed node's.
	*
	* An alternative would be to favor the preferred node even if the best
	* node is better, when the best node's weight is less than INFINITY.
	*/
	} else if ((chosen->weight < 0) \|\| (chosen->weight < best->weight)) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
	chosen->details->uname, rsc->id);
	chosen = NULL;

	} else if (!can_run_resources(chosen)) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
	chosen->details->uname, rsc->id);
	chosen = NULL;

	} else {
	pe_rsc_trace(rsc,
	"Chose preferred node %s for %s (ignoring %d candidates)",
	chosen->details->uname, rsc->id, length);
	}
	}

	if ((chosen == NULL) && nodes) {
	/* Either there is no preferred node, or the preferred node is not
	* available, but there are other nodes allowed to run the resource.
	*/

	chosen = best;
	pe_rsc_trace(rsc, "Chose node %s for %s from %d candidates",
	chosen ? chosen->details->uname : "<none>", rsc->id, length);

	if (!pe_rsc_is_unique_clone(rsc->parent)
	&& chosen && (chosen->weight > 0) && can_run_resources(chosen)) {
	/* If the resource is already running on a node, prefer that node if
	* it is just as good as the chosen node.
	*
	* We don't do this for unique clone instances, because
	* distribute_children() has already assigned instances to their
	* running nodes when appropriate, and if we get here, we don't want
	* remaining unallocated instances to prefer a node that's already
	* running another instance.
	*/
	pe_node_t *running = pe__current_node(rsc);

	if (running && (can_run_resources(running) == FALSE)) {
	pe_rsc_trace(rsc, "Current node for %s (%s) can't run resources",
	rsc->id, running->details->uname);
	} else if (running) {
	for (GList *iter = nodes->next; iter; iter = iter->next) {
	pe_node_t tmp = (pe_node_t ) iter->data;

	if (tmp->weight != chosen->weight) {
	// The nodes are sorted by weight, so no more are equal
	break;
	}
	if (tmp->details == running->details) {
	// Scores are equal, so prefer the current node
	chosen = tmp;
	}
	multiple++;
	}
	}
	}
	}

	if (multiple > 1) {
	static char score[33];
	int log_level = (chosen->weight >= INFINITY)? LOG_WARNING : LOG_INFO;

	score2char_stack(chosen->weight, score, sizeof(score));
	do_crm_log(log_level,
	"Chose node %s for %s from %d nodes with score %s",
	chosen->details->uname, rsc->id, multiple, score);
	}

	result = native_assign_node(rsc, chosen, FALSE);
	g_list_free(nodes);
	return result;
	}

	/*!
	* \internal
	* \brief Find score of highest-scored node that matches colocation attribute
	*
	* \param[in] rsc Resource whose allowed nodes should be searched
	* \param[in] attr Colocation attribute name (must not be NULL)
	* \param[in] value Colocation attribute value to require
	*/
	static int
	best_node_score_matching_attr(const pe_resource_t rsc, const char attr,
	const char *value)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	int best_score = -INFINITY;
	const char *best_node = NULL;

	// Find best allowed node with matching attribute
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {

	if ((node->weight > best_score) && can_run_resources(node)
	&& pcmk__str_eq(value, pe_node_attribute_raw(node, attr), pcmk__str_casei)) {

	best_score = node->weight;
	best_node = node->details->uname;
	}
	}

	if (!pcmk__str_eq(attr, CRM_ATTR_UNAME, pcmk__str_casei)) {
	if (best_node == NULL) {
	crm_info("No allowed node for %s matches node attribute %s=%s",
	rsc->id, attr, value);
	} else {
	crm_info("Allowed node %s for %s had best score (%d) "
	"of those matching node attribute %s=%s",
	best_node, rsc->id, best_score, attr, value);
	}
	}
	return best_score;
	}

	/*!
	* \internal
	* \brief Add resource's colocation matches to current node allocation scores
	*
	* For each node in a given table, if any of a given resource's allowed nodes
	* have a matching value for the colocation attribute, add the highest of those
	* nodes' scores to the node's score.
	*
	* \param[in,out] nodes Hash table of nodes with allocation scores so far
	* \param[in] rsc Resource whose allowed nodes should be compared
	* \param[in] attr Colocation attribute that must match (NULL for default)
	* \param[in] factor Factor by which to multiply scores being added
	* \param[in] only_positive Whether to add only positive scores
	*/
	static void
	add_node_scores_matching_attr(GHashTable nodes, const pe_resource_t rsc,
	const char *attr, float factor,
	bool only_positive)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (attr == NULL) {
	attr = CRM_ATTR_UNAME;
	}

	// Iterate through each node
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	float weight_f = 0;
	int weight = 0;
	int score = 0;
	int new_score = 0;

	score = best_node_score_matching_attr(rsc, attr,
	pe_node_attribute_raw(node, attr));

	if ((factor < 0) && (score < 0)) {
	/* Negative preference for a node with a negative score
	* should not become a positive preference.
	*
	* @TODO Consider filtering only if weight is -INFINITY
	*/
	crm_trace("%s: Filtering %d + %f * %d (double negative disallowed)",
	node->details->uname, node->weight, factor, score);
	continue;
	}

	if (node->weight == INFINITY_HACK) {
	crm_trace("%s: Filtering %d + %f * %d (node was marked unusable)",
	node->details->uname, node->weight, factor, score);
	continue;
	}

	weight_f = factor * score;

	// Round the number; see http://c-faq.com/fp/round.html
	weight = (int) ((weight_f < 0)? (weight_f - 0.5) : (weight_f + 0.5));

	/* Small factors can obliterate the small scores that are often actually
	* used in configurations. If the score and factor are nonzero, ensure
	* that the result is nonzero as well.
	*/
	if ((weight == 0) && (score != 0)) {
	if (factor > 0.0) {
	weight = 1;
	} else if (factor < 0.0) {
	weight = -1;
	}
	}

	new_score = pe__add_scores(weight, node->weight);

	if (only_positive && (new_score < 0) && (node->weight > 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d "
	"(negative disallowed, marking node unusable)",
	node->details->uname, node->weight, factor, score,
	new_score);
	node->weight = INFINITY_HACK;
	continue;
	}

	if (only_positive && (new_score < 0) && (node->weight == 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d (negative disallowed)",
	node->details->uname, node->weight, factor, score,
	new_score);
	continue;
	}

	crm_trace("%s: %d + %f * %d = %d", node->details->uname,
	node->weight, factor, score, new_score);
	node->weight = new_score;
	}
	}

	static inline bool
	is_nonempty_group(pe_resource_t *rsc)
	{
	return rsc && (rsc->variant == pe_group) && (rsc->children != NULL);
	}

	/*!
	* \internal
	* \brief Incorporate colocation constraint scores into node weights
	*
	* \param[in,out] rsc Resource being placed
	* \param[in] rhs ID of 'with' resource
	* \param[in,out] nodes Nodes, with scores as of this point
	* \param[in] attr Colocation attribute (ID by default)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pe_weights values
	*
	* \return Nodes, with scores modified by this constraint
	* \note This function assumes ownership of the nodes argument. The caller
	* should free the returned copy rather than the original.
	*/
	GHashTable *
	pcmk__native_merge_weights(pe_resource_t rsc, const char rhs,
	GHashTable nodes, const char attr, float factor,
	uint32_t flags)
	{
	GHashTable *work = NULL;

	// Avoid infinite recursion
	if (pcmk_is_set(rsc->flags, pe_rsc_merging)) {
	pe_rsc_info(rsc, "%s: Breaking dependency loop at %s", rhs, rsc->id);
	return nodes;
	}
	pe__set_resource_flags(rsc, pe_rsc_merging);

	if (pcmk_is_set(flags, pe_weights_init)) {
	if (is_nonempty_group(rsc)) {
	GList *last = g_list_last(rsc->children);
	pe_resource_t *last_rsc = last->data;

	pe_rsc_trace(rsc, "%s: Merging scores from group %s "
	"using last member %s (at %.6f)",
	rhs, rsc->id, last_rsc->id, factor);
	work = pcmk__native_merge_weights(last_rsc, rhs, NULL, attr, factor,
	flags);
	} else {
	work = pcmk__copy_node_table(rsc->allowed_nodes);
	}
	clear_node_weights_flags(flags, rsc, pe_weights_init);

	} else if (is_nonempty_group(rsc)) {
	/* The first member of the group will recursively incorporate any
	* constraints involving other members (including the group internal
	* colocation).
	*
	* @TODO The indirect colocations from the dependent group's other
	* members will be incorporated at full strength rather than by
	* factor, so the group's combined stickiness will be treated as
	* (factor + (#members - 1)) * stickiness. It is questionable what
	* the right approach should be.
	*/
	pe_rsc_trace(rsc, "%s: Merging scores from first member of group %s "
	"(at %.6f)", rhs, rsc->id, factor);
	work = pcmk__copy_node_table(nodes);
	work = pcmk__native_merge_weights(rsc->children->data, rhs, work, attr,
	factor, flags);

	} else {
	pe_rsc_trace(rsc, "%s: Merging scores from %s (at %.6f)",
	rhs, rsc->id, factor);
	work = pcmk__copy_node_table(nodes);
	add_node_scores_matching_attr(work, rsc, attr, factor,
	pcmk_is_set(flags, pe_weights_positive));
	}

	if (can_run_any(work)) {
	GList *gIter = NULL;
	int multiplier = (factor < 0)? -1 : 1;

	if (pcmk_is_set(flags, pe_weights_forward)) {
	gIter = rsc->rsc_cons;
	pe_rsc_trace(rsc,
	"Checking additional %d optional '%s with' constraints",
	g_list_length(gIter), rsc->id);

	} else if (is_nonempty_group(rsc)) {
	pe_resource_t *last_rsc = g_list_last(rsc->children)->data;

	gIter = last_rsc->rsc_cons_lhs;
	pe_rsc_trace(rsc, "Checking additional %d optional 'with group %s' "
	"constraints using last member %s",
	g_list_length(gIter), rsc->id, last_rsc->id);

	} else {
	gIter = rsc->rsc_cons_lhs;
	pe_rsc_trace(rsc,
	"Checking additional %d optional 'with %s' constraints",
	g_list_length(gIter), rsc->id);
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *other = NULL;
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	if (pcmk_is_set(flags, pe_weights_forward)) {
	other = constraint->rsc_rh;
	} else if (!pcmk__colocation_has_influence(constraint, NULL)) {
	continue;
	} else {
	other = constraint->rsc_lh;
	}

	pe_rsc_trace(rsc, "Optionally merging score of '%s' constraint (%s with %s)",
	constraint->id, constraint->rsc_lh->id,
	constraint->rsc_rh->id);
	work = pcmk__native_merge_weights(other, rhs, work,
	constraint->node_attribute,
	multiplier * constraint->score / (float) INFINITY,
	flags\|pe_weights_rollback);
	pe__show_node_weights(true, NULL, rhs, work, rsc->cluster);
	}

	} else if (pcmk_is_set(flags, pe_weights_rollback)) {
	pe_rsc_info(rsc, "%s: Rolling back optional scores from %s",
	rhs, rsc->id);
	g_hash_table_destroy(work);
	pe__clear_resource_flags(rsc, pe_rsc_merging);
	return nodes;
	}


	if (pcmk_is_set(flags, pe_weights_positive)) {
	pe_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->weight == INFINITY_HACK) {
	node->weight = 1;
	}
	}
	}

	if (nodes) {
	g_hash_table_destroy(nodes);
	}

	pe__clear_resource_flags(rsc, pe_rsc_merging);
	return work;
	}

	static inline bool
	node_has_been_unfenced(pe_node_t *node)
	{
	const char *unfenced = pe_node_attribute_raw(node, CRM_ATTR_UNFENCED);

	return !pcmk__str_eq(unfenced, "0", pcmk__str_null_matches);
	}

	static inline bool
	is_unfence_device(pe_resource_t rsc, pe_working_set_t data_set)
	{
	return pcmk_is_set(rsc->flags, pe_rsc_fence_device)
	&& pcmk_is_set(data_set->flags, pe_flag_enable_unfencing);
	}

	pe_node_t *
	pcmk__native_allocate(pe_resource_t rsc, pe_node_t prefer,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;

	if (rsc->parent && !pcmk_is_set(rsc->parent->flags, pe_rsc_allocating)) {
	/* never allocate children on their own */
	pe_rsc_debug(rsc, "Escalating allocation of %s to its parent: %s", rsc->id,
	rsc->parent->id);
	rsc->parent->cmds->allocate(rsc->parent, prefer, data_set);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	pe__set_resource_flags(rsc, pe_rsc_allocating);
	pe__show_node_weights(true, rsc, "Pre-alloc", rsc->allowed_nodes, data_set);

	for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	GHashTable *archive = NULL;
	pe_resource_t *rsc_rh = constraint->rsc_rh;

	if ((constraint->role_lh >= RSC_ROLE_PROMOTED)
	\|\| (constraint->score < 0 && constraint->score > -INFINITY)) {
	archive = pcmk__copy_node_table(rsc->allowed_nodes);
	}

	pe_rsc_trace(rsc,
	"%s: Allocating %s first (constraint=%s score=%d role=%s)",
	rsc->id, rsc_rh->id, constraint->id,
	constraint->score, role2text(constraint->role_lh));
	rsc_rh->cmds->allocate(rsc_rh, NULL, data_set);
	rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint, data_set);
	if (archive && can_run_any(rsc->allowed_nodes) == FALSE) {
	pe_rsc_info(rsc, "%s: Rolling back scores from %s", rsc->id, rsc_rh->id);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive) {
	g_hash_table_destroy(archive);
	}
	}

	pe__show_node_weights(true, rsc, "Post-coloc", rsc->allowed_nodes, data_set);

	for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	if (!pcmk__colocation_has_influence(constraint, NULL)) {
	continue;
	}
	pe_rsc_trace(rsc, "Merging score of '%s' constraint (%s with %s)",
	constraint->id, constraint->rsc_lh->id,
	constraint->rsc_rh->id);
	rsc->allowed_nodes =
	constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute,
	(float)constraint->score / INFINITY,
	pe_weights_rollback);
	}

	if (rsc->next_role == RSC_ROLE_STOPPED) {
	pe_rsc_trace(rsc, "Making sure %s doesn't get allocated", rsc->id);
	/* make sure it doesn't come up again */
	resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);

	} else if(rsc->next_role > rsc->role
	&& !pcmk_is_set(data_set->flags, pe_flag_have_quorum)
	&& data_set->no_quorum_policy == no_quorum_freeze) {
	crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role));
	pe__set_next_role(rsc, rsc->role, "no-quorum-policy=freeze");
	}

	pe__show_node_weights(!pcmk_is_set(data_set->flags, pe_flag_show_scores),
	rsc, __func__, rsc->allowed_nodes, data_set);
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)
	&& !pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	const char *reason = NULL;
	pe_node_t *assign_to = NULL;

	pe__set_next_role(rsc, rsc->role, "unmanaged");
	assign_to = pe__current_node(rsc);
	if (assign_to == NULL) {
	reason = "inactive";
	} else if (rsc->role == RSC_ROLE_PROMOTED) {
	reason = "promoted";
	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	reason = "failed";
	} else {
	reason = "active";
	}
	pe_rsc_info(rsc, "Unmanaged resource %s allocated to %s: %s", rsc->id,
	(assign_to? assign_to->details->uname : "no node"), reason);
	native_assign_node(rsc, assign_to, TRUE);

	} else if (pcmk_is_set(data_set->flags, pe_flag_stop_everything)) {
	pe_rsc_debug(rsc, "Forcing %s to stop", rsc->id);
	native_assign_node(rsc, NULL, TRUE);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_provisional)
	&& native_choose_node(rsc, prefer, data_set)) {
	pe_rsc_trace(rsc, "Allocated resource %s to %s", rsc->id,
	rsc->allocated_to->details->uname);

	} else if (rsc->allocated_to == NULL) {
	if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
	} else if (rsc->running_on != NULL) {
	pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
	}

	} else {
	pe_rsc_debug(rsc, "Pre-Allocated resource %s to %s", rsc->id,
	rsc->allocated_to->details->uname);
	}

	pe__clear_resource_flags(rsc, pe_rsc_allocating);

	if (rsc->is_remote_node) {
	pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);

	CRM_ASSERT(remote_node != NULL);
	if (rsc->allocated_to && rsc->next_role != RSC_ROLE_STOPPED) {
	crm_trace("Setting Pacemaker Remote node %s to ONLINE",
	remote_node->details->id);
	remote_node->details->online = TRUE;
	/* We shouldn't consider an unseen remote-node unclean if we are going
	* to try and connect to it. Otherwise we get an unnecessary fence */
	if (remote_node->details->unseen == TRUE) {
	remote_node->details->unclean = FALSE;
	}

	} else {
	crm_trace("Setting Pacemaker Remote node %s to SHUTDOWN (next role %s, %sallocated)",
	remote_node->details->id, role2text(rsc->next_role),
	(rsc->allocated_to? "" : "un"));
	remote_node->details->shutdown = TRUE;
	}
	}

	return rsc->allocated_to;
	}

	static gboolean
	is_op_dup(pe_resource_t rsc, const char name, guint interval_ms)
	{
	gboolean dup = FALSE;
	const char *id = NULL;
	const char *value = NULL;
	xmlNode *operation = NULL;
	guint interval2_ms = 0;

	CRM_ASSERT(rsc);
	for (operation = pcmk__xe_first_child(rsc->ops_xml); operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	value = crm_element_value(operation, "name");
	if (!pcmk__str_eq(value, name, pcmk__str_casei)) {
	continue;
	}

	value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval2_ms = crm_parse_interval_spec(value);
	if (interval_ms != interval2_ms) {
	continue;
	}

	if (id == NULL) {
	id = ID(operation);

	} else {
	pcmk__config_err("Operation %s is duplicate of %s (do not use "
	"same name and interval combination more "
	"than once per resource)", ID(operation), id);
	dup = TRUE;
	}
	}
	}

	return dup;
	}

	static bool
	op_cannot_recur(const char *name)
	{
	return pcmk__strcase_any_of(name, RSC_STOP, RSC_START, RSC_DEMOTE, RSC_PROMOTE, NULL);
	}

	static void
	RecurringOp(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *node_uname = node? node->details->uname : "n/a";

	guint interval_ms = 0;
	pe_action_t *mon = NULL;
	gboolean is_optional = TRUE;
	GList *possible_matches = NULL;

	CRM_ASSERT(rsc);

	/* Only process for the operations without role="Stopped" */
	role = crm_element_value(operation, "role");
	if (role && text2role(role) == RSC_ROLE_STOPPED) {
	return;
	}

	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_parse_interval_spec(interval_spec);
	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval_ms)) {
	crm_trace("Not creating duplicate recurring action %s for %dms %s",
	ID(operation), interval_ms, name);
	return;
	}

	if (op_cannot_recur(name)) {
	pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
	ID(operation), name);
	return;
	}

	key = pcmk__op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	crm_trace("Not creating recurring action %s for disabled resource %s",
	ID(operation), rsc->id);
	free(key);
	return;
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s in role %s on %s",
	ID(operation), rsc->id, role2text(rsc->next_role), node_uname);

	if (start != NULL) {
	pe_rsc_trace(rsc, "Marking %s %s due to %s", key,
	pcmk_is_set(start->flags, pe_action_optional)? "optional" : "mandatory",
	start->uuid);
	is_optional = (rsc->cmds->action_flags(start, NULL) & pe_action_optional);
	} else {
	pe_rsc_trace(rsc, "Marking %s optional", key);
	is_optional = TRUE;
	}

	/* start a monitor for an already active resource */
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches == NULL) {
	is_optional = FALSE;
	pe_rsc_trace(rsc, "Marking %s mandatory: not active", key);

	} else {
	GList *gIter = NULL;

	for (gIter = possible_matches; gIter != NULL; gIter = gIter->next) {
	pe_action_t op = (pe_action_t ) gIter->data;

	if (pcmk_is_set(op->flags, pe_action_reschedule)) {
	is_optional = FALSE;
	break;
	}
	}
	g_list_free(possible_matches);
	}

	if (((rsc->next_role == RSC_ROLE_PROMOTED) && (role == NULL))
	\|\| (role != NULL && text2role(role) != rsc->next_role)) {
	int log_level = LOG_TRACE;
	const char *result = "Ignoring";

	if (is_optional) {
	char *after_key = NULL;
	pe_action_t *cancel_op = NULL;

	// It's running, so cancel it
	log_level = LOG_INFO;
	result = "Cancelling";
	cancel_op = pe_cancel_op(rsc, name, interval_ms, node, data_set);

	switch (rsc->role) {
	case RSC_ROLE_UNPROMOTED:
	case RSC_ROLE_STARTED:
	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	after_key = promote_key(rsc);

	} else if (rsc->next_role == RSC_ROLE_STOPPED) {
	after_key = stop_key(rsc);
	}

	break;
	case RSC_ROLE_PROMOTED:
	after_key = demote_key(rsc);
	break;
	default:
	break;
	}

	if (after_key) {
	custom_action_order(rsc, NULL, cancel_op, rsc, after_key, NULL,
	pe_order_runnable_left, data_set);
	}
	}

	do_crm_log(log_level, "%s action %s (%s vs. %s)",
	result, key, role ? role : role2text(RSC_ROLE_UNPROMOTED),
	role2text(rsc->next_role));

	free(key);
	return;
	}

	mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set);
	key = mon->uuid;
	if (is_optional) {
	pe_rsc_trace(rsc, "%s\t %s (optional)", node_uname, mon->uuid);
	}

	if ((start == NULL) \|\| !pcmk_is_set(start->flags, pe_action_runnable)) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : start un-runnable)",
	node_uname, mon->uuid);
	update_action_flags(mon, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);

	} else if (node == NULL \|\| node->details->online == FALSE \|\| node->details->unclean) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)",
	node_uname, mon->uuid);
	update_action_flags(mon, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);

	} else if (!pcmk_is_set(mon->flags, pe_action_optional)) {
	pe_rsc_info(rsc, " Start recurring %s (%us) for %s on %s",
	mon->task, interval_ms / 1000, rsc->id, node_uname);
	}

	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	char *running_promoted = pcmk__itoa(PCMK_OCF_RUNNING_PROMOTED);

	add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_promoted);
	free(running_promoted);
	}

	if ((node == NULL) \|\| pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(rsc, start_key(rsc), NULL,
	NULL, strdup(key), mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);

	custom_action_order(rsc, reload_key(rsc), NULL,
	NULL, strdup(key), mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);

	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	custom_action_order(rsc, promote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional \| pe_order_runnable_left, data_set);

	} else if (rsc->role == RSC_ROLE_PROMOTED) {
	custom_action_order(rsc, demote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional \| pe_order_runnable_left, data_set);
	}
	}
	}

	static void
	Recurring(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_maintenance) &&
	(node == NULL \|\| node->details->maintenance == FALSE)) {
	xmlNode *operation = NULL;

	for (operation = pcmk__xe_first_child(rsc->ops_xml);
	operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	RecurringOp(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	static void
	RecurringOp_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *node_uname = node? node->details->uname : "n/a";

	guint interval_ms = 0;
	GList *possible_matches = NULL;
	GList *gIter = NULL;

	/* Only process for the operations with role="Stopped" */
	role = crm_element_value(operation, "role");
	if (role == NULL \|\| text2role(role) != RSC_ROLE_STOPPED) {
	return;
	}

	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_parse_interval_spec(interval_spec);
	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval_ms)) {
	crm_trace("Not creating duplicate recurring action %s for %dms %s",
	ID(operation), interval_ms, name);
	return;
	}

	if (op_cannot_recur(name)) {
	pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
	ID(operation), name);
	return;
	}

	key = pcmk__op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	crm_trace("Not creating recurring action %s for disabled resource %s",
	ID(operation), rsc->id);
	free(key);
	return;
	}

	// @TODO add support
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	crm_notice("Ignoring %s (recurring monitors for Stopped role are "
	"not supported for anonymous clones)",
	ID(operation));
	return;
	}

	pe_rsc_trace(rsc,
	"Creating recurring action %s for %s in role %s on nodes where it should not be running",
	ID(operation), rsc->id, role2text(rsc->next_role));

	/* if the monitor exists on the node where the resource will be running, cancel it */
	if (node != NULL) {
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches) {
	pe_action_t *cancel_op = NULL;

	g_list_free(possible_matches);

	cancel_op = pe_cancel_op(rsc, name, interval_ms, node, data_set);

	if ((rsc->next_role == RSC_ROLE_STARTED)
	\|\| (rsc->next_role == RSC_ROLE_UNPROMOTED)) {
	/* rsc->role == RSC_ROLE_STOPPED: cancel the monitor before start */
	/* rsc->role == RSC_ROLE_STARTED: for a migration, cancel the monitor on the target node before start */
	custom_action_order(rsc, NULL, cancel_op, rsc, start_key(rsc), NULL,
	pe_order_runnable_left, data_set);
	}

	pe_rsc_info(rsc, "Cancel action %s (%s vs. %s) on %s",
	key, role, role2text(rsc->next_role), node_uname);
	}
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t stop_node = (pe_node_t ) gIter->data;
	const char *stop_node_uname = stop_node->details->uname;
	gboolean is_optional = TRUE;
	gboolean probe_is_optional = TRUE;
	gboolean stop_is_optional = TRUE;
	pe_action_t *stopped_mon = NULL;
	char *rc_inactive = NULL;
	GList *probe_complete_ops = NULL;
	GList *stop_ops = NULL;
	GList *local_gIter = NULL;

	if (node && pcmk__str_eq(stop_node_uname, node_uname, pcmk__str_casei)) {
	continue;
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s on %s",
	ID(operation), rsc->id, crm_str(stop_node_uname));

	/* start a monitor for an already stopped resource */
	possible_matches = find_actions_exact(rsc->actions, key, stop_node);
	if (possible_matches == NULL) {
	pe_rsc_trace(rsc, "Marking %s mandatory on %s: not active", key,
	crm_str(stop_node_uname));
	is_optional = FALSE;
	} else {
	pe_rsc_trace(rsc, "Marking %s optional on %s: already active", key,
	crm_str(stop_node_uname));
	is_optional = TRUE;
	g_list_free(possible_matches);
	}

	stopped_mon = custom_action(rsc, strdup(key), name, stop_node, is_optional, TRUE, data_set);

	rc_inactive = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
	add_hash_param(stopped_mon->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
	free(rc_inactive);

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	GList *probes = pe__resource_actions(rsc, stop_node, RSC_STATUS,
	FALSE);
	GList *pIter = NULL;

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	pe_action_t probe = (pe_action_t ) pIter->data;

	order_actions(probe, stopped_mon, pe_order_runnable_left);
	crm_trace("%s then %s on %s", probe->uuid, stopped_mon->uuid, stop_node->details->uname);
	}

	g_list_free(probes);
	}

	if (probe_complete_ops) {
	g_list_free(probe_complete_ops);
	}

	stop_ops = pe__resource_actions(rsc, stop_node, RSC_STOP, TRUE);

	for (local_gIter = stop_ops; local_gIter != NULL; local_gIter = local_gIter->next) {
	pe_action_t stop = (pe_action_t ) local_gIter->data;

	if (!pcmk_is_set(stop->flags, pe_action_optional)) {
	stop_is_optional = FALSE;
	}

	if (!pcmk_is_set(stop->flags, pe_action_runnable)) {
	crm_debug("%s\t %s (cancelled : stop un-runnable)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	update_action_flags(stopped_mon, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(rsc, stop_key(rsc), stop,
	NULL, strdup(key), stopped_mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);
	}

	}

	if (stop_ops) {
	g_list_free(stop_ops);
	}

	if (is_optional == FALSE && probe_is_optional && stop_is_optional
	&& !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Marking %s optional on %s due to unmanaged",
	key, crm_str(stop_node_uname));
	update_action_flags(stopped_mon, pe_action_optional, __func__,
	__LINE__);
	}

	if (pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
	pe_rsc_trace(rsc, "%s\t %s (optional)", crm_str(stop_node_uname), stopped_mon->uuid);
	}

	if (stop_node->details->online == FALSE \|\| stop_node->details->unclean) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	update_action_flags(stopped_mon, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	}

	if (pcmk_is_set(stopped_mon->flags, pe_action_runnable)
	&& !pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
	crm_notice(" Start recurring %s (%us) for %s on %s", stopped_mon->task,
	interval_ms / 1000, rsc->id, crm_str(stop_node_uname));
	}
	}

	free(key);
	}

	static void
	Recurring_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_maintenance) &&
	(node == NULL \|\| node->details->maintenance == FALSE)) {
	xmlNode *operation = NULL;

	for (operation = pcmk__xe_first_child(rsc->ops_xml);
	operation != NULL;
	operation = pcmk__xe_next(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	RecurringOp_Stopped(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	static void
	handle_migration_actions(pe_resource_t * rsc, pe_node_t current, pe_node_t chosen, pe_working_set_t * data_set)
	{
	pe_action_t *migrate_to = NULL;
	pe_action_t *migrate_from = NULL;
	pe_action_t *start = NULL;
	pe_action_t *stop = NULL;
	gboolean partial = rsc->partial_migration_target ? TRUE : FALSE;

	pe_rsc_trace(rsc, "Processing migration actions %s moving from %s to %s . partial migration = %s",
	rsc->id, current->details->id, chosen->details->id, partial ? "TRUE" : "FALSE");
	start = start_action(rsc, chosen, TRUE);
	stop = stop_action(rsc, current, TRUE);

	if (partial == FALSE) {
	migrate_to = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	RSC_MIGRATE, current, TRUE, TRUE, data_set);
	}

	migrate_from = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
	RSC_MIGRATED, chosen, TRUE, TRUE, data_set);

	if ((migrate_to && migrate_from) \|\| (migrate_from && partial)) {

	pe__set_action_flags(start, pe_action_migrate_runnable);
	pe__set_action_flags(stop, pe_action_migrate_runnable);

	// This is easier than trying to delete it from the graph
	update_action_flags(start, pe_action_pseudo, __func__, __LINE__);

	/* order probes before migrations */
	if (partial) {
	pe__set_action_flags(migrate_from, pe_action_migrate_runnable);
	migrate_from->needs = start->needs;

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
	NULL, pe_order_optional, data_set);

	} else {
	pe__set_action_flags(migrate_from, pe_action_migrate_runnable);
	pe__set_action_flags(migrate_to, pe_action_migrate_runnable);
	migrate_to->needs = start->needs;

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	NULL, pe_order_optional, data_set);
	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	pe_order_optional\|pe_order_implies_first_migratable,
	data_set);
	}

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_optional\|pe_order_implies_first_migratable,
	data_set);
	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	pe_order_optional\|pe_order_implies_first_migratable\|pe_order_pseudo_left,
	data_set);

	}

	if (migrate_to) {
	add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
	add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);

	/* Pacemaker Remote connections don't require pending to be recorded in
	* the CIB. We can reduce CIB writes by not setting PENDING for them.
	*/
	if (rsc->is_remote_node == FALSE) {
	/* migrate_to takes place on the source node, but can
	* have an effect on the target node depending on how
	* the agent is written. Because of this, we have to maintain
	* a record that the migrate_to occurred, in case the source node
	* loses membership while the migrate_to action is still in-flight.
	*/
	add_hash_param(migrate_to->meta, XML_OP_ATTR_PENDING, "true");
	}
	}

	if (migrate_from) {
	add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
	add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
	}
	}

	void
	native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_action_t *start = NULL;
	pe_node_t *chosen = NULL;
	pe_node_t *current = NULL;
	gboolean need_stop = FALSE;
	bool need_promote = FALSE;
	gboolean is_moving = FALSE;
	gboolean allow_migrate = pcmk_is_set(rsc->flags, pe_rsc_allow_migrate)? TRUE : FALSE;

	GList *gIter = NULL;
	unsigned int num_all_active = 0;
	unsigned int num_clean_active = 0;
	bool multiply_active = FALSE;
	enum rsc_role_e role = RSC_ROLE_UNKNOWN;
	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;

	CRM_ASSERT(rsc);
	chosen = rsc->allocated_to;
	next_role = rsc->next_role;
	if (next_role == RSC_ROLE_UNKNOWN) {
	pe__set_next_role(rsc,
	(chosen == NULL)? RSC_ROLE_STOPPED : RSC_ROLE_STARTED,
	"allocation");
	}
	pe_rsc_trace(rsc, "Creating all actions for %s transition from %s to %s (%s) on %s",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role),
	((next_role == RSC_ROLE_UNKNOWN)? "implicit" : "explicit"),
	((chosen == NULL)? "no node" : chosen->details->uname));

	current = pe__find_active_on(rsc, &num_all_active, &num_clean_active);

	for (gIter = rsc->dangling_migrations; gIter != NULL; gIter = gIter->next) {
	pe_node_t dangling_source = (pe_node_t ) gIter->data;

	pe_action_t *stop = NULL;

	pe_rsc_trace(rsc, "Creating stop action %sfor %s on %s due to dangling migration",
	pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)? "and cleanup " : "",
	rsc->id, dangling_source->details->uname);
	stop = stop_action(rsc, dangling_source, FALSE);
	pe__set_action_flags(stop, pe_action_dangle);
	if (pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, dangling_source, FALSE, data_set);
	}
	}

	if ((num_all_active == 2) && (num_clean_active == 2) && chosen
	&& rsc->partial_migration_source && rsc->partial_migration_target
	&& (current->details == rsc->partial_migration_source->details)
	&& (chosen->details == rsc->partial_migration_target->details)) {

	/* The chosen node is still the migration target from a partial
	* migration. Attempt to continue the migration instead of recovering
	* by stopping the resource everywhere and starting it on a single node.
	*/
	pe_rsc_trace(rsc, "Will attempt to continue with partial migration "
	"to target %s from %s",
	rsc->partial_migration_target->details->id,
	rsc->partial_migration_source->details->id);

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	/* If a resource has "requires" set to nothing or quorum, don't consider
	* it active on unclean nodes (similar to how all resources behave when
	* stonith-enabled is false). We can start such resources elsewhere
	* before fencing completes, and if we considered the resource active on
	* the failed node, we would attempt recovery for being active on
	* multiple nodes.
	*/
	multiply_active = (num_clean_active > 1);
	} else {
	multiply_active = (num_all_active > 1);
	}

	if (multiply_active) {
	if (rsc->partial_migration_target && rsc->partial_migration_source) {
	// Migration was in progress, but we've chosen a different target
	crm_notice("Resource %s can no longer migrate from %s to %s "
	"(will stop on both nodes)",
	rsc->id, rsc->partial_migration_source->details->uname,
	rsc->partial_migration_target->details->uname);

	} else {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	// Resource was incorrectly multiply active
	pe_proc_err("%s resource %s is active on %u nodes (%s)",
	crm_str(class), rsc->id, num_all_active,
	recovery2text(rsc->recovery_type));
	crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ#Resource_is_Too_Active for more information");
	}

	if (rsc->recovery_type == recovery_stop_start) {
	need_stop = TRUE;
	}

	/* If by chance a partial migration is in process, but the migration
	* target is not chosen still, clear all partial migration data.
	*/
	rsc->partial_migration_source = rsc->partial_migration_target = NULL;
	allow_migrate = FALSE;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
	pe_rsc_trace(rsc, "Creating start action for %s to represent already pending start",
	rsc->id);
	start = start_action(rsc, chosen, TRUE);
	pe__set_action_flags(start, pe_action_print_always);
	}

	if (current && chosen && current->details != chosen->details) {
	pe_rsc_trace(rsc, "Moving %s from %s to %s",
	rsc->id, crm_str(current->details->uname),
	crm_str(chosen->details->uname));
	is_moving = TRUE;
	need_stop = TRUE;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	if (pcmk_is_set(rsc->flags, pe_rsc_stop)) {
	need_stop = TRUE;
	pe_rsc_trace(rsc, "Recovering %s", rsc->id);
	} else {
	pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	need_promote = TRUE;
	}
	}

	} else if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	pe_rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
	need_stop = TRUE;

	} else if (rsc->role > RSC_ROLE_STARTED && current != NULL && chosen != NULL) {
	pe_rsc_trace(rsc, "Creating start action for promoted resource %s",
	rsc->id);
	start = start_action(rsc, chosen, TRUE);
	if (!pcmk_is_set(start->flags, pe_action_optional)) {
	// Recovery of a promoted resource
	pe_rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
	need_stop = TRUE;
	}
	}

	/* Create any additional actions required when bringing resource down and
	* back up to same level.
	*/
	role = rsc->role;
	while (role != RSC_ROLE_STOPPED) {
	next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
	pe_rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
	(need_stop? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	if (rsc_action_matrix[role][next_role] (rsc, current, !need_stop, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}


	while ((rsc->role <= rsc->next_role) && (role != rsc->role)
	&& !pcmk_is_set(rsc->flags, pe_rsc_block)) {
	bool required = need_stop;

	next_role = rsc_state_matrix[role][rsc->role];
	if ((next_role == RSC_ROLE_PROMOTED) && need_promote) {
	required = true;
	}
	pe_rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
	(required? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	if (rsc_action_matrix[role][next_role](rsc, chosen, !required,
	data_set) == FALSE) {
	break;
	}
	role = next_role;
	}
	role = rsc->role;

	/* Required steps from this role to the next */
	while (role != rsc->next_role) {
	next_role = rsc_state_matrix[role][rsc->next_role];
	pe_rsc_trace(rsc, "Creating action to take %s from %s to %s (ending at %s)",
	rsc->id, role2text(role), role2text(next_role),
	role2text(rsc->next_role));
	if (rsc_action_matrix[role][next_role] (rsc, chosen, FALSE, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	pe_rsc_trace(rsc, "Not creating recurring monitors for blocked resource %s",
	rsc->id);

	} else if ((rsc->next_role != RSC_ROLE_STOPPED)
	\|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Creating recurring monitors for %s resource %s",
	((rsc->next_role == RSC_ROLE_STOPPED)? "unmanaged" : "active"),
	rsc->id);
	start = start_action(rsc, chosen, TRUE);
	Recurring(rsc, start, chosen, data_set);
	Recurring_Stopped(rsc, start, chosen, data_set);

	} else {
	pe_rsc_trace(rsc, "Creating recurring monitors for inactive resource %s",
	rsc->id);
	Recurring_Stopped(rsc, NULL, NULL, data_set);
	}

	/* if we are stuck in a partial migration, where the target
	* of the partial migration no longer matches the chosen target.
	* A full stop/start is required */
	if (rsc->partial_migration_target && (chosen == NULL \|\| rsc->partial_migration_target->details != chosen->details)) {
	pe_rsc_trace(rsc, "Not allowing partial migration of %s to continue",
	rsc->id);
	allow_migrate = FALSE;

	} else if (!is_moving \|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)
	\|\| pcmk_any_flags_set(rsc->flags,
	pe_rsc_failed\|pe_rsc_start_pending)
	\|\| (current && current->details->unclean)
	\|\| rsc->next_role < RSC_ROLE_STARTED) {

	allow_migrate = FALSE;
	}

	if (allow_migrate) {
	handle_migration_actions(rsc, current, chosen, data_set);
	}
	}

	static void
	rsc_avoids_remote_nodes(pe_resource_t *rsc)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->details->remote_rsc) {
	node->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Return allowed nodes as (possibly sorted) list
	*
	* Convert a resource's hash table of allowed nodes to a list. If printing to
	* stdout, sort the list, to keep action ID numbers consistent for regression
	* test output (while avoiding the performance hit on a live cluster).
	*
	* \param[in] rsc Resource to check for allowed nodes
	* \param[in] data_set Cluster working set
	*
	* \return List of resource's allowed nodes
	* \note Callers should take care not to rely on the list being sorted.
	*/
	static GList *
	allowed_nodes_as_list(pe_resource_t rsc, pe_working_set_t data_set)
	{
	GList *allowed_nodes = NULL;

	if (rsc->allowed_nodes) {
	allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	if (!pcmk__is_daemon) {
	allowed_nodes = g_list_sort(allowed_nodes, sort_node_uname);
	}

	return allowed_nodes;
	}

	void
	native_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	/* This function is on the critical path and worth optimizing as much as possible */

	pe_resource_t *top = NULL;
	GList *allowed_nodes = NULL;
	bool check_unfencing = FALSE;
	bool check_utilization = FALSE;

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc,
	"Skipping native constraints for unmanaged resource: %s",
	rsc->id);
	return;
	}

	top = uber_parent(rsc);

	// Whether resource requires unfencing
	check_unfencing = !pcmk_is_set(rsc->flags, pe_rsc_fence_device)
	&& pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)
	&& pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing);

	// Whether a non-default placement strategy is used
	check_utilization = (g_hash_table_size(rsc->utilization) > 0)
	&& !pcmk__str_eq(data_set->placement_strategy,
	"default", pcmk__str_casei);

	// Order stops before starts (i.e. restart)
	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	pe_order_optional\|pe_order_implies_then\|pe_order_restart,
	data_set);

	// Promotable ordering: demote before stop, start before promote
	if (pcmk_is_set(top->flags, pe_rsc_promotable)
	\|\| (rsc->role > RSC_ROLE_UNPROMOTED)) {

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_DEMOTE, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_promoted_implies_first, data_set);

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_PROMOTE, 0), NULL,
	pe_order_runnable_left, data_set);
	}

	// Don't clear resource history if probing on same node
	custom_action_order(rsc, pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0),
	NULL, rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0),
	NULL, pe_order_same_node\|pe_order_then_cancels_first,
	data_set);

	// Certain checks need allowed nodes
	if (check_unfencing \|\| check_utilization \|\| rsc->container) {
	allowed_nodes = allowed_nodes_as_list(rsc, data_set);
	}

	if (check_unfencing) {
	/* Check if the node needs to be unfenced first */

	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *node = item->data;
	pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);

	crm_debug("Ordering any stops of %s before %s, and any starts after",
	rsc->id, unfence->uuid);

	/*
	* It would be more efficient to order clone resources once,
	* rather than order each instance, but ordering the instance
	* allows us to avoid unnecessary dependencies that might conflict
	* with user constraints.
	*
	* @TODO: This constraint can still produce a transition loop if the
	* resource has a stop scheduled on the node being unfenced, and
	* there is a user ordering constraint to start some other resource
	* (which will be ordered after the unfence) before stopping this
	* resource. An example is "start some slow-starting cloned service
	* before stopping an associated virtual IP that may be moving to
	* it":
	* stop this -> unfencing -> start that -> stop this
	*/
	custom_action_order(rsc, stop_key(rsc), NULL,
	NULL, strdup(unfence->uuid), unfence,
	pe_order_optional\|pe_order_same_node, data_set);

	custom_action_order(NULL, strdup(unfence->uuid), unfence,
	rsc, start_key(rsc), NULL,
	pe_order_implies_then_on_node\|pe_order_same_node,
	data_set);
	}
	}

	if (check_utilization) {
	GList *gIter = NULL;

	pe_rsc_trace(rsc, "Creating utilization constraints for %s - strategy: %s",
	rsc->id, data_set->placement_strategy);

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t current = (pe_node_t ) gIter->data;

	char *load_stopped_task = crm_strdup_printf(LOAD_STOPPED "_%s",
	current->details->uname);
	pe_action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = pe__copy_node(current);
	update_action_flags(load_stopped, pe_action_optional \| pe_action_clear,
	__func__, __LINE__);
	}

	custom_action_order(rsc, stop_key(rsc), NULL,
	NULL, load_stopped_task, load_stopped, pe_order_load, data_set);
	}

	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *next = item->data;
	char *load_stopped_task = crm_strdup_printf(LOAD_STOPPED "_%s",
	next->details->uname);
	pe_action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = pe__copy_node(next);
	update_action_flags(load_stopped, pe_action_optional \| pe_action_clear,
	__func__, __LINE__);
	}

	custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
	rsc, start_key(rsc), NULL, pe_order_load, data_set);

	custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
	rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	NULL, pe_order_load, data_set);

	free(load_stopped_task);
	}
	}

	if (rsc->container) {
	pe_resource_t *remote_rsc = NULL;

	if (rsc->is_remote_node) {
	// rsc is the implicit remote connection for a guest or bundle node

	/* Do not allow a guest resource to live on a Pacemaker Remote node,
	* to avoid nesting remotes. However, allow bundles to run on remote
	* nodes.
	*/
	if (!pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
	rsc_avoids_remote_nodes(rsc->container);
	}

	/* If someone cleans up a guest or bundle node's container, we will
	* likely schedule a (re-)probe of the container and recovery of the
	* connection. Order the connection stop after the container probe,
	* so that if we detect the container running, we will trigger a new
	* transition and avoid the unnecessary recovery.
	*/
	new_rsc_order(rsc->container, RSC_STATUS, rsc, RSC_STOP,
	pe_order_optional, data_set);

	/* A user can specify that a resource must start on a Pacemaker Remote
	* node by explicitly configuring it with the container=NODENAME
	* meta-attribute. This is of questionable merit, since location
	* constraints can accomplish the same thing. But we support it, so here
	* we check whether a resource (that is not itself a remote connection)
	* has container set to a remote node or guest node resource.
	*/
	} else if (rsc->container->is_remote_node) {
	remote_rsc = rsc->container;
	} else {
	remote_rsc = pe__resource_contains_guest_node(data_set,
	rsc->container);
	}

	if (remote_rsc) {
	/* Force the resource on the Pacemaker Remote node instead of
	* colocating the resource with the container resource.
	*/
	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *node = item->data;

	if (node->details->remote_rsc != remote_rsc) {
	node->weight = -INFINITY;
	}
	}

	} else {
	/* This resource is either a filler for a container that does NOT
	* represent a Pacemaker Remote node, or a Pacemaker Remote
	* connection resource for a guest node or bundle.
	*/
	int score;

	crm_trace("Order and colocate %s relative to its container %s",
	rsc->id, rsc->container->id);

	custom_action_order(rsc->container,
	pcmk__op_key(rsc->container->id, RSC_START, 0),
	NULL, rsc, pcmk__op_key(rsc->id, RSC_START, 0),
	NULL,
	pe_order_implies_then\|pe_order_runnable_left,
	data_set);

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc->container,
	pcmk__op_key(rsc->container->id, RSC_STOP, 0),
	NULL, pe_order_implies_first, data_set);

	if (pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
	score = 10000; /* Highly preferred but not essential */
	} else {
	score = INFINITY; /* Force them to run on the same host */
	}
	pcmk__new_colocation("resource-with-container", NULL, score, rsc,
	rsc->container, NULL, NULL, true, data_set);
	}
	}

	if (rsc->is_remote_node \|\| pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
	/* don't allow remote nodes to run stonith devices
	* or remote connection resources.*/
	rsc_avoids_remote_nodes(rsc);
	}
	g_list_free(allowed_nodes);
	}

	void
	native_rsc_colocation_lh(pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	if (rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", constraint->id);
	return;

	} else if (constraint->rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;
	}

	pe_rsc_trace(rsc_lh, "Processing colocation constraint between %s and %s", rsc_lh->id,
	rsc_rh->id);

	rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint, data_set);
	}

	enum filter_colocation_res
	filter_colocation_constraint(pe_resource_t * rsc_lh, pe_resource_t * rsc_rh,
	pcmk__colocation_t *constraint, gboolean preview)
	{
	/* rh side must be allocated before we can process constraint */
	if (!preview && pcmk_is_set(rsc_rh->flags, pe_rsc_provisional)) {
	return influence_nothing;
	}

	if ((constraint->role_lh >= RSC_ROLE_UNPROMOTED) &&
	rsc_lh->parent && pcmk_is_set(rsc_lh->parent->flags, pe_rsc_promotable)
	&& !pcmk_is_set(rsc_lh->flags, pe_rsc_provisional)) {

	/* LH and RH resources have already been allocated, place the correct
	* priority on LH rsc for the given promotable clone resource role */
	return influence_rsc_priority;
	}

	if (!preview && !pcmk_is_set(rsc_lh->flags, pe_rsc_provisional)) {
	// Log an error if we violated a mandatory colocation constraint
	const pe_node_t *rh_node = rsc_rh->allocated_to;

	if (rsc_lh->allocated_to == NULL) {
	// Dependent resource isn't allocated, so constraint doesn't matter
	return influence_nothing;
	}

	if (constraint->score >= INFINITY) {
	// Dependent resource must colocate with rh_node

	if ((rh_node == NULL)
	\|\| (rh_node->details != rsc_lh->allocated_to->details)) {
	crm_err("%s must be colocated with %s but is not (%s vs. %s)",
	rsc_lh->id, rsc_rh->id,
	rsc_lh->allocated_to->details->uname,
	(rh_node? rh_node->details->uname : "unallocated"));
	}

	} else if (constraint->score <= -INFINITY) {
	// Dependent resource must anti-colocate with rh_node

	if ((rh_node != NULL)
	&& (rsc_lh->allocated_to->details == rh_node->details)) {
	crm_err("%s and %s must be anti-colocated but are allocated "
	"to the same node (%s)",
	rsc_lh->id, rsc_rh->id, rh_node->details->uname);
	}
	}
	return influence_nothing;
	}

	if (constraint->score > 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh != rsc_lh->next_role) {
	crm_trace("LH: Skipping constraint: \"%s\" state filter nextrole is %s",
	role2text(constraint->role_lh), role2text(rsc_lh->next_role));
	return influence_nothing;
	}

	if (constraint->score > 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh != rsc_rh->next_role) {
	crm_trace("RH: Skipping constraint: \"%s\" state filter", role2text(constraint->role_rh));
	return influence_nothing;
	}

	if (constraint->score < 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh == rsc_lh->next_role) {
	crm_trace("LH: Skipping negative constraint: \"%s\" state filter",
	role2text(constraint->role_lh));
	return influence_nothing;
	}

	if (constraint->score < 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh == rsc_rh->next_role) {
	crm_trace("RH: Skipping negative constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return influence_nothing;
	}

	return influence_rsc_location;
	}

	static void
	influence_priority(pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	pcmk__colocation_t *constraint)
	{
	const char *rh_value = NULL;
	const char *lh_value = NULL;
	const char *attribute = CRM_ATTR_ID;
	int score_multiplier = 1;

	if (!rsc_rh->allocated_to \|\| !rsc_lh->allocated_to) {
	return;
	}

	if (constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	lh_value = pe_node_attribute_raw(rsc_lh->allocated_to, attribute);
	rh_value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);

	if (!pcmk__str_eq(lh_value, rh_value, pcmk__str_casei)) {
	if ((constraint->score == INFINITY)
	&& (constraint->role_lh == RSC_ROLE_PROMOTED)) {
	rsc_lh->priority = -INFINITY;
	}
	return;
	}

	if (constraint->role_rh && (constraint->role_rh != rsc_rh->next_role)) {
	return;
	}

	if (constraint->role_lh == RSC_ROLE_UNPROMOTED) {
	score_multiplier = -1;
	}

	rsc_lh->priority = pe__add_scores(score_multiplier * constraint->score,
	rsc_lh->priority);
	}

	static void
	colocation_match(pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	pcmk__colocation_t *constraint)
	{
	const char *attribute = CRM_ATTR_ID;
	const char *value = NULL;
	GHashTable *work = NULL;
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	if (rsc_rh->allocated_to) {
	value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);

	} else if (constraint->score < 0) {
	// Nothing to do (anti-colocation with something that is not running)
	return;
	}

	work = pcmk__copy_node_table(rsc_lh->allowed_nodes);

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (rsc_rh->allocated_to == NULL) {
	pe_rsc_trace(rsc_lh, "%s: %s@%s -= %d (%s inactive)",
	constraint->id, rsc_lh->id, node->details->uname,
	constraint->score, rsc_rh->id);
	node->weight = pe__add_scores(-constraint->score, node->weight);

	} else if (pcmk__str_eq(pe_node_attribute_raw(node, attribute), value, pcmk__str_casei)) {
	if (constraint->score < CRM_SCORE_INFINITY) {
	pe_rsc_trace(rsc_lh, "%s: %s@%s += %d",
	constraint->id, rsc_lh->id,
	node->details->uname, constraint->score);
	node->weight = pe__add_scores(constraint->score, node->weight);
	}

	} else if (constraint->score >= CRM_SCORE_INFINITY) {
	pe_rsc_trace(rsc_lh, "%s: %s@%s -= %d (%s mismatch)",
	constraint->id, rsc_lh->id, node->details->uname,
	constraint->score, attribute);
	node->weight = pe__add_scores(-constraint->score, node->weight);
	}
	}

	if (can_run_any(work)
	\|\| constraint->score <= -INFINITY \|\| constraint->score >= INFINITY) {
	g_hash_table_destroy(rsc_lh->allowed_nodes);
	rsc_lh->allowed_nodes = work;
	work = NULL;

	} else {
	pe_rsc_info(rsc_lh,
	"%s: Rolling back scores from %s (no available nodes)",
	rsc_lh->id, rsc_rh->id);
	}

	if (work) {
	g_hash_table_destroy(work);
	}
	}

	void
	native_rsc_colocation_rh(pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	enum filter_colocation_res filter_results;

	CRM_ASSERT(rsc_lh);
	CRM_ASSERT(rsc_rh);
	filter_results = filter_colocation_constraint(rsc_lh, rsc_rh, constraint, FALSE);
	pe_rsc_trace(rsc_lh, "%s %s with %s (%s, score=%d, filter=%d)",
	((constraint->score > 0)? "Colocating" : "Anti-colocating"),
	rsc_lh->id, rsc_rh->id, constraint->id, constraint->score, filter_results);

	switch (filter_results) {
	case influence_rsc_priority:
	influence_priority(rsc_lh, rsc_rh, constraint);
	break;
	case influence_rsc_location:
	colocation_match(rsc_lh, rsc_rh, constraint);
	break;
	case influence_nothing:
	default:
	return;
	}
	}

	static gboolean
	filter_rsc_ticket(pe_resource_t * rsc_lh, rsc_ticket_t * rsc_ticket)
	{
	if (rsc_ticket->role_lh != RSC_ROLE_UNKNOWN && rsc_ticket->role_lh != rsc_lh->role) {
	pe_rsc_trace(rsc_lh, "LH: Skipping constraint: \"%s\" state filter",
	role2text(rsc_ticket->role_lh));
	return FALSE;
	}

	return TRUE;
	}

	void
	rsc_ticket_constraint(pe_resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working_set_t * data_set)
	{
	if (rsc_ticket == NULL) {
	pe_err("rsc_ticket was NULL");
	return;
	}

	if (rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", rsc_ticket->id);
	return;
	}

	if (rsc_ticket->ticket->granted && rsc_ticket->ticket->standby == FALSE) {
	return;
	}

	if (rsc_lh->children) {
	GList *gIter = rsc_lh->children;

	pe_rsc_trace(rsc_lh, "Processing ticket dependencies from %s", rsc_lh->id);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	rsc_ticket_constraint(child_rsc, rsc_ticket, data_set);
	}
	return;
	}

	pe_rsc_trace(rsc_lh, "%s: Processing ticket dependency on %s (%s, %s)",
	rsc_lh->id, rsc_ticket->ticket->id, rsc_ticket->id,
	role2text(rsc_ticket->role_lh));

	if ((rsc_ticket->ticket->granted == FALSE)
	&& (rsc_lh->running_on != NULL)) {

	GList *gIter = NULL;

	switch (rsc_ticket->loss_policy) {
	case loss_ticket_stop:
	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
	break;

	case loss_ticket_demote:
	// Promotion score will be set to -INFINITY in promotion_order()
	if (rsc_ticket->role_lh != RSC_ROLE_PROMOTED) {
	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
	}
	break;

	case loss_ticket_fence:
	if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
	return;
	}

	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);

	for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	pe_fence_node(data_set, node, "deadman ticket was lost", FALSE);
	}
	break;

	case loss_ticket_freeze:
	if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
	return;
	}
	if (rsc_lh->running_on != NULL) {
	pe__clear_resource_flags(rsc_lh, pe_rsc_managed);
	pe__set_resource_flags(rsc_lh, pe_rsc_block);
	}
	break;
	}

	} else if (rsc_ticket->ticket->granted == FALSE) {

	if ((rsc_ticket->role_lh != RSC_ROLE_PROMOTED)
	\|\| (rsc_ticket->loss_policy == loss_ticket_stop)) {
	resource_location(rsc_lh, NULL, -INFINITY, "__no_ticket__", data_set);
	}

	} else if (rsc_ticket->ticket->standby) {

	if ((rsc_ticket->role_lh != RSC_ROLE_PROMOTED)
	\|\| (rsc_ticket->loss_policy == loss_ticket_stop)) {
	resource_location(rsc_lh, NULL, -INFINITY, "__ticket_standby__", data_set);
	}
	}
	}

	enum pe_action_flags
	native_action_flags(pe_action_t * action, pe_node_t * node)
	{
	return action->flags;
	}

	static inline bool
	is_primitive_action(pe_action_t *action)
	{
	return action && action->rsc && (action->rsc->variant == pe_native);
	}

	/*!
	* \internal
	* \brief Set action bits appropriately when pe_restart_order is used
	*
	* \param[in] first 'First' action in an ordering with pe_restart_order
	* \param[in] then 'Then' action in an ordering with pe_restart_order
	* \param[in] filter What ordering flags to care about
	*
	* \note pe_restart_order is set for "stop resource before starting it" and
	* "stop later group member before stopping earlier group member"
	*/
	static void
	handle_restart_ordering(pe_action_t first, pe_action_t then,
	enum pe_action_flags filter)
	{
	const char *reason = NULL;

	CRM_ASSERT(is_primitive_action(first));
	CRM_ASSERT(is_primitive_action(then));

	// We need to update the action in two cases:

	// ... if 'then' is required
	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {
	reason = "restart";
	}

	/* ... if 'then' is unrunnable start of managed resource (if a resource
	* should restart but can't start, we still want to stop)
	*/
	if (pcmk_is_set(filter, pe_action_runnable)
	&& !pcmk_is_set(then->flags, pe_action_runnable)
	&& pcmk_is_set(then->rsc->flags, pe_rsc_managed)
	&& pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)) {
	reason = "stop";
	}

	if (reason == NULL) {
	return;
	}

	pe_rsc_trace(first->rsc, "Handling %s -> %s for %s",
	first->uuid, then->uuid, reason);

	// Make 'first' required if it is runnable
	if (pcmk_is_set(first->flags, pe_action_runnable)) {
	pe_action_implies(first, then, pe_action_optional);
	}

	// Make 'first' required if 'then' is required
	if (!pcmk_is_set(then->flags, pe_action_optional)) {
	pe_action_implies(first, then, pe_action_optional);
	}

	// Make 'first' unmigratable if 'then' is unmigratable
	if (!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	pe_action_implies(first, then, pe_action_migrate_runnable);
	}

	// Make 'then' unrunnable if 'first' is required but unrunnable
	if (!pcmk_is_set(first->flags, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_runnable)) {
	pe_action_implies(then, first, pe_action_runnable);
	}
	}

	enum pe_graph_flags
	native_update_actions(pe_action_t first, pe_action_t then, pe_node_t *node,
	enum pe_action_flags flags, enum pe_action_flags filter,
	enum pe_ordering type, pe_working_set_t *data_set)
	{
	/* flags == get_action_flags(first, then_node) called from update_action() */
	enum pe_graph_flags changed = pe_graph_none;
	enum pe_action_flags then_flags = then->flags;
	enum pe_action_flags first_flags = first->flags;

	crm_trace( "Testing %s on %s (0x%.6x) with %s 0x%.6x",
	first->uuid, first->node ? first->node->details->uname : "[none]",
	first->flags, then->uuid, then->flags);

	if (type & pe_order_asymmetrical) {
	pe_resource_t *then_rsc = then->rsc;
	enum rsc_role_e then_rsc_role = then_rsc ? then_rsc->fns->state(then_rsc, TRUE) : 0;

	if (!then_rsc) {
	/* ignore */
	} else if ((then_rsc_role == RSC_ROLE_STOPPED) && pcmk__str_eq(then->task, RSC_STOP, pcmk__str_casei)) {
	/* ignore... if 'then' is supposed to be stopped after 'first', but
	* then is already stopped, there is nothing to be done when non-symmetrical. */
	} else if ((then_rsc_role >= RSC_ROLE_STARTED)
	&& pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)
	&& pcmk_is_set(then->flags, pe_action_optional)
	&& then->node
	&& pcmk__list_of_1(then_rsc->running_on)
	&& then->node->details == ((pe_node_t *) then_rsc->running_on->data)->details) {
	/* Ignore. If 'then' is supposed to be started after 'first', but
	* 'then' is already started, there is nothing to be done when
	* asymmetrical -- unless the start is mandatory, which indicates
	* the resource is restarting, and the ordering is still needed.
	*/
	} else if (!(first->flags & pe_action_runnable)) {
	/* prevent 'then' action from happening if 'first' is not runnable and
	* 'then' has not yet occurred. */
	pe_action_implies(then, first, pe_action_optional);
	pe_action_implies(then, first, pe_action_runnable);

	pe_rsc_trace(then->rsc, "Unset optional and runnable on %s", then->uuid);
	} else {
	/* ignore... then is allowed to start/stop if it wants to. */
	}
	}

	if (type & pe_order_implies_first) {
	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(flags /* Should be then_flags? */, pe_action_optional)) {
	// Needs pcmk_is_set(first_flags, pe_action_optional) too?
	pe_rsc_trace(first->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_optional);
	}

	if (pcmk_is_set(flags, pe_action_migrate_runnable) &&
	!pcmk_is_set(then->flags, pe_action_migrate_runnable) &&
	!pcmk_is_set(then->flags, pe_action_optional)) {

	pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s",
	first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_migrate_runnable);
	}
	}

	if (type & pe_order_promoted_implies_first) {
	if ((filter & pe_action_optional) &&
	((then->flags & pe_action_optional) == FALSE) &&
	(then->rsc != NULL) && (then->rsc->role == RSC_ROLE_PROMOTED)) {
	pe_action_implies(first, then, pe_action_optional);

	if (pcmk_is_set(first->flags, pe_action_migrate_runnable) &&
	!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {

	pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_migrate_runnable);
	}
	pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
	}
	}

	if ((type & pe_order_implies_first_migratable)
	&& pcmk_is_set(filter, pe_action_optional)) {

	if (((then->flags & pe_action_migrate_runnable) == FALSE) \|\|
	((then->flags & pe_action_runnable) == FALSE)) {

	pe_rsc_trace(then->rsc, "Unset runnable on %s because %s is neither runnable or migratable", first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_runnable);
	}

	if ((then->flags & pe_action_optional) == 0) {
	pe_rsc_trace(then->rsc, "Unset optional on %s because %s is not optional", first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_optional);
	}
	}

	if ((type & pe_order_pseudo_left)
	&& pcmk_is_set(filter, pe_action_optional)) {

	if ((first->flags & pe_action_runnable) == FALSE) {
	pe_action_implies(then, first, pe_action_migrate_runnable);
	pe__clear_action_flags(then, pe_action_pseudo);
	pe_rsc_trace(then->rsc, "Unset pseudo on %s because %s is not runnable", then->uuid, first->uuid);
	}

	}

	if (pcmk_is_set(type, pe_order_runnable_left)
	&& pcmk_is_set(filter, pe_action_runnable)
	&& pcmk_is_set(then->flags, pe_action_runnable)
	&& !pcmk_is_set(flags, pe_action_runnable)) {
	pe_rsc_trace(then->rsc, "Unset runnable on %s because of %s", then->uuid, first->uuid);
	pe_action_implies(then, first, pe_action_runnable);
	pe_action_implies(then, first, pe_action_migrate_runnable);
	}

	if (pcmk_is_set(type, pe_order_implies_then)
	&& pcmk_is_set(filter, pe_action_optional)
	&& pcmk_is_set(then->flags, pe_action_optional)
	&& !pcmk_is_set(flags, pe_action_optional)) {

	/* in this case, treat migrate_runnable as if first is optional */
	if (!pcmk_is_set(first->flags, pe_action_migrate_runnable)) {
	pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", then->uuid, first->uuid);
	pe_action_implies(then, first, pe_action_optional);
	}
	}

	if (pcmk_is_set(type, pe_order_restart)) {
	handle_restart_ordering(first, then, filter);
	}

	if (then_flags != then->flags) {
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	pe_rsc_trace(then->rsc,
	"Then: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
	then->uuid, then->node ? then->node->details->uname : "[none]", then->flags,
	then_flags, first->uuid, first->flags);

	if(then->rsc && then->rsc->parent) {
	/* "X_stop then X_start" doesn't get handled for cloned groups unless we do this */
	update_action(then, data_set);
	}
	}

	if (first_flags != first->flags) {
	pe__set_graph_flags(changed, first, pe_graph_updated_first);
	pe_rsc_trace(first->rsc,
	"First: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
	first->uuid, first->node ? first->node->details->uname : "[none]",
	first->flags, first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	void
	native_rsc_location(pe_resource_t rsc, pe__location_t constraint)
	{
	GList *gIter = NULL;
	bool need_role = false;

	CRM_CHECK((constraint != NULL) && (rsc != NULL), return);

	// If a role was specified, ensure constraint is applicable
	need_role = (constraint->role_filter > RSC_ROLE_UNKNOWN);
	if (need_role && (constraint->role_filter != rsc->next_role)) {
	pe_rsc_trace(rsc,
	"Not applying %s to %s because role will be %s not %s",
	constraint->id, rsc->id, role2text(rsc->next_role),
	role2text(constraint->role_filter));
	return;
	}

	if (constraint->node_list_rh == NULL) {
	pe_rsc_trace(rsc, "Not applying %s to %s because no nodes match",
	constraint->id, rsc->id);
	return;
	}

	pe_rsc_trace(rsc, "Applying %s%s%s to %s", constraint->id,
	(need_role? " for role " : ""),
	(need_role? role2text(constraint->role_filter) : ""), rsc->id);

	for (gIter = constraint->node_list_rh; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;
	pe_node_t *other_node = NULL;

	other_node = (pe_node_t *) pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (other_node != NULL) {
	pe_rsc_trace(rsc, "* + %d on %s",
	node->weight, node->details->uname);
	other_node->weight = pe__add_scores(other_node->weight,
	node->weight);

	} else {
	pe_rsc_trace(rsc, "* = %d on %s",
	node->weight, node->details->uname);
	other_node = pe__copy_node(node);
	g_hash_table_insert(rsc->allowed_nodes, (gpointer) other_node->details->id, other_node);
	}

	if (other_node->rsc_discover_mode < constraint->discover_mode) {
	if (constraint->discover_mode == pe_discover_exclusive) {
	rsc->exclusive_discover = TRUE;
	}
	/* exclusive > never > always... always is default */
	other_node->rsc_discover_mode = constraint->discover_mode;
	}
	}
	}

	void
	native_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Processing actions from %s", rsc->id);

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	crm_trace("processing action %d for rsc=%s", action->id, rsc->id);
	graph_element_from_action(action, data_set);
	}

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->expand(child_rsc, data_set);
	}
	}

	#define STOP_SANITY_ASSERT(lineno) do { \
	if(current && current->details->unclean) { \
	/* It will be a pseudo op */ \
	} else if(stop == NULL) { \
	crm_err("%s:%d: No stop action exists for %s", \
	__func__, lineno, rsc->id); \
	CRM_ASSERT(stop != NULL); \
	} else if (pcmk_is_set(stop->flags, pe_action_optional)) { \
	crm_err("%s:%d: Action %s is still optional", \
	__func__, lineno, stop->uuid); \
	CRM_ASSERT(!pcmk_is_set(stop->flags, pe_action_optional)); \
	} \
	} while(0)

	void
	LogActions(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pcmk__output_t *out = data_set->priv;

	pe_node_t *next = NULL;
	pe_node_t *current = NULL;

	gboolean moving = FALSE;

	if(rsc->variant == pe_container) {
	pcmk__bundle_log_actions(rsc, data_set);
	return;
	}

	if (rsc->children) {
	GList *gIter = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	LogActions(child_rsc, data_set);
	}
	return;
	}

	next = rsc->allocated_to;
	if (rsc->running_on) {
	current = pe__current_node(rsc);
	if (rsc->role == RSC_ROLE_STOPPED) {
	/*
	* This can occur when resources are being recovered
	* We fiddle with the current role in native_create_actions()
	*/
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	if ((current == NULL) && pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	/* Don't log stopped orphans */
	return;
	}

	out->message(out, "rsc-action", rsc, current, next, moving);
	}

	gboolean
	StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t current = (pe_node_t ) gIter->data;
	pe_action_t *stop;

	if (rsc->partial_migration_target) {
	if (rsc->partial_migration_target->details == current->details) {
	pe_rsc_trace(rsc, "Filtered %s -> %s %s", current->details->uname,
	next->details->uname, rsc->id);
	continue;
	} else {
	pe_rsc_trace(rsc, "Forced on %s %s", current->details->uname, rsc->id);
	optional = FALSE;
	}
	}

	pe_rsc_trace(rsc, "%s on %s", rsc->id, current->details->uname);
	stop = stop_action(rsc, current, optional);

	if(rsc->allocated_to == NULL) {
	pe_action_set_reason(stop, "node availability", TRUE);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	update_action_flags(stop, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	}

	if (pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, current, optional, data_set);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
	pe_action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, FALSE, data_set);

	order_actions(stop, unfence, pe_order_implies_first);
	if (!node_has_been_unfenced(current)) {
	pe_proc_err("Stopping %s until %s can be unfenced", rsc->id, current->details->uname);
	}
	}
	}

	return TRUE;
	}

	static void
	order_after_unfencing(pe_resource_t rsc, pe_node_t node, pe_action_t *action,
	enum pe_ordering order, pe_working_set_t *data_set)
	{
	/* When unfencing is in use, we order unfence actions before any probe or
	* start of resources that require unfencing, and also of fence devices.
	*
	* This might seem to violate the principle that fence devices require
	* only quorum. However, fence agents that unfence often don't have enough
	* information to even probe or start unless the node is first unfenced.
	*/
	if (is_unfence_device(rsc, data_set)
	\|\| pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {

	/* Start with an optional ordering. Requiring unfencing would result in
	* the node being unfenced, and all its resources being stopped,
	* whenever a new resource is added -- which would be highly suboptimal.
	*/
	pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);

	order_actions(unfence, action, order);

	if (!node_has_been_unfenced(node)) {
	// But unfencing is required if it has never been done
	char *reason = crm_strdup_printf("required by %s %s",
	rsc->id, action->task);

	trigger_unfencing(NULL, node, reason, NULL, data_set);
	free(reason);
	}
	}
	}

	gboolean
	StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	pe_action_t *start = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s on %s %d %d", rsc->id, next ? next->details->uname : "N/A", optional, next ? next->weight : 0);
	start = start_action(rsc, next, TRUE);

	order_after_unfencing(rsc, next, start, pe_order_implies_then, data_set);

	if (pcmk_is_set(start->flags, pe_action_runnable) && !optional) {
	update_action_flags(start, pe_action_optional \| pe_action_clear,
	__func__, __LINE__);
	}


	return TRUE;
	}

	gboolean
	PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	gboolean runnable = TRUE;
	GList *action_list = NULL;

	CRM_ASSERT(rsc);
	CRM_CHECK(next != NULL, return FALSE);
	pe_rsc_trace(rsc, "%s on %s", rsc->id, next->details->uname);

	action_list = pe__resource_actions(rsc, next, RSC_START, TRUE);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t start = (pe_action_t ) gIter->data;

	if (!pcmk_is_set(start->flags, pe_action_runnable)) {
	runnable = FALSE;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	promote_action(rsc, next, optional);
	return TRUE;
	}

	pe_rsc_debug(rsc, "%s\tPromote %s (canceled)", next->details->uname, rsc->id);

	action_list = pe__resource_actions(rsc, next, RSC_PROMOTE, TRUE);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t promote = (pe_action_t ) gIter->data;

	update_action_flags(promote, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	}

	g_list_free(action_list);
	return TRUE;
	}

	gboolean
	DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);

	/* CRM_CHECK(rsc->next_role == RSC_ROLE_UNPROMOTED, return FALSE); */
	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t current = (pe_node_t ) gIter->data;

	pe_rsc_trace(rsc, "%s on %s", rsc->id, next ? next->details->uname : "N/A");
	demote_action(rsc, current, optional);
	}
	return TRUE;
	}

	gboolean
	RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	CRM_ASSERT(rsc);
	crm_err("%s on %s", rsc->id, next ? next->details->uname : "N/A");
	CRM_CHECK(FALSE, return FALSE);
	return FALSE;
	}

	gboolean
	NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);
	return FALSE;
	}

	gboolean
	DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set)
	{
	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_trace(rsc, "Resource %s not deleted from %s: failed", rsc->id, node->details->uname);
	return FALSE;

	} else if (node == NULL) {
	pe_rsc_trace(rsc, "Resource %s not deleted: NULL node", rsc->id);
	return FALSE;

	} else if (node->details->unclean \|\| node->details->online == FALSE) {
	pe_rsc_trace(rsc, "Resource %s not deleted from %s: unrunnable", rsc->id,
	node->details->uname);
	return FALSE;
	}

	crm_notice("Removing %s from %s", rsc->id, node->details->uname);

	delete_action(rsc, node, optional);

	new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE,
	optional ? pe_order_implies_then : pe_order_optional, data_set);

	new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START,
	optional ? pe_order_implies_then : pe_order_optional, data_set);

	return TRUE;
	}

	gboolean
	native_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
	gboolean force, pe_working_set_t * data_set)
	{
	enum pe_ordering flags = pe_order_optional;
	char *key = NULL;
	pe_action_t *probe = NULL;
	pe_node_t *running = NULL;
	pe_node_t *allowed = NULL;
	pe_resource_t *top = uber_parent(rsc);

	static const char *rc_promoted = NULL;
	static const char *rc_inactive = NULL;

	if (rc_inactive == NULL) {
	rc_inactive = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
	rc_promoted = pcmk__itoa(PCMK_OCF_RUNNING_PROMOTED);
	}

	CRM_CHECK(node != NULL, return FALSE);
	if (!force && !pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
	pe_rsc_trace(rsc, "Skipping active resource detection for %s", rsc->id);
	return FALSE;
	}

	if (pe__is_guest_or_remote_node(node)) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because Pacemaker Remote nodes cannot run stonith agents",
	rsc->id, node->details->id);
	return FALSE;
	} else if (pe__is_guest_node(node)
	&& pe__resource_contains_guest_node(data_set, rsc)) {
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because guest nodes cannot run resources containing guest nodes",
	rsc->id, node->details->id);
	return FALSE;
	} else if (rsc->is_remote_node) {
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because Pacemaker Remote nodes cannot host remote connections",
	rsc->id, node->details->id);
	return FALSE;
	}
	}

	if (rsc->children) {
	GList *gIter = NULL;
	gboolean any_created = FALSE;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	any_created = child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set)
	\|\| any_created;
	}

	return any_created;

	} else if ((rsc->container) && (!rsc->is_remote_node)) {
	pe_rsc_trace(rsc, "Skipping %s: it is within container %s", rsc->id, rsc->container->id);
	return FALSE;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_rsc_trace(rsc, "Skipping orphan: %s", rsc->id);
	return FALSE;
	}

	// Check whether resource is already known on node
	if (!force && g_hash_table_lookup(rsc->known_on, node->details->id)) {
	pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname);
	return FALSE;
	}

	allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (rsc->exclusive_discover \|\| top->exclusive_discover) {
	if (allowed == NULL) {
	/* exclusive discover is enabled and this node is not in the allowed list. */
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, A", rsc->id, node->details->id);
	return FALSE;
	} else if (allowed->rsc_discover_mode != pe_discover_exclusive) {
	/* exclusive discover is enabled and this node is not marked
	* as a node this resource should be discovered on */
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, B", rsc->id, node->details->id);
	return FALSE;
	}
	}

	if(allowed == NULL && node->rsc_discover_mode == pe_discover_never) {
	/* If this node was allowed to host this resource it would
	* have been explicitly added to the 'allowed_nodes' list.
	* However it wasn't and the node has discovery disabled, so
	* no need to probe for this resource.
	*/
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, C", rsc->id, node->details->id);
	return FALSE;
	}

	if (allowed && allowed->rsc_discover_mode == pe_discover_never) {
	/* this resource is marked as not needing to be discovered on this node */
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, discovery mode", rsc->id, node->details->id);
	return FALSE;
	}

	if (pe__is_guest_node(node)) {
	pe_resource_t *remote = node->details->remote_rsc->container;

	if(remote->role == RSC_ROLE_STOPPED) {
	/* If the container is stopped, then we know anything that
	* might have been inside it is also stopped and there is
	* no need to probe.
	*
	* If we don't know the container's state on the target
	* either:
	*
	* - the container is running, the transition will abort
	* and we'll end up in a different case next time, or
	*
	* - the container is stopped
	*
	* Either way there is no need to probe.
	*
	*/
	if(remote->allocated_to
	&& g_hash_table_lookup(remote->known_on, remote->allocated_to->details->id) == NULL) {
	/* For safety, we order the 'rsc' start after 'remote'
	* has been probed.
	*
	* Using 'top' helps for groups, but we may need to
	* follow the start's ordering chain backwards.
	*/
	custom_action_order(remote,
	pcmk__op_key(remote->id, RSC_STATUS, 0),
	NULL, top,
	pcmk__op_key(top->id, RSC_START, 0), NULL,
	pe_order_optional, data_set);
	}
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopped",
	rsc->id, node->details->id, remote->id);
	return FALSE;

	/* Here we really we want to check if remote->stop is required,
	* but that information doesn't exist yet
	*/
	} else if(node->details->remote_requires_reset
	\|\| node->details->unclean
	\|\| pcmk_is_set(remote->flags, pe_rsc_failed)
	\|\| remote->next_role == RSC_ROLE_STOPPED
	\|\| (remote->allocated_to
	&& pe_find_node(remote->running_on, remote->allocated_to->details->uname) == NULL)
	) {
	/* The container is stopping or restarting, don't start
	* 'rsc' until 'remote' stops as this also implies that
	* 'rsc' is stopped - avoiding the need to probe
	*/
	custom_action_order(remote, pcmk__op_key(remote->id, RSC_STOP, 0),
	NULL, top, pcmk__op_key(top->id, RSC_START, 0),
	NULL, pe_order_optional, data_set);
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopping, restarting or moving",
	rsc->id, node->details->id, remote->id);
	return FALSE;
	/* } else {
	* The container is running so there is no problem probing it
	*/
	}
	}

	key = pcmk__op_key(rsc->id, RSC_STATUS, 0);
	probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
	update_action_flags(probe, pe_action_optional \| pe_action_clear, __func__,
	__LINE__);

	order_after_unfencing(rsc, node, probe, pe_order_optional, data_set);

	/*
	* We need to know if it's running_on (not just known_on) this node
	* to correctly determine the target rc.
	*/
	running = pe_find_node_id(rsc->running_on, node->details->id);
	if (running == NULL) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);

	} else if (rsc->role == RSC_ROLE_PROMOTED) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_promoted);
	}

	crm_debug("Probing %s on %s (%s) %d %p", rsc->id, node->details->uname, role2text(rsc->role),
	pcmk_is_set(probe->flags, pe_action_runnable), rsc->running_on);

	if (is_unfence_device(rsc, data_set) \|\| !pe_rsc_is_clone(top)) {
	top = rsc;
	} else {
	crm_trace("Probing %s on %s (%s) as %s", rsc->id, node->details->uname, role2text(rsc->role), top->id);
	}

	if (!pcmk_is_set(probe->flags, pe_action_runnable)
	&& (rsc->running_on == NULL)) {
	/* Prevent the start from occurring if rsc isn't active, but
	* don't cause it to stop if it was active already
	*/
	pe__set_order_flags(flags, pe_order_runnable_left);
	}

	custom_action_order(rsc, NULL, probe,
	top, pcmk__op_key(top->id, RSC_START, 0), NULL,
	flags, data_set);

	- /* Before any reloads, if they exist */
	+ // Order the probe before any agent reload
	custom_action_order(rsc, NULL, probe,
	top, reload_key(rsc), NULL,
	pe_order_optional, data_set);

	#if 0
	// complete is always null currently
	if (!is_unfence_device(rsc, data_set)) {
	/* Normally rsc.start depends on probe complete which depends
	* on rsc.probe. But this can't be the case for fence devices
	* with unfencing, as it would create graph loops.
	*
	* So instead we explicitly order 'rsc.probe then rsc.start'
	*/
	order_actions(probe, complete, pe_order_implies_then);
	}
	#endif
	return TRUE;
	}

	/*!
	* \internal
	* \brief Check whether a resource is known on a particular node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return TRUE if resource (or parent if an anonymous clone) is known
	*/
	static bool
	rsc_is_known_on(pe_resource_t rsc, const pe_node_t node)
	{
	if (pe_hash_table_lookup(rsc->known_on, node->details->id)) {
	return TRUE;

	} else if ((rsc->variant == pe_native)
	&& pe_rsc_is_anon_clone(rsc->parent)
	&& pe_hash_table_lookup(rsc->parent->known_on, node->details->id)) {
	/* We check only the parent, not the uber-parent, because we cannot
	* assume that the resource is known if it is in an anonymously cloned
	* group (which may be only partially known).
	*/
	return TRUE;
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Order a resource's start and promote actions relative to fencing
	*
	* \param[in] rsc Resource to be ordered
	* \param[in] stonith_op Fence action
	* \param[in] data_set Cluster information
	*/
	static void
	native_start_constraints(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
	{
	pe_node_t *target;
	GList *gIter = NULL;

	CRM_CHECK(stonith_op && stonith_op->node, return);
	target = stonith_op->node;

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	switch (action->needs) {
	case rsc_req_nothing:
	// Anything other than start or promote requires nothing
	break;

	case rsc_req_stonith:
	order_actions(stonith_op, action, pe_order_optional);
	break;

	case rsc_req_quorum:
	if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
	&& pe_hash_table_lookup(rsc->allowed_nodes, target->details->id)
	&& !rsc_is_known_on(rsc, target)) {

	/* If we don't know the status of the resource on the node
	* we're about to shoot, we have to assume it may be active
	* there. Order the resource start after the fencing. This
	* is analogous to waiting for all the probes for a resource
	* to complete before starting it.
	*
	* The most likely explanation is that the DC died and took
	* its status with it.
	*/
	pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid,
	target->details->uname);
	order_actions(stonith_op, action,
	pe_order_optional \| pe_order_runnable_left);
	}
	break;
	}
	}
	}

	static void
	native_stop_constraints(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	GList *action_list = NULL;
	bool order_implicit = false;

	pe_resource_t *top = uber_parent(rsc);
	pe_action_t *parent_stop = NULL;
	pe_node_t *target;

	CRM_CHECK(stonith_op && stonith_op->node, return);
	target = stonith_op->node;

	/* Get a list of stop actions potentially implied by the fencing */
	action_list = pe__resource_actions(rsc, target, RSC_STOP, FALSE);

	/* If resource requires fencing, implicit actions must occur after fencing.
	*
	* Implied stops and demotes of resources running on guest nodes are always
	* ordered after fencing, even if the resource does not require fencing,
	* because guest node "fencing" is actually just a resource stop.
	*/
	if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)
	\|\| pe__is_guest_node(target)) {

	order_implicit = true;
	}

	if (action_list && order_implicit) {
	parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
	}

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	// The stop would never complete, so convert it into a pseudo-action.
	update_action_flags(action, pe_action_pseudo\|pe_action_runnable,
	__func__, __LINE__);

	if (order_implicit) {
	update_action_flags(action, pe_action_implied_by_stonith,
	__func__, __LINE__);

	/* Order the stonith before the parent stop (if any).
	*
	* Also order the stonith before the resource stop, unless the
	* resource is inside a bundle -- that would cause a graph loop.
	* We can rely on the parent stop's ordering instead.
	*
	* User constraints must not order a resource in a guest node
	* relative to the guest node container resource. The
	* pe_order_preserve flag marks constraints as generated by the
	* cluster and thus immune to that check (and is irrelevant if
	* target is not a guest).
	*/
	if (!pe_rsc_is_bundled(rsc)) {
	order_actions(stonith_op, action, pe_order_preserve);
	}
	order_actions(stonith_op, parent_stop, pe_order_preserve);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	crm_notice("Stop of failed resource %s is implicit %s %s is fenced",
	rsc->id, (order_implicit? "after" : "because"),
	target->details->uname);
	} else {
	crm_info("%s is implicit %s %s is fenced",
	action->uuid, (order_implicit? "after" : "because"),
	target->details->uname);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_notify)) {
	/* Create a second notification that will be delivered
	* immediately after the node is fenced
	*
	* Basic problem:
	* - C is a clone active on the node to be shot and stopping on another
	* - R is a resource that depends on C
	*
	* + C.stop depends on R.stop
	* + C.stopped depends on STONITH
	* + C.notify depends on C.stopped
	* + C.healthy depends on C.notify
	* + R.stop depends on C.healthy
	*
	* The extra notification here changes
	* + C.healthy depends on C.notify
	* into:
	* + C.healthy depends on C.notify'
	* + C.notify' depends on STONITH'
	* thus breaking the loop
	*/
	create_secondary_notification(action, rsc, stonith_op, data_set);
	}

	/* From Bug #1601, successful fencing must be an input to a failed resources stop action.

	However given group(rA, rB) running on nodeX and B.stop has failed,
	A := stop healthy resource (rA.stop)
	B := stop failed resource (pseudo operation B.stop)
	C := stonith nodeX
	A requires B, B requires C, C requires A
	This loop would prevent the cluster from making progress.

	This block creates the "C requires A" dependency and therefore must (at least
	for now) be disabled.

	Instead, run the block above and treat all resources on nodeX as B would be
	(marked as a pseudo op depending on the STONITH).

	TODO: Break the "A requires B" dependency in update_action() and re-enable this block

	} else if(is_stonith == FALSE) {
	crm_info("Moving healthy resource %s"
	" off %s before fencing",
	rsc->id, node->details->uname);

	* stop healthy resources before the
	* stonith op
	*
	custom_action_order(
	rsc, stop_key(rsc), NULL,
	NULL,strdup(CRM_OP_FENCE),stonith_op,
	pe_order_optional, data_set);
	*/
	}

	g_list_free(action_list);

	/* Get a list of demote actions potentially implied by the fencing */
	action_list = pe__resource_actions(rsc, target, RSC_DEMOTE, FALSE);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (action->node->details->online == FALSE \|\| action->node->details->unclean == TRUE
	\|\| pcmk_is_set(rsc->flags, pe_rsc_failed)) {

	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_info(rsc,
	"Demote of failed resource %s is implicit after %s is fenced",
	rsc->id, target->details->uname);
	} else {
	pe_rsc_info(rsc, "%s is implicit after %s is fenced",
	action->uuid, target->details->uname);
	}

	/* The demote would never complete and is now implied by the
	* fencing, so convert it into a pseudo-action.
	*/
	update_action_flags(action, pe_action_pseudo\|pe_action_runnable,
	__func__, __LINE__);

	if (pe_rsc_is_bundled(rsc)) {
	/* Do nothing, let the recovery be ordered after the parent's implied stop */

	} else if (order_implicit) {
	order_actions(stonith_op, action, pe_order_preserve\|pe_order_optional);
	}
	}
	}

	g_list_free(action_list);
	}

	void
	rsc_stonith_ordering(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
	{
	if (rsc->children) {
	GList *gIter = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	rsc_stonith_ordering(child_rsc, stonith_op, data_set);
	}

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id);

	} else {
	native_start_constraints(rsc, stonith_op, data_set);
	native_stop_constraints(rsc, stonith_op, data_set);
	}
	}

	void
	ReloadRsc(pe_resource_t * rsc, pe_node_t node, pe_working_set_t data_set)
	{
	GList *gIter = NULL;
	pe_action_t *reload = NULL;

	if (rsc->children) {
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	ReloadRsc(child_rsc, node, data_set);
	}
	return;

	} else if (rsc->variant > pe_native) {
	/* Complex resource with no children */
	return;

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "%s: unmanaged", rsc->id);
	return;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	/* We don't need to specify any particular actions here, normal failure
	* recovery will apply.
	*/
	- pe_rsc_trace(rsc, "%s: preventing reload because failed", rsc->id);
	+ pe_rsc_trace(rsc, "%s: preventing agent reload because failed",
	+ rsc->id);
	return;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
	/* If a resource's configuration changed while a start was pending,
	* force a full restart.
	*/
	- pe_rsc_trace(rsc, "%s: preventing reload because start pending", rsc->id);
	+ pe_rsc_trace(rsc, "%s: preventing agent reload because start pending",
	+ rsc->id);
	stop_action(rsc, node, FALSE);
	return;

	} else if (node == NULL) {
	pe_rsc_trace(rsc, "%s: not active", rsc->id);
	return;
	}

	pe_rsc_trace(rsc, "Processing %s", rsc->id);
	pe__set_resource_flags(rsc, pe_rsc_reload);

	- reload = custom_action(
	- rsc, reload_key(rsc), CRMD_ACTION_RELOAD, node, FALSE, TRUE, data_set);
	+ reload = custom_action(rsc, reload_key(rsc), CRMD_ACTION_RELOAD_AGENT, node,
	+ FALSE, TRUE, data_set);
	pe_action_set_reason(reload, "resource definition change", FALSE);

	custom_action_order(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
	pe_order_optional\|pe_order_then_cancels_first,
	data_set);
	custom_action_order(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
	pe_order_optional\|pe_order_then_cancels_first,
	data_set);
	}

	void
	native_append_meta(pe_resource_t * rsc, xmlNode * xml)
	{
	char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
	pe_resource_t *parent;

	if (value) {
	char *name = NULL;

	name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
	crm_xml_add(xml, name, value);
	free(name);
	}

	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
	if (value) {
	char *name = NULL;

	name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	for (parent = rsc; parent != NULL; parent = parent->parent) {
	if (parent->container) {
	crm_xml_add(xml, CRM_META"_"XML_RSC_ATTR_CONTAINER, parent->container->id);
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_utils.c b/lib/pacemaker/pcmk_sched_utils.c
	index 5fbb82d3d3..b97c0304aa 100644
	--- a/lib/pacemaker/pcmk_sched_utils.c
	+++ b/lib/pacemaker/pcmk_sched_utils.c
	@@ -1,785 +1,792 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>
	#include <crm/lrmd.h> // lrmd_event_data_t
	#include <crm/common/xml_internal.h>
	#include <pacemaker-internal.h>

	pe__location_t *
	rsc2node_new(const char id, pe_resource_t rsc,
	int node_weight, const char *discover_mode,
	pe_node_t foo_node, pe_working_set_t data_set)
	{
	pe__location_t *new_con = NULL;

	if (rsc == NULL \|\| id == NULL) {
	pe_err("Invalid constraint %s for rsc=%p", crm_str(id), rsc);
	return NULL;

	} else if (foo_node == NULL) {
	CRM_CHECK(node_weight == 0, return NULL);
	}

	new_con = calloc(1, sizeof(pe__location_t));
	if (new_con != NULL) {
	new_con->id = strdup(id);
	new_con->rsc_lh = rsc;
	new_con->node_list_rh = NULL;
	new_con->role_filter = RSC_ROLE_UNKNOWN;


	if (pcmk__str_eq(discover_mode, "always", pcmk__str_null_matches \| pcmk__str_casei)) {
	new_con->discover_mode = pe_discover_always;
	} else if (pcmk__str_eq(discover_mode, "never", pcmk__str_casei)) {
	new_con->discover_mode = pe_discover_never;
	} else if (pcmk__str_eq(discover_mode, "exclusive", pcmk__str_casei)) {
	new_con->discover_mode = pe_discover_exclusive;
	rsc->exclusive_discover = TRUE;
	} else {
	pe_err("Invalid %s value %s in location constraint", XML_LOCATION_ATTR_DISCOVERY, discover_mode);
	}

	if (foo_node != NULL) {
	pe_node_t *copy = pe__copy_node(foo_node);

	copy->weight = node_weight;
	new_con->node_list_rh = g_list_prepend(NULL, copy);
	}

	data_set->placement_constraints = g_list_prepend(data_set->placement_constraints, new_con);
	rsc->rsc_location = g_list_prepend(rsc->rsc_location, new_con);
	}

	return new_con;
	}

	gboolean
	can_run_resources(const pe_node_t * node)
	{
	if (node == NULL) {
	return FALSE;
	}
	#if 0
	if (node->weight < 0) {
	return FALSE;
	}
	#endif

	if (node->details->online == FALSE
	\|\| node->details->shutdown \|\| node->details->unclean
	\|\| node->details->standby \|\| node->details->maintenance) {
	crm_trace("%s: online=%d, unclean=%d, standby=%d, maintenance=%d",
	node->details->uname, node->details->online,
	node->details->unclean, node->details->standby, node->details->maintenance);
	return FALSE;
	}
	return TRUE;
	}

	/*!
	* \internal
	* \brief Copy a hash table of node objects
	*
	* \param[in] nodes Hash table to copy
	*
	* \return New copy of nodes (or NULL if nodes is NULL)
	*/
	GHashTable *
	pcmk__copy_node_table(GHashTable *nodes)
	{
	GHashTable *new_table = NULL;
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (nodes == NULL) {
	return NULL;
	}
	new_table = pcmk__strkey_table(NULL, free);
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	pe_node_t *new_node = pe__copy_node(node);

	g_hash_table_insert(new_table, (gpointer) new_node->details->id,
	new_node);
	}
	return new_table;
	}

	/*!
	* \internal
	* \brief Copy a list of node objects
	*
	* \param[in] list List to copy
	* \param[in] reset Set copies' scores to 0
	*
	* \return New list of shallow copies of nodes in original list
	*/
	GList *
	pcmk__copy_node_list(const GList *list, bool reset)
	{
	GList *result = NULL;

	for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pe_node_t *new_node = NULL;
	pe_node_t this_node = (pe_node_t ) gIter->data;

	new_node = pe__copy_node(this_node);
	if (reset) {
	new_node->weight = 0;
	}
	result = g_list_prepend(result, new_node);
	}
	return result;
	}

	struct node_weight_s {
	pe_node_t *active;
	pe_working_set_t *data_set;
	};

	/* return -1 if 'a' is more preferred
	* return 1 if 'b' is more preferred
	*/

	static gint
	sort_node_weight(gconstpointer a, gconstpointer b, gpointer data)
	{
	const pe_node_t node1 = (const pe_node_t )a;
	const pe_node_t node2 = (const pe_node_t )b;
	struct node_weight_s *nw = data;

	int node1_weight = 0;
	int node2_weight = 0;

	int result = 0;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	node1_weight = node1->weight;
	node2_weight = node2->weight;

	if (can_run_resources(node1) == FALSE) {
	node1_weight = -INFINITY;
	}
	if (can_run_resources(node2) == FALSE) {
	node2_weight = -INFINITY;
	}

	if (node1_weight > node2_weight) {
	crm_trace("%s (%d) > %s (%d) : weight",
	node1->details->uname, node1_weight, node2->details->uname, node2_weight);
	return -1;
	}

	if (node1_weight < node2_weight) {
	crm_trace("%s (%d) < %s (%d) : weight",
	node1->details->uname, node1_weight, node2->details->uname, node2_weight);
	return 1;
	}

	crm_trace("%s (%d) == %s (%d) : weight",
	node1->details->uname, node1_weight, node2->details->uname, node2_weight);

	if (pcmk__str_eq(nw->data_set->placement_strategy, "minimal", pcmk__str_casei)) {
	goto equal;
	}

	if (pcmk__str_eq(nw->data_set->placement_strategy, "balanced", pcmk__str_casei)) {
	result = compare_capacity(node1, node2);
	if (result < 0) {
	crm_trace("%s > %s : capacity (%d)",
	node1->details->uname, node2->details->uname, result);
	return -1;
	} else if (result > 0) {
	crm_trace("%s < %s : capacity (%d)",
	node1->details->uname, node2->details->uname, result);
	return 1;
	}
	}

	/* now try to balance resources across the cluster */
	if (node1->details->num_resources < node2->details->num_resources) {
	crm_trace("%s (%d) > %s (%d) : resources",
	node1->details->uname, node1->details->num_resources,
	node2->details->uname, node2->details->num_resources);
	return -1;

	} else if (node1->details->num_resources > node2->details->num_resources) {
	crm_trace("%s (%d) < %s (%d) : resources",
	node1->details->uname, node1->details->num_resources,
	node2->details->uname, node2->details->num_resources);
	return 1;
	}

	if (nw->active && nw->active->details == node1->details) {
	crm_trace("%s (%d) > %s (%d) : active",
	node1->details->uname, node1->details->num_resources,
	node2->details->uname, node2->details->num_resources);
	return -1;
	} else if (nw->active && nw->active->details == node2->details) {
	crm_trace("%s (%d) < %s (%d) : active",
	node1->details->uname, node1->details->num_resources,
	node2->details->uname, node2->details->num_resources);
	return 1;
	}
	equal:
	crm_trace("%s = %s", node1->details->uname, node2->details->uname);
	return strcmp(node1->details->uname, node2->details->uname);
	}

	GList *
	sort_nodes_by_weight(GList nodes, pe_node_t active_node,
	pe_working_set_t *data_set)
	{
	struct node_weight_s nw = { active_node, data_set };

	return g_list_sort_with_data(nodes, sort_node_weight, &nw);
	}

	void
	native_deallocate(pe_resource_t * rsc)
	{
	if (rsc->allocated_to) {
	pe_node_t *old = rsc->allocated_to;

	crm_info("Deallocating %s from %s", rsc->id, old->details->uname);
	pe__set_resource_flags(rsc, pe_rsc_provisional);
	rsc->allocated_to = NULL;

	old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc, rsc);
	old->details->num_resources--;
	/* old->count--; */
	calculate_utilization(old->details->utilization, rsc->utilization, TRUE);
	free(old);
	}
	}

	gboolean
	native_assign_node(pe_resource_t rsc, pe_node_t chosen, gboolean force)
	{
	pcmk__output_t *out = rsc->cluster->priv;

	CRM_ASSERT(rsc->variant == pe_native);

	if (force == FALSE && chosen != NULL) {
	bool unset = FALSE;

	if(chosen->weight < 0) {
	unset = TRUE;

	// Allow the graph to assume that the remote resource will come up
	} else if (!can_run_resources(chosen) && !pe__is_guest_node(chosen)) {
	unset = TRUE;
	}

	if(unset) {
	crm_debug("All nodes for resource %s are unavailable"
	", unclean or shutting down (%s: %d, %d)",
	rsc->id, chosen->details->uname, can_run_resources(chosen), chosen->weight);
	pe__set_next_role(rsc, RSC_ROLE_STOPPED, "node availability");
	chosen = NULL;
	}
	}

	/* todo: update the old node for each resource to reflect its
	* new resource count
	*/

	native_deallocate(rsc);
	pe__clear_resource_flags(rsc, pe_rsc_provisional);

	if (chosen == NULL) {
	GList *gIter = NULL;
	char *rc_inactive = pcmk__itoa(PCMK_OCF_NOT_RUNNING);

	crm_debug("Could not allocate a node for %s", rsc->id);
	pe__set_next_role(rsc, RSC_ROLE_STOPPED, "unable to allocate");

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t op = (pe_action_t ) gIter->data;
	const char *interval_ms_s = g_hash_table_lookup(op->meta, XML_LRM_ATTR_INTERVAL_MS);

	crm_debug("Processing %s", op->uuid);
	if(pcmk__str_eq(RSC_STOP, op->task, pcmk__str_casei)) {
	update_action_flags(op, pe_action_optional \| pe_action_clear,
	__func__, __LINE__);

	} else if(pcmk__str_eq(RSC_START, op->task, pcmk__str_casei)) {
	update_action_flags(op, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	//pe__set_resource_flags(rsc, pe_rsc_block);

	} else if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	if(pcmk__str_eq(rc_inactive, g_hash_table_lookup(op->meta, XML_ATTR_TE_TARGET_RC), pcmk__str_casei)) {
	/* This is a recurring monitor for the stopped state, leave it alone */

	} else {
	/* Normal monitor operation, cancel it */
	update_action_flags(op, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	}
	}
	}

	free(rc_inactive);
	return FALSE;
	}

	crm_debug("Assigning %s to %s", chosen->details->uname, rsc->id);
	rsc->allocated_to = pe__copy_node(chosen);

	chosen->details->allocated_rsc = g_list_prepend(chosen->details->allocated_rsc, rsc);
	chosen->details->num_resources++;
	chosen->count++;
	calculate_utilization(chosen->details->utilization, rsc->utilization, FALSE);

	if (pcmk_is_set(rsc->cluster->flags, pe_flag_show_utilization)) {
	out->message(out, "resource-util", rsc, chosen, __func__);
	}

	return TRUE;
	}

	void
	log_action(unsigned int log_level, const char pre_text, pe_action_t action, gboolean details)
	{
	const char *node_uname = NULL;
	const char *node_uuid = NULL;
	const char *desc = NULL;

	if (action == NULL) {
	crm_trace("%s%s: <NULL>", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
	return;
	}

	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	node_uname = NULL;
	node_uuid = NULL;

	} else if (action->node != NULL) {
	node_uname = action->node->details->uname;
	node_uuid = action->node->details->id;
	} else {
	node_uname = "<none>";
	node_uuid = NULL;
	}

	switch (text2task(action->task)) {
	case stonith_node:
	case shutdown_crm:
	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	desc = "Pseudo ";
	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	desc = "Optional ";
	} else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else if (pcmk_is_set(action->flags, pe_action_processed)) {
	desc = "";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	default:
	if (pcmk_is_set(action->flags, pe_action_optional)) {
	desc = "Optional ";
	} else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	desc = "Pseudo ";
	} else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else if (pcmk_is_set(action->flags, pe_action_processed)) {
	desc = "";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(action->rsc? action->rsc->id : "<none>"),
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	}

	if (details) {
	GList *gIter = NULL;

	crm_trace("\t\t====== Preceding Actions");

	gIter = action->actions_before;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) gIter->data;

	log_action(log_level + 1, "\t\t", other->action, FALSE);
	}

	crm_trace("\t\t====== Subsequent Actions");

	gIter = action->actions_after;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) gIter->data;

	log_action(log_level + 1, "\t\t", other->action, FALSE);
	}

	crm_trace("\t\t====== End");

	} else {
	crm_trace("\t\t(before=%d, after=%d)",
	g_list_length(action->actions_before), g_list_length(action->actions_after));
	}
	}

	gboolean
	can_run_any(GHashTable * nodes)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (nodes == NULL) {
	return FALSE;
	}

	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (can_run_resources(node) && node->weight >= 0) {
	return TRUE;
	}
	}

	return FALSE;
	}

	pe_action_t *
	create_pseudo_resource_op(pe_resource_t * rsc, const char task, bool optional, bool runnable, pe_working_set_t data_set)
	{
	pe_action_t *action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0),
	task, NULL, optional, TRUE, data_set);
	update_action_flags(action, pe_action_pseudo, __func__, __LINE__);
	update_action_flags(action, pe_action_runnable, __func__, __LINE__);
	if(runnable) {
	update_action_flags(action, pe_action_runnable, __func__, __LINE__);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Create an executor cancel op
	*
	* \param[in] rsc Resource of action to cancel
	* \param[in] task Name of action to cancel
	* \param[in] interval_ms Interval of action to cancel
	* \param[in] node Node of action to cancel
	* \param[in] data_set Working set of cluster
	*
	* \return Created op
	*/
	pe_action_t *
	pe_cancel_op(pe_resource_t rsc, const char task, guint interval_ms,
	pe_node_t node, pe_working_set_t data_set)
	{
	pe_action_t *cancel_op;
	char *interval_ms_s = crm_strdup_printf("%u", interval_ms);

	// @TODO dangerous if possible to schedule another action with this key
	char *key = pcmk__op_key(rsc->id, task, interval_ms);

	cancel_op = custom_action(rsc, key, RSC_CANCEL, node, FALSE, TRUE,
	data_set);

	free(cancel_op->task);
	cancel_op->task = strdup(RSC_CANCEL);

	free(cancel_op->cancel_task);
	cancel_op->cancel_task = strdup(task);

	add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, task);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL_MS, interval_ms_s);
	free(interval_ms_s);

	return cancel_op;
	}

	/*!
	* \internal
	* \brief Create a shutdown op for a scheduler transition
	*
	* \param[in] node Node being shut down
	* \param[in] data_set Working set of cluster
	*
	* \return Created op
	*/
	pe_action_t *
	sched_shutdown_op(pe_node_t node, pe_working_set_t data_set)
	{
	char *shutdown_id = crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN,
	node->details->uname);

	pe_action_t *shutdown_op = custom_action(NULL, shutdown_id, CRM_OP_SHUTDOWN,
	node, FALSE, TRUE, data_set);

	crm_notice("Scheduling shutdown of node %s", node->details->uname);
	shutdown_constraints(node, shutdown_op, data_set);
	add_hash_param(shutdown_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	return shutdown_op;
	}

	static char *
	generate_transition_magic(const char *transition_key, int op_status, int op_rc)
	{
	CRM_CHECK(transition_key != NULL, return NULL);
	return crm_strdup_printf("%d:%d;%s", op_status, op_rc, transition_key);
	}

	static void
	append_digest(lrmd_event_data_t op, xmlNode update, const char *version,
	const char *magic, int level)
	{
	/* this will enable us to later determine that the
	* resource's parameters have changed and we should force
	* a restart
	*/
	char *digest = NULL;
	xmlNode *args_xml = NULL;

	if (op->params == NULL) {
	return;
	}

	args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
	g_hash_table_foreach(op->params, hash2field, args_xml);
	pcmk__filter_op_for_digest(args_xml);
	digest = calculate_operation_digest(args_xml, version);

	#if 0
	if (level < get_crm_log_level()
	&& op->interval_ms == 0 && pcmk__str_eq(op->op_type, CRMD_ACTION_START, pcmk__str_none)) {
	char *digest_source = dump_xml_unformatted(args_xml);

	do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n",
	digest, ID(update), magic, digest_source);
	free(digest_source);
	}
	#endif
	crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);

	free_xml(args_xml);
	free(digest);
	}

	#define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"

	/*!
	* \internal
	* \brief Create XML for resource operation history update
	*
	* \param[in,out] parent Parent XML node to add to
	* \param[in,out] op Operation event data
	* \param[in] caller_version DC feature set
	* \param[in] target_rc Expected result of operation
	* \param[in] node Name of node on which operation was performed
	* \param[in] origin Arbitrary description of update source
	* \param[in] level A log message will be logged at this level
	*
	* \return Newly created XML node for history update
	*/
	xmlNode *
	pcmk__create_history_xml(xmlNode parent, lrmd_event_data_t op,
	const char *caller_version, int target_rc,
	const char node, const char origin, int level)
	{
	char *key = NULL;
	char *magic = NULL;
	char *op_id = NULL;
	char *op_id_additional = NULL;
	char *local_user_data = NULL;
	const char *exit_reason = NULL;

	xmlNode *xml_op = NULL;
	const char *task = NULL;

	CRM_CHECK(op != NULL, return NULL);
	do_crm_log(level, "%s: Updating resource %s after %s op %s (interval=%u)",
	origin, op->rsc_id, op->op_type, services_lrm_status_str(op->op_status),
	op->interval_ms);

	crm_trace("DC version: %s", caller_version);

	task = op->op_type;

	- /* Record a successful reload as a start, and a failed reload as a monitor,
	- * to make life easier for the scheduler when determining the current state.
	+ /* Record a successful agent reload as a start, and a failed one as a
	+ * monitor, to make life easier for the scheduler when determining the
	+ * current state.
	+ *
	+ * @COMPAT We should check "reload" here only if the operation was for a
	+ * pre-OCF-1.1 resource agent, but we don't know that here, and we should
	+ * only ever get results for actions scheduled by us, so we can reasonably
	+ * assume any "reload" is actually a pre-1.1 agent reload.
	*/
	- if (pcmk__str_eq(task, "reload", pcmk__str_none)) {
	+ if (pcmk__str_any_of(task, CRMD_ACTION_RELOAD, CRMD_ACTION_RELOAD_AGENT,
	+ NULL)) {
	if (op->op_status == PCMK_LRM_OP_DONE) {
	task = CRMD_ACTION_START;
	} else {
	task = CRMD_ACTION_STATUS;
	}
	}

	key = pcmk__op_key(op->rsc_id, task, op->interval_ms);
	if (pcmk__str_eq(task, CRMD_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = crm_meta_value(op->params, "notify_type");
	const char *n_task = crm_meta_value(op->params, "notify_operation");

	CRM_LOG_ASSERT(n_type != NULL);
	CRM_LOG_ASSERT(n_task != NULL);
	op_id = pcmk__notify_key(op->rsc_id, n_type, n_task);

	if (op->op_status != PCMK_LRM_OP_PENDING) {
	/* Ignore notify errors.
	*
	* @TODO It might be better to keep the correct result here, and
	* ignore it in process_graph_event().
	*/
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = 0;
	}

	} else if (did_rsc_op_fail(op, target_rc)) {
	op_id = pcmk__op_key(op->rsc_id, "last_failure", 0);
	if (op->interval_ms == 0) {
	// Ensure 'last' gets updated, in case record-pending is true
	op_id_additional = pcmk__op_key(op->rsc_id, "last", 0);
	}
	exit_reason = op->exit_reason;

	} else if (op->interval_ms > 0) {
	op_id = strdup(key);

	} else {
	op_id = pcmk__op_key(op->rsc_id, "last", 0);
	}

	again:
	xml_op = pcmk__xe_match(parent, XML_LRM_TAG_RSC_OP, XML_ATTR_ID, op_id);
	if (xml_op == NULL) {
	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
	}

	if (op->user_data == NULL) {
	crm_debug("Generating fake transition key for: " PCMK__OP_FMT
	" %d from %s", op->rsc_id, op->op_type, op->interval_ms,
	op->call_id, origin);
	local_user_data = pcmk__transition_key(-1, op->call_id, target_rc,
	FAKE_TE_ID);
	op->user_data = local_user_data;
	}

	if(magic == NULL) {
	magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
	}

	crm_xml_add(xml_op, XML_ATTR_ID, op_id);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
	crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
	crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
	crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, exit_reason == NULL ? "" : exit_reason);
	crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); /* For context during triage */

	crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
	crm_xml_add_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, op->interval_ms);

	if (compare_version("2.1", caller_version) <= 0) {
	if (op->t_run \|\| op->t_rcchange \|\| op->exec_time \|\| op->queue_time) {
	crm_trace("Timing data (" PCMK__OP_FMT
	"): last=%u change=%u exec=%u queue=%u",
	op->rsc_id, op->op_type, op->interval_ms,
	op->t_run, op->t_rcchange, op->exec_time, op->queue_time);

	if ((op->interval_ms != 0) && (op->t_rcchange != 0)) {
	// Recurring ops may have changed rc after initial run
	crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
	(long long) op->t_rcchange);
	} else {
	crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
	(long long) op->t_run);
	}

	crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
	crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
	}
	}

	if (pcmk__str_any_of(op->op_type, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
	/*
	* Record migrate_source and migrate_target always for migrate ops.
	*/
	const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;

	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));

	name = XML_LRM_ATTR_MIGRATE_TARGET;
	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
	}

	append_digest(op, xml_op, caller_version, magic, LOG_DEBUG);

	if (op_id_additional) {
	free(op_id);
	op_id = op_id_additional;
	op_id_additional = NULL;
	goto again;
	}

	if (local_user_data) {
	free(local_user_data);
	op->user_data = NULL;
	}
	free(magic);
	free(op_id);
	free(key);
	return xml_op;
	}

	pcmk__output_t *
	pcmk__new_logger(void)
	{
	int rc = pcmk_rc_ok;
	pcmk__output_t *out = NULL;
	const char* argv[] = { "", NULL };
	pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_LOG,
	{ NULL, NULL, NULL }
	};

	pcmk__register_formats(NULL, formats);
	rc = pcmk__output_new(&out, "log", NULL, (char**)argv);
	if ((rc != pcmk_rc_ok) \|\| (out == NULL)) {
	crm_err("Can't log resource details due to internal error: %s\n",
	pcmk_rc_str(rc));
	return NULL;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);
	return out;
	}
	diff --git a/tools/cibsecret.in b/tools/cibsecret.in
	index ce57a1845c..4569863af7 100644
	--- a/tools/cibsecret.in
	+++ b/tools/cibsecret.in
	@@ -1,440 +1,440 @@
	#!@BASH_PATH@

	# Copyright 2011-2020 the Pacemaker project contributors
	#
	# The version control history for this file may have further details.
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#

	# cibsecret
	#
	# Manage the secrets directory (by default, /var/lib/pacemaker/lrm/secrets).
	# Secrets are ASCII files, holding one value per file:
	# <secrets-directory>/<rsc>/<param>

	# These constants must track crm_exit_t values
	CRM_EX_OK=0
	CRM_EX_ERROR=1
	CRM_EX_NOT_INSTALLED=5
	CRM_EX_USAGE=64
	CRM_EX_UNAVAILABLE=69
	CRM_EX_OSFILE=72
	CRM_EX_CONFIG=78
	CRM_EX_DIGEST=104
	CRM_EX_NOSUCH=105
	CRM_EX_EXISTS=108

	LRM_CIBSECRETS="@LRM_CIBSECRETS_DIR@"

	PROG="$(basename "$0")"
	SSH_OPTS="-o StrictHostKeyChecking=no"
	MAGIC="lrm://"

	usage() {
	cat <<EOF
	cibsecret - manage sensitive information in Pacemaker CIB

	Usage:
	$PROG [<options>] <command> [<parameters>]

	Options:
	--help Show this message, then exit
	--version Display version information, then exit
	-C Don't read or write the CIB

	Commands and their parameters:
	set <resource-id> <resource-parameter> <value>
	Set the value of a sensitive resource parameter.

	get <resource-id> <resource-parameter>
	Display the locally stored value of a sensitive resource parameter.

	check <resource-id> <resource-parameter>
	Verify that the locally stored value of a sensitive resource parameter
	matches its locally stored MD5 hash.

	stash <resource-id> <resource-parameter>
	Make a non-sensitive resource parameter that is already in the CIB
	sensitive (move its value to a locally stored and protected file).
	This may not be used with -C.

	unstash <resource-id> <resource-parameter>
	Make a sensitive resource parameter that is already in the CIB
	non-sensitive (move its value from the locally stored file to the CIB).
	This may not be used with -C.

	delete <resource-id> <resource-parameter>
	Remove a sensitive resource parameter value.

	sync
	Copy all locally stored secrets to all other nodes.

	This command manages sensitive resource parameter values that should not be
	stored directly in Pacemaker's Cluster Information Base (CIB). Such values
	are handled by storing a special string directly in the CIB that tells
	Pacemaker to look in a separate, protected file for the actual value.

	The secret files are not encrypted, but protected by file system permissions
	such that only root can read or modify them.

	Since the secret files are stored locally, they must be synchronized across all
	cluster nodes. This command handles the synchronization using (in order of
	preference) pssh, pdsh, or ssh, so one of those must be installed. Before
	synchronizing, this command will ping the cluster nodes to determine which are
	alive, using fping if it is installed, otherwise the ping command. Installing
	fping is strongly recommended for better performance.

	Known limitations:

	This command can only be run from full cluster nodes (not Pacemaker Remote
	nodes).

	Changes are not atomic, so the cluster may use different values while a
	change is in progress. To avoid problems, it is recommended to put the
	cluster in maintenance mode when making changes with this command.

	- Changes in secret values do not trigger a reload or restart of the affected
	- resource, since they do not change the CIB. If a response is desired before
	- the next cluster recheck interval, any CIB change (such as setting a node
	- attribute) will trigger it.
	+ Changes in secret values do not trigger an agent reload or restart of the
	+ affected resource, since they do not change the CIB. If a response is
	+ desired before the next cluster recheck interval, any CIB change (such as
	+ setting a node attribute) will trigger it.

	If any node is down when changes to secrets are made, or a new node is
	later added to the cluster, it may have different values when it joins the
	cluster, before "$PROG sync" is run. To avoid this, it is recommended to
	run the sync command (from another node) before starting Pacemaker on the
	node.

	Examples:

	$PROG set ipmi_node1 passwd SecreT_PASS

	$PROG get ipmi_node1 passwd

	$PROG check ipmi_node1 passwd

	$PROG stash ipmi_node2 passwd

	$PROG sync
	EOF
	exit "$1"
	}

	check_usage() {
	case "$1" in
	set) [ "$2" -ne 4 ] && [ "$2" -ne 3 ] && usage 1 ;;
	get) [ "$2" -ne 3 ] && usage 1 ;;
	check) [ "$2" -ne 3 ] && usage 1 ;;
	stash) [ "$2" -ne 3 ] && usage 1 ;;
	unstash) [ "$2" -ne 3 ] && usage 1 ;;
	delete) [ "$2" -ne 3 ] && usage 1 ;;
	sync) [ "$2" -ne 1 ] && usage 1 ;;
	--help) usage $CRM_EX_OK ;;
	--version) crm_attribute --version; exit $? ;;
	*) usage $CRM_EX_USAGE ;;
	esac
	}

	fatal() {
	rc=$1
	shift
	echo "ERROR: $*"
	exit $rc
	}

	warn() {
	echo "WARNING: $*"
	}

	info() {
	echo "INFO: $*"
	}

	check_env() {
	which md5sum >/dev/null 2>&1 \|\|
	fatal $CRM_EX_NOT_INSTALLED "please install md5sum to run $PROG"
	if which pssh >/dev/null 2>&1; then
	rsh=pssh_fun
	rcp_to_from=pscp_fun

	# -q is a SUSE patch not present in upstream pssh
	PSSH_QUIET_OPTION=""
	pssh -q 2>&1\|grep "no such option: -q" > /dev/null \|\|
	PSSH_QUIET_OPTION="-q"
	elif which pdsh >/dev/null 2>&1; then
	rsh=pdsh_fun
	rcp_to_from=pdcp_fun
	elif which ssh >/dev/null 2>&1; then
	rsh=ssh_fun
	rcp_to_from=scp_fun
	else
	fatal $CRM_EX_NOT_INSTALLED "please install pssh, pdsh, or ssh to run $PROG"
	fi
	ps -ef \| grep '[p]acemaker-controld' >/dev/null \|\|
	fatal $CRM_EX_UNAVAILABLE "pacemaker not running? $PROG needs pacemaker"
	}

	# This must be called (and return success) before calling $rsh or $rcp_to_from
	get_live_peers() {
	# Get local node name
	GLP_LOCAL_NODE="$(crm_node -n)"
	[ $? -eq 0 ] \|\| fatal $CRM_EX_UNAVAILABLE "couldn't get local node name"

	# Get a list of all other cluster nodes
	GLP_ALL_PEERS="$(crmadmin -N -q)"
	[ $? -eq 0 ] \|\| fatal $CRM_EX_UNAVAILABLE "couldn't determine cluster nodes"
	GLP_ALL_PEERS="$(echo "$GLP_ALL_PEERS" \| grep -v "^${GLP_LOCAL_NODE}$")"

	# Make a list of those that respond to pings
	if [ "$(id -u)" = "0" ] && which fping >/dev/null 2>&1; then
	LIVE_NODES=$(fping -a $GLP_ALL_PEERS 2>/dev/null)
	else
	LIVE_NODES=""
	for GLP_NODE in $GLP_ALL_PEERS; do \
	ping -c 2 -q "$GLP_NODE" >/dev/null 2>&1 &&
	LIVE_NODES="$LIVE_NODES $GLP_NODE"
	done
	fi

	# Warn the user about any that didn't respond to pings
	GLP_DOWN="$( (for GLP_NODE in $LIVE_NODES $GLP_ALL_PEERS; do echo "$GLP_NODE"; done) \| sort \| uniq -u)"
	if [ "$(echo "$GLP_DOWN" \| wc -w)" = "1" ]; then
	warn "node $GLP_DOWN is down"
	warn "you'll need to update it using \"$PROG sync\" later"
	elif [ -n "$GLP_DOWN" ]; then
	warn "nodes $(echo "$GLP_DOWN" \| tr '\n' ' ')are down"
	warn "you'll need to update them using \"$PROG sync\" later"
	fi

	if [ "$LIVE_NODES" = "" ]; then
	info "no other nodes live"
	return 1
	fi
	return 0
	}

	pssh_fun() {
	pssh $PSSH_QUIET_OPTION -i -H "$LIVE_NODES" -x "$SSH_OPTS" -- "$@"
	}

	pscp_fun() {
	PSCP_DEST="$1"
	shift
	pscp $PSSH_QUIET_OPTION -H "$LIVE_NODES" -x "-pr" -x "$SSH_OPTS" -- "$@" "$PSCP_DEST"
	}

	pdsh_fun() {
	PDSH_NODES=$(echo "$LIVE_NODES" \| tr '[:space:]' ',')
	export PDSH_SSH_ARGS_APPEND="$SSH_OPTS"
	pdsh -w "$PDSH_NODES" -- "$@"
	}

	pdcp_fun() {
	PDCP_DEST="$1"
	shift
	PDCP_NODES=$(echo "$LIVE_NODES" \| tr '[:space:]' ',')
	export PDSH_SSH_ARGS_APPEND="$SSH_OPTS"
	pdcp -pr -w "$PDCP_NODES" -- "$@" "$PDCP_DEST"
	}

	ssh_fun() {
	for SSH_NODE in $LIVE_NODES; do
	ssh $SSH_OPTS "$SSH_NODE" -- "$@" \|\| return
	done
	}

	scp_fun() {
	SCP_DEST="$1"
	shift
	for SCP_NODE in $LIVE_NODES; do
	scp -pqr $SSH_OPTS "$@" "$SCP_NODE:$SCP_DEST" \|\| return
	done
	}

	# TODO: this procedure should be replaced with csync2
	# provided that csync2 has already been configured
	sync_files() {
	get_live_peers \|\| return
	if [ "$cmd" != "delete" ]; then
	info "syncing $LRM_CIBSECRETS to $(echo "$LIVE_NODES" \| tr '\n' ' ') ..."
	else
	info "deleting $LRM_CIBSECRETS from $(echo "$LIVE_NODES" \| tr '\n' ' ') ..."
	fi
	$rsh rm -rf "$LRM_CIBSECRETS" &&
	$rsh mkdir -p "$(dirname "$LRM_CIBSECRETS")" &&
	$rcp_to_from "$(dirname "$LRM_CIBSECRETS")" "$LRM_CIBSECRETS"
	}

	sync_one() {
	SO_FILE="$1"
	get_live_peers \|\| return
	if [ "$cmd" != "delete" ]; then
	info "syncing $SO_FILE to $(echo "$LIVE_NODES" \| tr '\n' ' ') ..."
	else
	info "deleting $SO_FILE from $(echo "$LIVE_NODES" \| tr '\n' ' ') ..."
	fi
	$rsh mkdir -p "$(dirname "$SO_FILE")" &&
	if [ -f "$SO_FILE" ]; then
	$rcp_to_from "$(dirname "$SO_FILE")" "$SO_FILE" "${SO_FILE}.sign"
	else
	$rsh rm -f "$SO_FILE" "${SO_FILE}.sign"
	fi
	}

	is_secret() {
	# assume that the secret is in the CIB if we cannot talk to cib
	[ "$NO_CRM" ] \|\| test "$1" = "$MAGIC"
	}

	check_cib_rsc() {
	CCR_OUT="$($NO_CRM crm_resource -r "$1" -W 2>&1)" \|\| fatal $CRM_EX_NOSUCH "$CCR_OUT"
	}

	get_cib_param() {
	GCP_RSC="$1"
	GCP_PARAM="$2"
	$NO_CRM crm_resource -r "$GCP_RSC" -g "$GCP_PARAM" 2>/dev/null
	}

	set_cib_param() {
	SET_RSC="$1"
	SET_PARAM="$2"
	SET_VAL="$3"
	$NO_CRM crm_resource -r "$SET_RSC" -p "$SET_PARAM" -v "$SET_VAL" 2>/dev/null
	}

	remove_cib_param() {
	RM_RSC="$1"
	RM_PARAM="$2"
	$NO_CRM crm_resource -r "$RM_RSC" -d "$RM_PARAM" 2>/dev/null
	}

	localfiles() {
	LF_CMD="$1"
	LF_RSC="$2"
	LF_PARAM="$3"
	LF_VALUE="$4"
	LF_FILE="$LRM_CIBSECRETS/$LF_RSC/$LF_PARAM"
	case "$LF_CMD" in
	get)
	cat "$LF_FILE" 2>/dev/null
	true
	;;

	getsum)
	cat "${LF_FILE}.sign" 2>/dev/null
	true
	;;

	set)
	LF_SUM="$(printf %s "$LF_VALUE" \| md5sum)" \|\|
	fatal $CRM_EX_ERROR "md5sum failed to produce hash for resource $LF_RSC parameter $LF_PARAM"
	LF_SUM="$(echo "$LF_SUM" \| awk '{print $1}')"
	mkdir -p "$(dirname "$LF_FILE")" &&
	echo "$LF_VALUE" > "$LF_FILE" &&
	echo "$LF_SUM" > "${LF_FILE}.sign" &&
	sync_one "$LF_FILE"
	;;

	remove)
	rm -f "$LF_FILE" "${LF_FILE}.sign"
	sync_one "$LF_FILE"
	;;
	esac
	}

	cibsecret_set() {
	CS_VALUE="$1"

	if [ "$2" -ne 4 ]; then
	read -p "Enter value: " CS_VALUE
	fi

	check_cib_rsc "$rsc"
	CIBSET_CURRENT="$(get_cib_param "$rsc" "$param")"
	[ -z "$NO_CRM" ] &&
	[ ! -z "$CIBSET_CURRENT" ] &&
	[ "$CIBSET_CURRENT" != "$MAGIC" ] &&
	[ "$CIBSET_CURRENT" != "$CS_VALUE" ] &&
	fatal $CRM_EX_CONFIG "CIB value <$CIBSET_CURRENT> different for $rsc parameter $param; please delete it first"
	localfiles set "$rsc" "$param" "$CS_VALUE" &&
	set_cib_param "$rsc" "$param" "$MAGIC"
	}

	cibsecret_check() {
	check_cib_rsc "$rsc"
	is_secret "$(get_cib_param "$rsc" "$param")" \|\|
	fatal $CRM_EX_CONFIG "resource $rsc parameter $param not set as secret, nothing to check"
	CSC_LOCAL_SUM="$(localfiles getsum "$rsc" "$param")"
	[ "$CSC_LOCAL_SUM" ] \|\|
	fatal $CRM_EX_OSFILE "no MD5 hash for resource $rsc parameter $param"
	CSC_LOCAL_VALUE="$(localfiles get "$rsc" "$param")"
	CSC_CALC_SUM="$(printf "%s" "$CSC_LOCAL_VALUE" \| md5sum \| awk '{print $1}')"
	[ "$CSC_CALC_SUM" = "$CSC_LOCAL_SUM" ] \|\|
	fatal $CRM_EX_DIGEST "MD5 hash mismatch for resource $rsc parameter $param"
	}

	cibsecret_get() {
	cibsecret_check
	localfiles get "$rsc" "$param"
	}

	cibsecret_delete() {
	check_cib_rsc "$rsc"
	localfiles remove "$rsc" "$param" && remove_cib_param "$rsc" "$param"
	}

	cibsecret_stash() {
	[ "$NO_CRM" ] && fatal $CRM_EX_USAGE "no access to Pacemaker, stash not supported"
	check_cib_rsc "$rsc"
	CIBSTASH_CURRENT="$(get_cib_param "$rsc" "$param")"
	[ "$CIBSTASH_CURRENT" = "" ] &&
	fatal $CRM_EX_NOSUCH "nothing to stash for resource $rsc parameter $param"
	is_secret "$CIBSTASH_CURRENT" &&
	fatal $CRM_EX_EXISTS "resource $rsc parameter $param already set as secret, nothing to stash"
	cibsecret_set "$CIBSTASH_CURRENT" 4
	}

	cibsecret_unstash() {
	[ "$NO_CRM" ] && fatal $CRM_EX_USAGE "no access to Pacemaker, unstash not supported"
	UNSTASH_LOCAL_VALUE="$(localfiles get "$rsc" "$param")"
	[ "$UNSTASH_LOCAL_VALUE" = "" ] &&
	fatal $CRM_EX_NOSUCH "nothing to unstash for resource $rsc parameter $param"
	check_cib_rsc "$rsc"
	is_secret "$(get_cib_param "$rsc" "$param")" \|\|
	warn "resource $rsc parameter $param not set as secret, but we have local value so proceeding anyway"
	localfiles remove "$rsc" "$param" &&
	set_cib_param "$rsc" "$param" "$UNSTASH_LOCAL_VALUE"
	}

	cibsecret_sync() {
	sync_files
	}

	# Grab arguments
	if [ "$1" = "-C" ]; then
	NO_CRM=':'
	shift
	fi
	cmd="$1"
	rsc="$2"
	param="$3"
	value="$4"

	# Ensure we have everything we need
	check_usage "$cmd" $#
	check_env
	umask 0077

	# for dirname() function (@TODO why are we replacing dirname?)
	. "@OCF_ROOT_DIR@/lib/heartbeat/ocf-shellfuncs"

	"cibsecret_$cmd" "$value" $#
	rc=$?

	if [ $rc -ne 0 ]; then
	fatal $CRM_EX_ERROR "$cmd(): failed with rc: $rc"
	fi

	# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80:

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 1:16 PM (1 d, 7 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1659989
Default Alt Text: (378 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions