No OneTemporary
Actions

Size

236 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
	index 57e22ef8e9..83ba8228ab 100644
	--- a/lib/pacemaker/pcmk_sched_native.c
	+++ b/lib/pacemaker/pcmk_sched_native.c
	@@ -1,3491 +1,3503 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>
	#include <crm/services.h>

	// The controller removes the resource from the CIB, making this redundant
	// #define DELETE_THEN_REFRESH 1

	#define INFINITY_HACK (INFINITY * -100)

	#define VARIANT_NATIVE 1
	#include <lib/pengine/variant.h>

	static void Recurring(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	pe_working_set_t *data_set);
	static void RecurringOp(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	xmlNode operation, pe_working_set_t data_set);
	static void Recurring_Stopped(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	pe_working_set_t *data_set);
	static void RecurringOp_Stopped(pe_resource_t rsc, pe_action_t start, pe_node_t *node,
	xmlNode operation, pe_working_set_t data_set);

	void ReloadRsc(pe_resource_t * rsc, pe_node_t node, pe_working_set_t data_set);
	gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
	gboolean StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional,
	pe_working_set_t * data_set);
	gboolean RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);

	/* This array says what the next role should be when transitioning from one
	* role to another. For example going from Stopped to Master, the next role is
	* RSC_ROLE_SLAVE, because the resource must be started before being promoted.
	* The current state then becomes Started, which is fed into this array again,
	* giving a next role of RSC_ROLE_MASTER.
	*/
	static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* Current state Next state*/
	/* Unknown Stopped Started Slave Master */
	/* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, },
	/* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, },
	/* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	/* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	/* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	};

	typedef gboolean (rsc_transition_fn)(pe_resource_t rsc, pe_node_t *next,
	gboolean optional,
	pe_working_set_t *data_set);

	// This array picks the function needed to transition from one role to another
	static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* Current state Next state */
	/* Unknown Stopped Started Slave Master */
	/* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
	/* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
	/* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
	/* Slave */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, },
	/* Master */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp , },
	};

	#define clear_node_weights_flags(nw_flags, nw_rsc, flags_to_clear) do { \
	flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Node weight", (nw_rsc)->id, (flags), \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	static gboolean
	native_choose_node(pe_resource_t * rsc, pe_node_t * prefer, pe_working_set_t * data_set)
	{
	GListPtr nodes = NULL;
	pe_node_t *chosen = NULL;
	pe_node_t *best = NULL;
	int multiple = 1;
	int length = 0;
	gboolean result = FALSE;

	process_utilization(rsc, &prefer, data_set);

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to ? TRUE : FALSE;
	}

	// Sort allowed nodes by weight
	if (rsc->allowed_nodes) {
	length = g_hash_table_size(rsc->allowed_nodes);
	}
	if (length > 0) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	nodes = sort_nodes_by_weight(nodes, pe__current_node(rsc), data_set);

	// First node in sorted list has the best score
	best = g_list_nth_data(nodes, 0);
	}

	if (prefer && nodes) {
	chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);

	if (chosen == NULL) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
	prefer->details->uname, rsc->id);

	/* Favor the preferred node as long as its weight is at least as good as
	* the best allowed node's.
	*
	* An alternative would be to favor the preferred node even if the best
	* node is better, when the best node's weight is less than INFINITY.
	*/
	} else if ((chosen->weight < 0) \|\| (chosen->weight < best->weight)) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
	chosen->details->uname, rsc->id);
	chosen = NULL;

	} else if (!can_run_resources(chosen)) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
	chosen->details->uname, rsc->id);
	chosen = NULL;

	} else {
	pe_rsc_trace(rsc,
	"Chose preferred node %s for %s (ignoring %d candidates)",
	chosen->details->uname, rsc->id, length);
	}
	}

	if ((chosen == NULL) && nodes) {
	/* Either there is no preferred node, or the preferred node is not
	* available, but there are other nodes allowed to run the resource.
	*/

	chosen = best;
	pe_rsc_trace(rsc, "Chose node %s for %s from %d candidates",
	chosen ? chosen->details->uname : "<none>", rsc->id, length);

	if (!pe_rsc_is_unique_clone(rsc->parent)
	&& chosen && (chosen->weight > 0) && can_run_resources(chosen)) {
	/* If the resource is already running on a node, prefer that node if
	* it is just as good as the chosen node.
	*
	* We don't do this for unique clone instances, because
	* distribute_children() has already assigned instances to their
	* running nodes when appropriate, and if we get here, we don't want
	* remaining unallocated instances to prefer a node that's already
	* running another instance.
	*/
	pe_node_t *running = pe__current_node(rsc);

	if (running && (can_run_resources(running) == FALSE)) {
	pe_rsc_trace(rsc, "Current node for %s (%s) can't run resources",
	rsc->id, running->details->uname);
	} else if (running) {
	for (GList *iter = nodes->next; iter; iter = iter->next) {
	pe_node_t tmp = (pe_node_t ) iter->data;

	if (tmp->weight != chosen->weight) {
	// The nodes are sorted by weight, so no more are equal
	break;
	}
	if (tmp->details == running->details) {
	// Scores are equal, so prefer the current node
	chosen = tmp;
	}
	multiple++;
	}
	}
	}
	}

	if (multiple > 1) {
	static char score[33];
	int log_level = (chosen->weight >= INFINITY)? LOG_WARNING : LOG_INFO;

	score2char_stack(chosen->weight, score, sizeof(score));
	do_crm_log(log_level,
	"Chose node %s for %s from %d nodes with score %s",
	chosen->details->uname, rsc->id, multiple, score);
	}

	result = native_assign_node(rsc, nodes, chosen, FALSE);
	g_list_free(nodes);
	return result;
	}

	/*!
	* \internal
	* \brief Find score of highest-scored node that matches colocation attribute
	*
	* \param[in] rsc Resource whose allowed nodes should be searched
	* \param[in] attr Colocation attribute name (must not be NULL)
	* \param[in] value Colocation attribute value to require
	*/
	static int
	best_node_score_matching_attr(const pe_resource_t rsc, const char attr,
	const char *value)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	int best_score = -INFINITY;
	const char *best_node = NULL;

	// Find best allowed node with matching attribute
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {

	if ((node->weight > best_score) && can_run_resources(node)
	&& pcmk__str_eq(value, pe_node_attribute_raw(node, attr), pcmk__str_casei)) {

	best_score = node->weight;
	best_node = node->details->uname;
	}
	}

	if (!pcmk__str_eq(attr, CRM_ATTR_UNAME, pcmk__str_casei)) {
	if (best_node == NULL) {
	crm_info("No allowed node for %s matches node attribute %s=%s",
	rsc->id, attr, value);
	} else {
	crm_info("Allowed node %s for %s had best score (%d) "
	"of those matching node attribute %s=%s",
	best_node, rsc->id, best_score, attr, value);
	}
	}
	return best_score;
	}

	/*!
	* \internal
	* \brief Add resource's colocation matches to current node allocation scores
	*
	* For each node in a given table, if any of a given resource's allowed nodes
	* have a matching value for the colocation attribute, add the highest of those
	* nodes' scores to the node's score.
	*
	* \param[in,out] nodes Hash table of nodes with allocation scores so far
	* \param[in] rsc Resource whose allowed nodes should be compared
	* \param[in] attr Colocation attribute that must match (NULL for default)
	* \param[in] factor Factor by which to multiply scores being added
	* \param[in] only_positive Whether to add only positive scores
	*/
	static void
	add_node_scores_matching_attr(GHashTable nodes, const pe_resource_t rsc,
	const char *attr, float factor,
	bool only_positive)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (attr == NULL) {
	attr = CRM_ATTR_UNAME;
	}

	// Iterate through each node
	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	float weight_f = 0;
	int weight = 0;
	int score = 0;
	int new_score = 0;

	score = best_node_score_matching_attr(rsc, attr,
	pe_node_attribute_raw(node, attr));

	if ((factor < 0) && (score < 0)) {
	/* Negative preference for a node with a negative score
	* should not become a positive preference.
	*
	* @TODO Consider filtering only if weight is -INFINITY
	*/
	crm_trace("%s: Filtering %d + %f * %d (double negative disallowed)",
	node->details->uname, node->weight, factor, score);
	continue;
	}

	if (node->weight == INFINITY_HACK) {
	crm_trace("%s: Filtering %d + %f * %d (node was marked unusable)",
	node->details->uname, node->weight, factor, score);
	continue;
	}

	weight_f = factor * score;

	// Round the number; see http://c-faq.com/fp/round.html
	weight = (int) ((weight_f < 0)? (weight_f - 0.5) : (weight_f + 0.5));

	/* Small factors can obliterate the small scores that are often actually
	* used in configurations. If the score and factor are nonzero, ensure
	* that the result is nonzero as well.
	*/
	if ((weight == 0) && (score != 0)) {
	if (factor > 0.0) {
	weight = 1;
	} else if (factor < 0.0) {
	weight = -1;
	}
	}

	new_score = pe__add_scores(weight, node->weight);

	if (only_positive && (new_score < 0) && (node->weight > 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d "
	"(negative disallowed, marking node unusable)",
	node->details->uname, node->weight, factor, score,
	new_score);
	node->weight = INFINITY_HACK;
	continue;
	}

	if (only_positive && (new_score < 0) && (node->weight == 0)) {
	crm_trace("%s: Filtering %d + %f * %d = %d (negative disallowed)",
	node->details->uname, node->weight, factor, score,
	new_score);
	continue;
	}

	crm_trace("%s: %d + %f * %d = %d", node->details->uname,
	node->weight, factor, score, new_score);
	node->weight = new_score;
	}
	}

	static inline bool
	is_nonempty_group(pe_resource_t *rsc)
	{
	return rsc && (rsc->variant == pe_group) && (rsc->children != NULL);
	}

	/*!
	* \internal
	* \brief Incorporate colocation constraint scores into node weights
	*
	* \param[in,out] rsc Resource being placed
	* \param[in] rhs ID of 'with' resource
	* \param[in,out] nodes Nodes, with scores as of this point
	* \param[in] attr Colocation attribute (ID by default)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pe_weights values
	*
	* \return Nodes, with scores modified by this constraint
	* \note This function assumes ownership of the nodes argument. The caller
	* should free the returned copy rather than the original.
	*/
	GHashTable *
	pcmk__native_merge_weights(pe_resource_t rsc, const char rhs,
	GHashTable nodes, const char attr, float factor,
	uint32_t flags)
	{
	GHashTable *work = NULL;

	// Avoid infinite recursion
	if (pcmk_is_set(rsc->flags, pe_rsc_merging)) {
	pe_rsc_info(rsc, "%s: Breaking dependency loop at %s", rhs, rsc->id);
	return nodes;
	}
	pe__set_resource_flags(rsc, pe_rsc_merging);

	if (pcmk_is_set(flags, pe_weights_init)) {
	if (is_nonempty_group(rsc)) {
	GList *last = g_list_last(rsc->children);
	pe_resource_t *last_rsc = last->data;

	pe_rsc_trace(rsc, "%s: Merging scores from group %s "
	"using last member %s (at %.6f)",
	rhs, rsc->id, last_rsc->id, factor);
	work = pcmk__native_merge_weights(last_rsc, rhs, NULL, attr, factor,
	flags);
	} else {
	work = pcmk__copy_node_table(rsc->allowed_nodes);
	}
	clear_node_weights_flags(flags, rsc, pe_weights_init);

	} else if (is_nonempty_group(rsc)) {
	/* The first member of the group will recursively incorporate any
	* constraints involving other members (including the group internal
	* colocation).
	*
	* @TODO The indirect colocations from the dependent group's other
	* members will be incorporated at full strength rather than by
	* factor, so the group's combined stickiness will be treated as
	* (factor + (#members - 1)) * stickiness. It is questionable what
	* the right approach should be.
	*/
	pe_rsc_trace(rsc, "%s: Merging scores from first member of group %s "
	"(at %.6f)", rhs, rsc->id, factor);
	work = pcmk__copy_node_table(nodes);
	work = pcmk__native_merge_weights(rsc->children->data, rhs, work, attr,
	factor, flags);

	} else {
	pe_rsc_trace(rsc, "%s: Merging scores from %s (at %.6f)",
	rhs, rsc->id, factor);
	work = pcmk__copy_node_table(nodes);
	add_node_scores_matching_attr(work, rsc, attr, factor,
	pcmk_is_set(flags, pe_weights_positive));
	}

	if (can_run_any(work)) {
	GListPtr gIter = NULL;
	int multiplier = (factor < 0)? -1 : 1;

	if (pcmk_is_set(flags, pe_weights_forward)) {
	gIter = rsc->rsc_cons;
	pe_rsc_trace(rsc,
	"Checking additional %d optional '%s with' constraints",
	g_list_length(gIter), rsc->id);

	} else if (is_nonempty_group(rsc)) {
	pe_resource_t *last_rsc = g_list_last(rsc->children)->data;

	gIter = last_rsc->rsc_cons_lhs;
	pe_rsc_trace(rsc, "Checking additional %d optional 'with group %s' "
	"constraints using last member %s",
	g_list_length(gIter), rsc->id, last_rsc->id);

	} else {
	gIter = rsc->rsc_cons_lhs;
	pe_rsc_trace(rsc,
	"Checking additional %d optional 'with %s' constraints",
	g_list_length(gIter), rsc->id);
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *other = NULL;
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	if (constraint->score == 0) {
	continue;
	}

	if (pcmk_is_set(flags, pe_weights_forward)) {
	other = constraint->rsc_rh;
	} else {
	other = constraint->rsc_lh;
	}

	pe_rsc_trace(rsc, "Optionally merging score of '%s' constraint (%s with %s)",
	constraint->id, constraint->rsc_lh->id,
	constraint->rsc_rh->id);
	work = pcmk__native_merge_weights(other, rhs, work,
	constraint->node_attribute,
	multiplier * constraint->score / (float) INFINITY,
	flags\|pe_weights_rollback);
	pe__show_node_weights(true, NULL, rhs, work);
	}

	} else if (pcmk_is_set(flags, pe_weights_rollback)) {
	pe_rsc_info(rsc, "%s: Rolling back optional scores from %s",
	rhs, rsc->id);
	g_hash_table_destroy(work);
	pe__clear_resource_flags(rsc, pe_rsc_merging);
	return nodes;
	}


	if (pcmk_is_set(flags, pe_weights_positive)) {
	pe_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->weight == INFINITY_HACK) {
	node->weight = 1;
	}
	}
	}

	if (nodes) {
	g_hash_table_destroy(nodes);
	}

	pe__clear_resource_flags(rsc, pe_rsc_merging);
	return work;
	}

	static inline bool
	node_has_been_unfenced(pe_node_t *node)
	{
	const char *unfenced = pe_node_attribute_raw(node, CRM_ATTR_UNFENCED);

	return !pcmk__str_eq(unfenced, "0", pcmk__str_null_matches);
	}

	static inline bool
	is_unfence_device(pe_resource_t rsc, pe_working_set_t data_set)
	{
	return pcmk_is_set(rsc->flags, pe_rsc_fence_device)
	&& pcmk_is_set(data_set->flags, pe_flag_enable_unfencing);
	}

	pe_node_t *
	pcmk__native_allocate(pe_resource_t rsc, pe_node_t prefer,
	pe_working_set_t *data_set)
	{
	GListPtr gIter = NULL;

	if (rsc->parent && !pcmk_is_set(rsc->parent->flags, pe_rsc_allocating)) {
	/* never allocate children on their own */
	pe_rsc_debug(rsc, "Escalating allocation of %s to its parent: %s", rsc->id,
	rsc->parent->id);
	rsc->parent->cmds->allocate(rsc->parent, prefer, data_set);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	pe__set_resource_flags(rsc, pe_rsc_allocating);
	pe__show_node_weights(true, rsc, "Pre-alloc", rsc->allowed_nodes);

	for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	GHashTable *archive = NULL;
	pe_resource_t *rsc_rh = constraint->rsc_rh;

	if (constraint->score == 0) {
	continue;
	}

	if (constraint->role_lh >= RSC_ROLE_MASTER
	\|\| (constraint->score < 0 && constraint->score > -INFINITY)) {
	archive = pcmk__copy_node_table(rsc->allowed_nodes);
	}

	pe_rsc_trace(rsc,
	"%s: Allocating %s first (constraint=%s score=%d role=%s)",
	rsc->id, rsc_rh->id, constraint->id,
	constraint->score, role2text(constraint->role_lh));
	rsc_rh->cmds->allocate(rsc_rh, NULL, data_set);
	rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint, data_set);
	if (archive && can_run_any(rsc->allowed_nodes) == FALSE) {
	pe_rsc_info(rsc, "%s: Rolling back scores from %s", rsc->id, rsc_rh->id);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive) {
	g_hash_table_destroy(archive);
	}
	}

	pe__show_node_weights(true, rsc, "Post-coloc", rsc->allowed_nodes);

	for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	if (constraint->score == 0) {
	continue;
	}
	pe_rsc_trace(rsc, "Merging score of '%s' constraint (%s with %s)",
	constraint->id, constraint->rsc_lh->id,
	constraint->rsc_rh->id);
	rsc->allowed_nodes =
	constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute,
	(float)constraint->score / INFINITY,
	pe_weights_rollback);
	}

	if (rsc->next_role == RSC_ROLE_STOPPED) {
	pe_rsc_trace(rsc, "Making sure %s doesn't get allocated", rsc->id);
	/* make sure it doesn't come up again */
	resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);

	} else if(rsc->next_role > rsc->role
	&& !pcmk_is_set(data_set->flags, pe_flag_have_quorum)
	&& data_set->no_quorum_policy == no_quorum_freeze) {
	crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role));
	rsc->next_role = rsc->role;
	}

	pe__show_node_weights(!show_scores, rsc, __func__, rsc->allowed_nodes);
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)
	&& !pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	const char *reason = NULL;
	pe_node_t *assign_to = NULL;

	rsc->next_role = rsc->role;
	assign_to = pe__current_node(rsc);
	if (assign_to == NULL) {
	reason = "inactive";
	} else if (rsc->role == RSC_ROLE_MASTER) {
	reason = "master";
	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	reason = "failed";
	} else {
	reason = "active";
	}
	pe_rsc_info(rsc, "Unmanaged resource %s allocated to %s: %s", rsc->id,
	(assign_to? assign_to->details->uname : "no node"), reason);
	native_assign_node(rsc, NULL, assign_to, TRUE);

	} else if (pcmk_is_set(data_set->flags, pe_flag_stop_everything)) {
	pe_rsc_debug(rsc, "Forcing %s to stop", rsc->id);
	native_assign_node(rsc, NULL, NULL, TRUE);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_provisional)
	&& native_choose_node(rsc, prefer, data_set)) {
	pe_rsc_trace(rsc, "Allocated resource %s to %s", rsc->id,
	rsc->allocated_to->details->uname);

	} else if (rsc->allocated_to == NULL) {
	if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
	} else if (rsc->running_on != NULL) {
	pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
	}

	} else {
	pe_rsc_debug(rsc, "Pre-Allocated resource %s to %s", rsc->id,
	rsc->allocated_to->details->uname);
	}

	pe__clear_resource_flags(rsc, pe_rsc_allocating);

	if (rsc->is_remote_node) {
	pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);

	CRM_ASSERT(remote_node != NULL);
	if (rsc->allocated_to && rsc->next_role != RSC_ROLE_STOPPED) {
	crm_trace("Setting Pacemaker Remote node %s to ONLINE",
	remote_node->details->id);
	remote_node->details->online = TRUE;
	/* We shouldn't consider an unseen remote-node unclean if we are going
	* to try and connect to it. Otherwise we get an unnecessary fence */
	if (remote_node->details->unseen == TRUE) {
	remote_node->details->unclean = FALSE;
	}

	} else {
	crm_trace("Setting Pacemaker Remote node %s to SHUTDOWN (next role %s, %sallocated)",
	remote_node->details->id, role2text(rsc->next_role),
	(rsc->allocated_to? "" : "un"));
	remote_node->details->shutdown = TRUE;
	}
	}

	return rsc->allocated_to;
	}

	static gboolean
	is_op_dup(pe_resource_t rsc, const char name, guint interval_ms)
	{
	gboolean dup = FALSE;
	const char *id = NULL;
	const char *value = NULL;
	xmlNode *operation = NULL;
	guint interval2_ms = 0;

	CRM_ASSERT(rsc);
	for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
	operation = __xml_next_element(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	value = crm_element_value(operation, "name");
	if (!pcmk__str_eq(value, name, pcmk__str_casei)) {
	continue;
	}

	value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval2_ms = crm_parse_interval_spec(value);
	if (interval_ms != interval2_ms) {
	continue;
	}

	if (id == NULL) {
	id = ID(operation);

	} else {
	pcmk__config_err("Operation %s is duplicate of %s (do not use "
	"same name and interval combination more "
	"than once per resource)", ID(operation), id);
	dup = TRUE;
	}
	}
	}

	return dup;
	}

	static bool
	op_cannot_recur(const char *name)
	{
	return pcmk__strcase_any_of(name, RSC_STOP, RSC_START, RSC_DEMOTE, RSC_PROMOTE, NULL);
	}

	static void
	RecurringOp(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *node_uname = node? node->details->uname : "n/a";

	guint interval_ms = 0;
	pe_action_t *mon = NULL;
	gboolean is_optional = TRUE;
	GListPtr possible_matches = NULL;

	CRM_ASSERT(rsc);

	/* Only process for the operations without role="Stopped" */
	role = crm_element_value(operation, "role");
	if (role && text2role(role) == RSC_ROLE_STOPPED) {
	return;
	}

	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_parse_interval_spec(interval_spec);
	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval_ms)) {
	crm_trace("Not creating duplicate recurring action %s for %dms %s",
	ID(operation), interval_ms, name);
	return;
	}

	if (op_cannot_recur(name)) {
	pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
	ID(operation), name);
	return;
	}

	key = pcmk__op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	crm_trace("Not creating recurring action %s for disabled resource %s",
	ID(operation), rsc->id);
	free(key);
	return;
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s in role %s on %s",
	ID(operation), rsc->id, role2text(rsc->next_role), node_uname);

	if (start != NULL) {
	pe_rsc_trace(rsc, "Marking %s %s due to %s", key,
	pcmk_is_set(start->flags, pe_action_optional)? "optional" : "mandatory",
	start->uuid);
	is_optional = (rsc->cmds->action_flags(start, NULL) & pe_action_optional);
	} else {
	pe_rsc_trace(rsc, "Marking %s optional", key);
	is_optional = TRUE;
	}

	/* start a monitor for an already active resource */
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches == NULL) {
	is_optional = FALSE;
	pe_rsc_trace(rsc, "Marking %s mandatory: not active", key);

	} else {
	GListPtr gIter = NULL;

	for (gIter = possible_matches; gIter != NULL; gIter = gIter->next) {
	pe_action_t op = (pe_action_t ) gIter->data;

	if (pcmk_is_set(op->flags, pe_action_reschedule)) {
	is_optional = FALSE;
	break;
	}
	}
	g_list_free(possible_matches);
	}

	if ((rsc->next_role == RSC_ROLE_MASTER && role == NULL)
	\|\| (role != NULL && text2role(role) != rsc->next_role)) {
	int log_level = LOG_TRACE;
	const char *result = "Ignoring";

	if (is_optional) {
	char *after_key = NULL;
	pe_action_t *cancel_op = NULL;

	// It's running, so cancel it
	log_level = LOG_INFO;
	result = "Cancelling";
	cancel_op = pe_cancel_op(rsc, name, interval_ms, node, data_set);

	switch (rsc->role) {
	case RSC_ROLE_SLAVE:
	case RSC_ROLE_STARTED:
	if (rsc->next_role == RSC_ROLE_MASTER) {
	after_key = promote_key(rsc);

	} else if (rsc->next_role == RSC_ROLE_STOPPED) {
	after_key = stop_key(rsc);
	}

	break;
	case RSC_ROLE_MASTER:
	after_key = demote_key(rsc);
	break;
	default:
	break;
	}

	if (after_key) {
	custom_action_order(rsc, NULL, cancel_op, rsc, after_key, NULL,
	pe_order_runnable_left, data_set);
	}
	}

	do_crm_log(log_level, "%s action %s (%s vs. %s)",
	result, key, role ? role : role2text(RSC_ROLE_SLAVE),
	role2text(rsc->next_role));

	free(key);
	return;
	}

	mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set);
	key = mon->uuid;
	if (is_optional) {
	pe_rsc_trace(rsc, "%s\t %s (optional)", node_uname, mon->uuid);
	}

	if ((start == NULL) \|\| !pcmk_is_set(start->flags, pe_action_runnable)) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : start un-runnable)",
	node_uname, mon->uuid);
	update_action_flags(mon, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);

	} else if (node == NULL \|\| node->details->online == FALSE \|\| node->details->unclean) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)",
	node_uname, mon->uuid);
	update_action_flags(mon, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);

	} else if (!pcmk_is_set(mon->flags, pe_action_optional)) {
	pe_rsc_info(rsc, " Start recurring %s (%us) for %s on %s",
	mon->task, interval_ms / 1000, rsc->id, node_uname);
	}

	if (rsc->next_role == RSC_ROLE_MASTER) {
	char *running_master = crm_itoa(PCMK_OCF_RUNNING_MASTER);

	add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_master);
	free(running_master);
	}

	if ((node == NULL) \|\| pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(rsc, start_key(rsc), NULL,
	NULL, strdup(key), mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);

	custom_action_order(rsc, reload_key(rsc), NULL,
	NULL, strdup(key), mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);

	if (rsc->next_role == RSC_ROLE_MASTER) {
	custom_action_order(rsc, promote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional \| pe_order_runnable_left, data_set);

	} else if (rsc->role == RSC_ROLE_MASTER) {
	custom_action_order(rsc, demote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional \| pe_order_runnable_left, data_set);
	}
	}
	}

	static void
	Recurring(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_maintenance) &&
	(node == NULL \|\| node->details->maintenance == FALSE)) {
	xmlNode *operation = NULL;

	for (operation = __xml_first_child_element(rsc->ops_xml);
	operation != NULL;
	operation = __xml_next_element(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	RecurringOp(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	static void
	RecurringOp_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *node_uname = node? node->details->uname : "n/a";

	guint interval_ms = 0;
	GListPtr possible_matches = NULL;
	GListPtr gIter = NULL;

	/* Only process for the operations with role="Stopped" */
	role = crm_element_value(operation, "role");
	if (role == NULL \|\| text2role(role) != RSC_ROLE_STOPPED) {
	return;
	}

	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_parse_interval_spec(interval_spec);
	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval_ms)) {
	crm_trace("Not creating duplicate recurring action %s for %dms %s",
	ID(operation), interval_ms, name);
	return;
	}

	if (op_cannot_recur(name)) {
	pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
	ID(operation), name);
	return;
	}

	key = pcmk__op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	crm_trace("Not creating recurring action %s for disabled resource %s",
	ID(operation), rsc->id);
	free(key);
	return;
	}

	// @TODO add support
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	crm_notice("Ignoring %s (recurring monitors for Stopped role are "
	"not supported for anonymous clones)",
	ID(operation));
	return;
	}

	pe_rsc_trace(rsc,
	"Creating recurring action %s for %s in role %s on nodes where it should not be running",
	ID(operation), rsc->id, role2text(rsc->next_role));

	/* if the monitor exists on the node where the resource will be running, cancel it */
	if (node != NULL) {
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches) {
	pe_action_t *cancel_op = NULL;

	g_list_free(possible_matches);

	cancel_op = pe_cancel_op(rsc, name, interval_ms, node, data_set);

	if (rsc->next_role == RSC_ROLE_STARTED \|\| rsc->next_role == RSC_ROLE_SLAVE) {
	/* rsc->role == RSC_ROLE_STOPPED: cancel the monitor before start */
	/* rsc->role == RSC_ROLE_STARTED: for a migration, cancel the monitor on the target node before start */
	custom_action_order(rsc, NULL, cancel_op, rsc, start_key(rsc), NULL,
	pe_order_runnable_left, data_set);
	}

	pe_rsc_info(rsc, "Cancel action %s (%s vs. %s) on %s",
	key, role, role2text(rsc->next_role), node_uname);
	}
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t stop_node = (pe_node_t ) gIter->data;
	const char *stop_node_uname = stop_node->details->uname;
	gboolean is_optional = TRUE;
	gboolean probe_is_optional = TRUE;
	gboolean stop_is_optional = TRUE;
	pe_action_t *stopped_mon = NULL;
	char *rc_inactive = NULL;
	GListPtr probe_complete_ops = NULL;
	GListPtr stop_ops = NULL;
	GListPtr local_gIter = NULL;

	if (node && pcmk__str_eq(stop_node_uname, node_uname, pcmk__str_casei)) {
	continue;
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s on %s",
	ID(operation), rsc->id, crm_str(stop_node_uname));

	/* start a monitor for an already stopped resource */
	possible_matches = find_actions_exact(rsc->actions, key, stop_node);
	if (possible_matches == NULL) {
	pe_rsc_trace(rsc, "Marking %s mandatory on %s: not active", key,
	crm_str(stop_node_uname));
	is_optional = FALSE;
	} else {
	pe_rsc_trace(rsc, "Marking %s optional on %s: already active", key,
	crm_str(stop_node_uname));
	is_optional = TRUE;
	g_list_free(possible_matches);
	}

	stopped_mon = custom_action(rsc, strdup(key), name, stop_node, is_optional, TRUE, data_set);

	rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING);
	add_hash_param(stopped_mon->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
	free(rc_inactive);

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	GList *probes = pe__resource_actions(rsc, stop_node, RSC_STATUS,
	FALSE);
	GListPtr pIter = NULL;

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	pe_action_t probe = (pe_action_t ) pIter->data;

	order_actions(probe, stopped_mon, pe_order_runnable_left);
	crm_trace("%s then %s on %s", probe->uuid, stopped_mon->uuid, stop_node->details->uname);
	}

	g_list_free(probes);
	}

	if (probe_complete_ops) {
	g_list_free(probe_complete_ops);
	}

	stop_ops = pe__resource_actions(rsc, stop_node, RSC_STOP, TRUE);

	for (local_gIter = stop_ops; local_gIter != NULL; local_gIter = local_gIter->next) {
	pe_action_t stop = (pe_action_t ) local_gIter->data;

	if (!pcmk_is_set(stop->flags, pe_action_optional)) {
	stop_is_optional = FALSE;
	}

	if (!pcmk_is_set(stop->flags, pe_action_runnable)) {
	crm_debug("%s\t %s (cancelled : stop un-runnable)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	update_action_flags(stopped_mon, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(rsc, stop_key(rsc), stop,
	NULL, strdup(key), stopped_mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);
	}

	}

	if (stop_ops) {
	g_list_free(stop_ops);
	}

	if (is_optional == FALSE && probe_is_optional && stop_is_optional
	&& !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Marking %s optional on %s due to unmanaged",
	key, crm_str(stop_node_uname));
	update_action_flags(stopped_mon, pe_action_optional, __func__,
	__LINE__);
	}

	if (pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
	pe_rsc_trace(rsc, "%s\t %s (optional)", crm_str(stop_node_uname), stopped_mon->uuid);
	}

	if (stop_node->details->online == FALSE \|\| stop_node->details->unclean) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	update_action_flags(stopped_mon, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	}

	if (pcmk_is_set(stopped_mon->flags, pe_action_runnable)
	&& !pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
	crm_notice(" Start recurring %s (%us) for %s on %s", stopped_mon->task,
	interval_ms / 1000, rsc->id, crm_str(stop_node_uname));
	}
	}

	free(key);
	}

	static void
	Recurring_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_maintenance) &&
	(node == NULL \|\| node->details->maintenance == FALSE)) {
	xmlNode *operation = NULL;

	for (operation = __xml_first_child_element(rsc->ops_xml);
	operation != NULL;
	operation = __xml_next_element(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	RecurringOp_Stopped(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	static void
	handle_migration_actions(pe_resource_t * rsc, pe_node_t current, pe_node_t chosen, pe_working_set_t * data_set)
	{
	pe_action_t *migrate_to = NULL;
	pe_action_t *migrate_from = NULL;
	pe_action_t *start = NULL;
	pe_action_t *stop = NULL;
	gboolean partial = rsc->partial_migration_target ? TRUE : FALSE;

	pe_rsc_trace(rsc, "Processing migration actions %s moving from %s to %s . partial migration = %s",
	rsc->id, current->details->id, chosen->details->id, partial ? "TRUE" : "FALSE");
	start = start_action(rsc, chosen, TRUE);
	stop = stop_action(rsc, current, TRUE);

	if (partial == FALSE) {
	migrate_to = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	RSC_MIGRATE, current, TRUE, TRUE, data_set);
	}

	migrate_from = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
	RSC_MIGRATED, chosen, TRUE, TRUE, data_set);

	if ((migrate_to && migrate_from) \|\| (migrate_from && partial)) {

	pe__set_action_flags(start, pe_action_migrate_runnable);
	pe__set_action_flags(stop, pe_action_migrate_runnable);

	// This is easier than trying to delete it from the graph
	update_action_flags(start, pe_action_pseudo, __func__, __LINE__);

	/* order probes before migrations */
	if (partial) {
	pe__set_action_flags(migrate_from, pe_action_migrate_runnable);
	migrate_from->needs = start->needs;

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
	NULL, pe_order_optional, data_set);

	} else {
	pe__set_action_flags(migrate_from, pe_action_migrate_runnable);
	pe__set_action_flags(migrate_to, pe_action_migrate_runnable);
	migrate_to->needs = start->needs;

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	NULL, pe_order_optional, data_set);
	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	pe_order_optional\|pe_order_implies_first_migratable,
	data_set);
	}

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_optional\|pe_order_implies_first_migratable,
	data_set);
	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	pe_order_optional\|pe_order_implies_first_migratable\|pe_order_pseudo_left,
	data_set);

	}

	if (migrate_to) {
	add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
	add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);

	/* Pacemaker Remote connections don't require pending to be recorded in
	* the CIB. We can reduce CIB writes by not setting PENDING for them.
	*/
	if (rsc->is_remote_node == FALSE) {
	/* migrate_to takes place on the source node, but can
	* have an effect on the target node depending on how
	* the agent is written. Because of this, we have to maintain
	* a record that the migrate_to occurred, in case the source node
	* loses membership while the migrate_to action is still in-flight.
	*/
	add_hash_param(migrate_to->meta, XML_OP_ATTR_PENDING, "true");
	}
	}

	if (migrate_from) {
	add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
	add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
	}
	}

	void
	native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_action_t *start = NULL;
	pe_node_t *chosen = NULL;
	pe_node_t *current = NULL;
	gboolean need_stop = FALSE;
	bool need_promote = FALSE;
	gboolean is_moving = FALSE;
	gboolean allow_migrate = pcmk_is_set(rsc->flags, pe_rsc_allow_migrate)? TRUE : FALSE;

	GListPtr gIter = NULL;
	unsigned int num_all_active = 0;
	unsigned int num_clean_active = 0;
	bool multiply_active = FALSE;
	enum rsc_role_e role = RSC_ROLE_UNKNOWN;
	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;

	CRM_ASSERT(rsc);
	chosen = rsc->allocated_to;
	- if (chosen != NULL && rsc->next_role == RSC_ROLE_UNKNOWN) {
	- rsc->next_role = RSC_ROLE_STARTED;
	- pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role));
	-
	- } else if (rsc->next_role == RSC_ROLE_UNKNOWN) {
	- rsc->next_role = RSC_ROLE_STOPPED;
	- pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role));
	+ next_role = rsc->next_role;
	+ if (next_role == RSC_ROLE_UNKNOWN) {
	+ rsc->next_role = (chosen == NULL)? RSC_ROLE_STOPPED : RSC_ROLE_STARTED;
	}
	-
	- pe_rsc_trace(rsc, "Processing state transition for %s %p: %s->%s", rsc->id, rsc,
	- role2text(rsc->role), role2text(rsc->next_role));
	+ pe_rsc_trace(rsc, "Creating all actions for %s transition from %s to %s (%s) on %s",
	+ rsc->id, role2text(rsc->role), role2text(rsc->next_role),
	+ ((next_role == RSC_ROLE_UNKNOWN)? "implicit" : "explicit"),
	+ ((chosen == NULL)? "no node" : chosen->details->uname));

	current = pe__find_active_on(rsc, &num_all_active, &num_clean_active);

	for (gIter = rsc->dangling_migrations; gIter != NULL; gIter = gIter->next) {
	pe_node_t dangling_source = (pe_node_t ) gIter->data;

	- pe_action_t *stop = stop_action(rsc, dangling_source, FALSE);
	+ pe_action_t *stop = NULL;

	- pe__set_action_flags(stop, pe_action_dangle);
	- pe_rsc_trace(rsc, "Forcing a cleanup of %s on %s",
	+ pe_rsc_trace(rsc, "Creating stop action %s cleanup for %s on %s due to dangling migration",
	+ (pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)? "and" : "without"),
	rsc->id, dangling_source->details->uname);
	-
	+ stop = stop_action(rsc, dangling_source, FALSE);
	+ pe__set_action_flags(stop, pe_action_dangle);
	if (pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, dangling_source, FALSE, data_set);
	}
	}

	if ((num_all_active == 2) && (num_clean_active == 2) && chosen
	&& rsc->partial_migration_source && rsc->partial_migration_target
	&& (current->details == rsc->partial_migration_source->details)
	&& (chosen->details == rsc->partial_migration_target->details)) {

	/* The chosen node is still the migration target from a partial
	* migration. Attempt to continue the migration instead of recovering
	* by stopping the resource everywhere and starting it on a single node.
	*/
	- pe_rsc_trace(rsc,
	- "Will attempt to continue with a partial migration to target %s from %s",
	+ pe_rsc_trace(rsc, "Will attempt to continue with partial migration "
	+ "to target %s from %s",
	rsc->partial_migration_target->details->id,
	rsc->partial_migration_source->details->id);

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	/* If a resource has "requires" set to nothing or quorum, don't consider
	* it active on unclean nodes (similar to how all resources behave when
	* stonith-enabled is false). We can start such resources elsewhere
	* before fencing completes, and if we considered the resource active on
	* the failed node, we would attempt recovery for being active on
	* multiple nodes.
	*/
	multiply_active = (num_clean_active > 1);
	} else {
	multiply_active = (num_all_active > 1);
	}

	if (multiply_active) {
	if (rsc->partial_migration_target && rsc->partial_migration_source) {
	// Migration was in progress, but we've chosen a different target
	- crm_notice("Resource %s can no longer migrate to %s. Stopping on %s too",
	- rsc->id, rsc->partial_migration_target->details->uname,
	- rsc->partial_migration_source->details->uname);
	+ crm_notice("Resource %s can no longer migrate from %s to %s "
	+ "(will stop on both nodes)",
	+ rsc->id, rsc->partial_migration_source->details->uname,
	+ rsc->partial_migration_target->details->uname);

	} else {
	// Resource was incorrectly multiply active
	pe_proc_err("Resource %s is active on %u nodes (%s)",
	rsc->id, num_all_active,
	recovery2text(rsc->recovery_type));
	crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ#Resource_is_Too_Active for more information");
	}

	if (rsc->recovery_type == recovery_stop_start) {
	need_stop = TRUE;
	}

	/* If by chance a partial migration is in process, but the migration
	* target is not chosen still, clear all partial migration data.
	*/
	rsc->partial_migration_source = rsc->partial_migration_target = NULL;
	allow_migrate = FALSE;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
	+ pe_rsc_trace(rsc, "Creating start action for %s to represent already pending start",
	+ rsc->id);
	start = start_action(rsc, chosen, TRUE);
	pe__set_action_flags(start, pe_action_print_always);
	}

	if (current && chosen && current->details != chosen->details) {
	- pe_rsc_trace(rsc, "Moving %s", rsc->id);
	+ pe_rsc_trace(rsc, "Moving %s from %s to %s",
	+ rsc->id, crm_str(current->details->uname),
	+ crm_str(chosen->details->uname));
	is_moving = TRUE;
	need_stop = TRUE;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	if (pcmk_is_set(rsc->flags, pe_rsc_stop)) {
	need_stop = TRUE;
	pe_rsc_trace(rsc, "Recovering %s", rsc->id);
	} else {
	pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
	if (rsc->next_role == RSC_ROLE_MASTER) {
	need_promote = TRUE;
	}
	}

	} else if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	- pe_rsc_trace(rsc, "Block %s", rsc->id);
	+ pe_rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
	need_stop = TRUE;

	} else if (rsc->role > RSC_ROLE_STARTED && current != NULL && chosen != NULL) {
	- /* Recovery of a promoted resource */
	+ pe_rsc_trace(rsc, "Creating start action for promoted resource %s",
	+ rsc->id);
	start = start_action(rsc, chosen, TRUE);
	if (!pcmk_is_set(start->flags, pe_action_optional)) {
	- pe_rsc_trace(rsc, "Forced start %s", rsc->id);
	+ // Recovery of a promoted resource
	+ pe_rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
	need_stop = TRUE;
	}
	}

	- pe_rsc_trace(rsc, "Creating actions for %s: %s->%s", rsc->id,
	- role2text(rsc->role), role2text(rsc->next_role));
	-
	/* Create any additional actions required when bringing resource down and
	* back up to same level.
	*/
	role = rsc->role;
	while (role != RSC_ROLE_STOPPED) {
	next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
	- pe_rsc_trace(rsc, "Down: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role),
	- rsc->id, need_stop ? " required" : "");
	+ pe_rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
	+ (need_stop? "required" : "optional"), rsc->id,
	+ role2text(role), role2text(next_role));
	if (rsc_action_matrix[role][next_role] (rsc, current, !need_stop, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}


	while ((rsc->role <= rsc->next_role) && (role != rsc->role)
	&& !pcmk_is_set(rsc->flags, pe_rsc_block)) {
	bool required = need_stop;

	next_role = rsc_state_matrix[role][rsc->role];
	if ((next_role == RSC_ROLE_MASTER) && need_promote) {
	required = true;
	}
	- pe_rsc_trace(rsc, "Up: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role),
	- rsc->id, (required? " required" : ""));
	+ pe_rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
	+ (required? "required" : "optional"), rsc->id,
	+ role2text(role), role2text(next_role));
	if (rsc_action_matrix[role][next_role](rsc, chosen, !required,
	data_set) == FALSE) {
	break;
	}
	role = next_role;
	}
	role = rsc->role;

	/* Required steps from this role to the next */
	while (role != rsc->next_role) {
	next_role = rsc_state_matrix[role][rsc->next_role];
	- pe_rsc_trace(rsc, "Role: Executing: %s->%s = (%s on %s)", role2text(role), role2text(next_role), rsc->id, chosen?chosen->details->uname:"NA");
	+ pe_rsc_trace(rsc, "Creating action to take %s from %s to %s (ending at %s)",
	+ rsc->id, role2text(role), role2text(next_role),
	+ role2text(rsc->next_role));
	if (rsc_action_matrix[role][next_role] (rsc, chosen, FALSE, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	- pe_rsc_trace(rsc, "No monitor additional ops for blocked resource");
	+ pe_rsc_trace(rsc, "Not creating recurring monitors for blocked resource %s",
	+ rsc->id);

	} else if ((rsc->next_role != RSC_ROLE_STOPPED)
	\|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	- pe_rsc_trace(rsc, "Monitor ops for active resource");
	+ pe_rsc_trace(rsc, "Creating recurring monitors for %s resource %s",
	+ ((rsc->next_role == RSC_ROLE_STOPPED)? "unmanaged" : "active"),
	+ rsc->id);
	start = start_action(rsc, chosen, TRUE);
	Recurring(rsc, start, chosen, data_set);
	Recurring_Stopped(rsc, start, chosen, data_set);
	+
	} else {
	- pe_rsc_trace(rsc, "Monitor ops for inactive resource");
	+ pe_rsc_trace(rsc, "Creating recurring monitors for inactive resource %s",
	+ rsc->id);
	Recurring_Stopped(rsc, NULL, NULL, data_set);
	}

	/* if we are stuck in a partial migration, where the target
	* of the partial migration no longer matches the chosen target.
	* A full stop/start is required */
	if (rsc->partial_migration_target && (chosen == NULL \|\| rsc->partial_migration_target->details != chosen->details)) {
	- pe_rsc_trace(rsc, "Not allowing partial migration to continue. %s", rsc->id);
	+ pe_rsc_trace(rsc, "Not allowing partial migration of %s to continue",
	+ rsc->id);
	allow_migrate = FALSE;

	} else if (!is_moving \|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)
	\|\| pcmk_any_flags_set(rsc->flags,
	pe_rsc_failed\|pe_rsc_start_pending)
	\|\| (current && current->details->unclean)
	\|\| rsc->next_role < RSC_ROLE_STARTED) {

	allow_migrate = FALSE;
	}

	if (allow_migrate) {
	handle_migration_actions(rsc, current, chosen, data_set);
	}
	}

	static void
	rsc_avoids_remote_nodes(pe_resource_t *rsc)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->details->remote_rsc) {
	node->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Return allowed nodes as (possibly sorted) list
	*
	* Convert a resource's hash table of allowed nodes to a list. If printing to
	* stdout, sort the list, to keep action ID numbers consistent for regression
	* test output (while avoiding the performance hit on a live cluster).
	*
	* \param[in] rsc Resource to check for allowed nodes
	* \param[in] data_set Cluster working set
	*
	* \return List of resource's allowed nodes
	* \note Callers should take care not to rely on the list being sorted.
	*/
	static GList *
	allowed_nodes_as_list(pe_resource_t rsc, pe_working_set_t data_set)
	{
	GList *allowed_nodes = NULL;

	if (rsc->allowed_nodes) {
	allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	if (pcmk_is_set(data_set->flags, pe_flag_stdout)) {
	allowed_nodes = g_list_sort(allowed_nodes, sort_node_uname);
	}
	return allowed_nodes;
	}

	void
	native_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	/* This function is on the critical path and worth optimizing as much as possible */

	pe_resource_t *top = NULL;
	GList *allowed_nodes = NULL;
	bool check_unfencing = FALSE;
	bool check_utilization = FALSE;

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc,
	"Skipping native constraints for unmanaged resource: %s",
	rsc->id);
	return;
	}

	top = uber_parent(rsc);

	// Whether resource requires unfencing
	check_unfencing = !pcmk_is_set(rsc->flags, pe_rsc_fence_device)
	&& pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)
	&& pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing);

	// Whether a non-default placement strategy is used
	check_utilization = (g_hash_table_size(rsc->utilization) > 0)
	&& !pcmk__str_eq(data_set->placement_strategy,
	"default", pcmk__str_casei);

	// Order stops before starts (i.e. restart)
	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	pe_order_optional\|pe_order_implies_then\|pe_order_restart,
	data_set);

	// Promotable ordering: demote before stop, start before promote
	if (pcmk_is_set(top->flags, pe_rsc_promotable)
	\|\| (rsc->role > RSC_ROLE_SLAVE)) {

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_DEMOTE, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_implies_first_master, data_set);

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_PROMOTE, 0), NULL,
	pe_order_runnable_left, data_set);
	}

	// Don't clear resource history if probing on same node
	custom_action_order(rsc, pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0),
	NULL, rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0),
	NULL, pe_order_same_node\|pe_order_then_cancels_first,
	data_set);

	// Certain checks need allowed nodes
	if (check_unfencing \|\| check_utilization \|\| rsc->container) {
	allowed_nodes = allowed_nodes_as_list(rsc, data_set);
	}

	if (check_unfencing) {
	/* Check if the node needs to be unfenced first */

	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *node = item->data;
	pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);

	crm_debug("Ordering any stops of %s before %s, and any starts after",
	rsc->id, unfence->uuid);

	/*
	* It would be more efficient to order clone resources once,
	* rather than order each instance, but ordering the instance
	* allows us to avoid unnecessary dependencies that might conflict
	* with user constraints.
	*
	* @TODO: This constraint can still produce a transition loop if the
	* resource has a stop scheduled on the node being unfenced, and
	* there is a user ordering constraint to start some other resource
	* (which will be ordered after the unfence) before stopping this
	* resource. An example is "start some slow-starting cloned service
	* before stopping an associated virtual IP that may be moving to
	* it":
	* stop this -> unfencing -> start that -> stop this
	*/
	custom_action_order(rsc, stop_key(rsc), NULL,
	NULL, strdup(unfence->uuid), unfence,
	pe_order_optional\|pe_order_same_node, data_set);

	custom_action_order(NULL, strdup(unfence->uuid), unfence,
	rsc, start_key(rsc), NULL,
	pe_order_implies_then_on_node\|pe_order_same_node,
	data_set);
	}
	}

	if (check_utilization) {
	GListPtr gIter = NULL;

	pe_rsc_trace(rsc, "Creating utilization constraints for %s - strategy: %s",
	rsc->id, data_set->placement_strategy);

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t current = (pe_node_t ) gIter->data;

	char *load_stopped_task = crm_strdup_printf(LOAD_STOPPED "_%s",
	current->details->uname);
	pe_action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = pe__copy_node(current);
	update_action_flags(load_stopped, pe_action_optional \| pe_action_clear,
	__func__, __LINE__);
	}

	custom_action_order(rsc, stop_key(rsc), NULL,
	NULL, load_stopped_task, load_stopped, pe_order_load, data_set);
	}

	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *next = item->data;
	char *load_stopped_task = crm_strdup_printf(LOAD_STOPPED "_%s",
	next->details->uname);
	pe_action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = pe__copy_node(next);
	update_action_flags(load_stopped, pe_action_optional \| pe_action_clear,
	__func__, __LINE__);
	}

	custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
	rsc, start_key(rsc), NULL, pe_order_load, data_set);

	custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
	rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
	NULL, pe_order_load, data_set);

	free(load_stopped_task);
	}
	}

	if (rsc->container) {
	pe_resource_t *remote_rsc = NULL;

	if (rsc->is_remote_node) {
	// rsc is the implicit remote connection for a guest or bundle node

	/* Do not allow a guest resource to live on a Pacemaker Remote node,
	* to avoid nesting remotes. However, allow bundles to run on remote
	* nodes.
	*/
	if (!pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
	rsc_avoids_remote_nodes(rsc->container);
	}

	/* If someone cleans up a guest or bundle node's container, we will
	* likely schedule a (re-)probe of the container and recovery of the
	* connection. Order the connection stop after the container probe,
	* so that if we detect the container running, we will trigger a new
	* transition and avoid the unnecessary recovery.
	*/
	new_rsc_order(rsc->container, RSC_STATUS, rsc, RSC_STOP,
	pe_order_optional, data_set);

	/* A user can specify that a resource must start on a Pacemaker Remote
	* node by explicitly configuring it with the container=NODENAME
	* meta-attribute. This is of questionable merit, since location
	* constraints can accomplish the same thing. But we support it, so here
	* we check whether a resource (that is not itself a remote connection)
	* has container set to a remote node or guest node resource.
	*/
	} else if (rsc->container->is_remote_node) {
	remote_rsc = rsc->container;
	} else {
	remote_rsc = pe__resource_contains_guest_node(data_set,
	rsc->container);
	}

	if (remote_rsc) {
	/* Force the resource on the Pacemaker Remote node instead of
	* colocating the resource with the container resource.
	*/
	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *node = item->data;

	if (node->details->remote_rsc != remote_rsc) {
	node->weight = -INFINITY;
	}
	}

	} else {
	/* This resource is either a filler for a container that does NOT
	* represent a Pacemaker Remote node, or a Pacemaker Remote
	* connection resource for a guest node or bundle.
	*/
	int score;

	crm_trace("Order and colocate %s relative to its container %s",
	rsc->id, rsc->container->id);

	custom_action_order(rsc->container,
	pcmk__op_key(rsc->container->id, RSC_START, 0),
	NULL, rsc, pcmk__op_key(rsc->id, RSC_START, 0),
	NULL,
	pe_order_implies_then\|pe_order_runnable_left,
	data_set);

	custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc->container,
	pcmk__op_key(rsc->container->id, RSC_STOP, 0),
	NULL, pe_order_implies_first, data_set);

	if (pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
	score = 10000; /* Highly preferred but not essential */
	} else {
	score = INFINITY; /* Force them to run on the same host */
	}
	rsc_colocation_new("resource-with-container", NULL, score, rsc,
	rsc->container, NULL, NULL, data_set);
	}
	}

	if (rsc->is_remote_node \|\| pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
	/* don't allow remote nodes to run stonith devices
	* or remote connection resources.*/
	rsc_avoids_remote_nodes(rsc);
	}
	g_list_free(allowed_nodes);
	}

	void
	native_rsc_colocation_lh(pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	rsc_colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	if (rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", constraint->id);
	return;

	} else if (constraint->rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;
	}

	if (constraint->score == 0) {
	return;
	}
	pe_rsc_trace(rsc_lh, "Processing colocation constraint between %s and %s", rsc_lh->id,
	rsc_rh->id);

	rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint, data_set);
	}

	enum filter_colocation_res
	filter_colocation_constraint(pe_resource_t * rsc_lh, pe_resource_t * rsc_rh,
	rsc_colocation_t * constraint, gboolean preview)
	{
	if (constraint->score == 0) {
	return influence_nothing;
	}

	/* rh side must be allocated before we can process constraint */
	if (!preview && pcmk_is_set(rsc_rh->flags, pe_rsc_provisional)) {
	return influence_nothing;
	}

	if ((constraint->role_lh >= RSC_ROLE_SLAVE) &&
	rsc_lh->parent && pcmk_is_set(rsc_lh->parent->flags, pe_rsc_promotable)
	&& !pcmk_is_set(rsc_lh->flags, pe_rsc_provisional)) {

	/* LH and RH resources have already been allocated, place the correct
	* priority on LH rsc for the given promotable clone resource role */
	return influence_rsc_priority;
	}

	if (!preview && !pcmk_is_set(rsc_lh->flags, pe_rsc_provisional)) {
	// Log an error if we violated a mandatory colocation constraint
	const pe_node_t *rh_node = rsc_rh->allocated_to;

	if (rsc_lh->allocated_to == NULL) {
	// Dependent resource isn't allocated, so constraint doesn't matter
	return influence_nothing;
	}

	if (constraint->score >= INFINITY) {
	// Dependent resource must colocate with rh_node

	if ((rh_node == NULL)
	\|\| (rh_node->details != rsc_lh->allocated_to->details)) {
	crm_err("%s must be colocated with %s but is not (%s vs. %s)",
	rsc_lh->id, rsc_rh->id,
	rsc_lh->allocated_to->details->uname,
	(rh_node? rh_node->details->uname : "unallocated"));
	}

	} else if (constraint->score <= -INFINITY) {
	// Dependent resource must anti-colocate with rh_node

	if ((rh_node != NULL)
	&& (rsc_lh->allocated_to->details == rh_node->details)) {
	crm_err("%s and %s must be anti-colocated but are allocated "
	"to the same node (%s)",
	rsc_lh->id, rsc_rh->id, rh_node->details->uname);
	}
	}
	return influence_nothing;
	}

	if (constraint->score > 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh != rsc_lh->next_role) {
	crm_trace("LH: Skipping constraint: \"%s\" state filter nextrole is %s",
	role2text(constraint->role_lh), role2text(rsc_lh->next_role));
	return influence_nothing;
	}

	if (constraint->score > 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh != rsc_rh->next_role) {
	crm_trace("RH: Skipping constraint: \"%s\" state filter", role2text(constraint->role_rh));
	return influence_nothing;
	}

	if (constraint->score < 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh == rsc_lh->next_role) {
	crm_trace("LH: Skipping negative constraint: \"%s\" state filter",
	role2text(constraint->role_lh));
	return influence_nothing;
	}

	if (constraint->score < 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh == rsc_rh->next_role) {
	crm_trace("RH: Skipping negative constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return influence_nothing;
	}

	return influence_rsc_location;
	}

	static void
	influence_priority(pe_resource_t * rsc_lh, pe_resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	const char *rh_value = NULL;
	const char *lh_value = NULL;
	const char *attribute = CRM_ATTR_ID;
	int score_multiplier = 1;

	if (constraint->score == 0) {
	return;
	}
	if (!rsc_rh->allocated_to \|\| !rsc_lh->allocated_to) {
	return;
	}

	if (constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	lh_value = pe_node_attribute_raw(rsc_lh->allocated_to, attribute);
	rh_value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);

	if (!pcmk__str_eq(lh_value, rh_value, pcmk__str_casei)) {
	if(constraint->score == INFINITY && constraint->role_lh == RSC_ROLE_MASTER) {
	rsc_lh->priority = -INFINITY;
	}
	return;
	}

	if (constraint->role_rh && (constraint->role_rh != rsc_rh->next_role)) {
	return;
	}

	if (constraint->role_lh == RSC_ROLE_SLAVE) {
	score_multiplier = -1;
	}

	rsc_lh->priority = pe__add_scores(score_multiplier * constraint->score,
	rsc_lh->priority);
	}

	static void
	colocation_match(pe_resource_t * rsc_lh, pe_resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	const char *attribute = CRM_ATTR_ID;
	const char *value = NULL;
	GHashTable *work = NULL;
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (constraint->score == 0) {
	return;
	}
	if (constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	if (rsc_rh->allocated_to) {
	value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);

	} else if (constraint->score < 0) {
	// Nothing to do (anti-colocation with something that is not running)
	return;
	}

	work = pcmk__copy_node_table(rsc_lh->allowed_nodes);

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (rsc_rh->allocated_to == NULL) {
	pe_rsc_trace(rsc_lh, "%s: %s@%s -= %d (%s inactive)",
	constraint->id, rsc_lh->id, node->details->uname,
	constraint->score, rsc_rh->id);
	node->weight = pe__add_scores(-constraint->score, node->weight);

	} else if (pcmk__str_eq(pe_node_attribute_raw(node, attribute), value, pcmk__str_casei)) {
	if (constraint->score < CRM_SCORE_INFINITY) {
	pe_rsc_trace(rsc_lh, "%s: %s@%s += %d",
	constraint->id, rsc_lh->id,
	node->details->uname, constraint->score);
	node->weight = pe__add_scores(constraint->score, node->weight);
	}

	} else if (constraint->score >= CRM_SCORE_INFINITY) {
	pe_rsc_trace(rsc_lh, "%s: %s@%s -= %d (%s mismatch)",
	constraint->id, rsc_lh->id, node->details->uname,
	constraint->score, attribute);
	node->weight = pe__add_scores(-constraint->score, node->weight);
	}
	}

	if (can_run_any(work)
	\|\| constraint->score <= -INFINITY \|\| constraint->score >= INFINITY) {
	g_hash_table_destroy(rsc_lh->allowed_nodes);
	rsc_lh->allowed_nodes = work;
	work = NULL;

	} else {
	pe_rsc_info(rsc_lh,
	"%s: Rolling back scores from %s (no available nodes)",
	rsc_lh->id, rsc_rh->id);
	}

	if (work) {
	g_hash_table_destroy(work);
	}
	}

	void
	native_rsc_colocation_rh(pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	rsc_colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	enum filter_colocation_res filter_results;

	CRM_ASSERT(rsc_lh);
	CRM_ASSERT(rsc_rh);
	filter_results = filter_colocation_constraint(rsc_lh, rsc_rh, constraint, FALSE);
	pe_rsc_trace(rsc_lh, "%s %s with %s (%s, score=%d, filter=%d)",
	((constraint->score >= 0)? "Colocating" : "Anti-colocating"),
	rsc_lh->id, rsc_rh->id, constraint->id, constraint->score, filter_results);

	switch (filter_results) {
	case influence_rsc_priority:
	influence_priority(rsc_lh, rsc_rh, constraint);
	break;
	case influence_rsc_location:
	colocation_match(rsc_lh, rsc_rh, constraint);
	break;
	case influence_nothing:
	default:
	return;
	}
	}

	static gboolean
	filter_rsc_ticket(pe_resource_t * rsc_lh, rsc_ticket_t * rsc_ticket)
	{
	if (rsc_ticket->role_lh != RSC_ROLE_UNKNOWN && rsc_ticket->role_lh != rsc_lh->role) {
	pe_rsc_trace(rsc_lh, "LH: Skipping constraint: \"%s\" state filter",
	role2text(rsc_ticket->role_lh));
	return FALSE;
	}

	return TRUE;
	}

	void
	rsc_ticket_constraint(pe_resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working_set_t * data_set)
	{
	if (rsc_ticket == NULL) {
	pe_err("rsc_ticket was NULL");
	return;
	}

	if (rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", rsc_ticket->id);
	return;
	}

	if (rsc_ticket->ticket->granted && rsc_ticket->ticket->standby == FALSE) {
	return;
	}

	if (rsc_lh->children) {
	GListPtr gIter = rsc_lh->children;

	pe_rsc_trace(rsc_lh, "Processing ticket dependencies from %s", rsc_lh->id);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	rsc_ticket_constraint(child_rsc, rsc_ticket, data_set);
	}
	return;
	}

	pe_rsc_trace(rsc_lh, "%s: Processing ticket dependency on %s (%s, %s)",
	rsc_lh->id, rsc_ticket->ticket->id, rsc_ticket->id,
	role2text(rsc_ticket->role_lh));

	if ((rsc_ticket->ticket->granted == FALSE)
	&& (rsc_lh->running_on != NULL)) {

	GListPtr gIter = NULL;

	switch (rsc_ticket->loss_policy) {
	case loss_ticket_stop:
	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
	break;

	case loss_ticket_demote:
	// Promotion score will be set to -INFINITY in promotion_order()
	if (rsc_ticket->role_lh != RSC_ROLE_MASTER) {
	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
	}
	break;

	case loss_ticket_fence:
	if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
	return;
	}

	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);

	for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	pe_fence_node(data_set, node, "deadman ticket was lost", FALSE);
	}
	break;

	case loss_ticket_freeze:
	if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
	return;
	}
	if (rsc_lh->running_on != NULL) {
	pe__clear_resource_flags(rsc_lh, pe_rsc_managed);
	pe__set_resource_flags(rsc_lh, pe_rsc_block);
	}
	break;
	}

	} else if (rsc_ticket->ticket->granted == FALSE) {

	if (rsc_ticket->role_lh != RSC_ROLE_MASTER \|\| rsc_ticket->loss_policy == loss_ticket_stop) {
	resource_location(rsc_lh, NULL, -INFINITY, "__no_ticket__", data_set);
	}

	} else if (rsc_ticket->ticket->standby) {

	if (rsc_ticket->role_lh != RSC_ROLE_MASTER \|\| rsc_ticket->loss_policy == loss_ticket_stop) {
	resource_location(rsc_lh, NULL, -INFINITY, "__ticket_standby__", data_set);
	}
	}
	}

	enum pe_action_flags
	native_action_flags(pe_action_t * action, pe_node_t * node)
	{
	return action->flags;
	}

	static inline bool
	is_primitive_action(pe_action_t *action)
	{
	return action && action->rsc && (action->rsc->variant == pe_native);
	}

	/*!
	* \internal
	* \brief Set action bits appropriately when pe_restart_order is used
	*
	* \param[in] first 'First' action in an ordering with pe_restart_order
	* \param[in] then 'Then' action in an ordering with pe_restart_order
	* \param[in] filter What ordering flags to care about
	*
	* \note pe_restart_order is set for "stop resource before starting it" and
	* "stop later group member before stopping earlier group member"
	*/
	static void
	handle_restart_ordering(pe_action_t first, pe_action_t then,
	enum pe_action_flags filter)
	{
	const char *reason = NULL;

	CRM_ASSERT(is_primitive_action(first));
	CRM_ASSERT(is_primitive_action(then));

	// We need to update the action in two cases:

	// ... if 'then' is required
	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {
	reason = "restart";
	}

	/* ... if 'then' is unrunnable start of managed resource (if a resource
	* should restart but can't start, we still want to stop)
	*/
	if (pcmk_is_set(filter, pe_action_runnable)
	&& !pcmk_is_set(then->flags, pe_action_runnable)
	&& pcmk_is_set(then->rsc->flags, pe_rsc_managed)
	&& pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)) {
	reason = "stop";
	}

	if (reason == NULL) {
	return;
	}

	pe_rsc_trace(first->rsc, "Handling %s -> %s for %s",
	first->uuid, then->uuid, reason);

	// Make 'first' required if it is runnable
	if (pcmk_is_set(first->flags, pe_action_runnable)) {
	pe_action_implies(first, then, pe_action_optional);
	}

	// Make 'first' required if 'then' is required
	if (!pcmk_is_set(then->flags, pe_action_optional)) {
	pe_action_implies(first, then, pe_action_optional);
	}

	// Make 'first' unmigratable if 'then' is unmigratable
	if (!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	pe_action_implies(first, then, pe_action_migrate_runnable);
	}

	// Make 'then' unrunnable if 'first' is required but unrunnable
	if (!pcmk_is_set(first->flags, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_runnable)) {
	pe_action_implies(then, first, pe_action_runnable);
	}
	}

	enum pe_graph_flags
	native_update_actions(pe_action_t first, pe_action_t then, pe_node_t *node,
	enum pe_action_flags flags, enum pe_action_flags filter,
	enum pe_ordering type, pe_working_set_t *data_set)
	{
	/* flags == get_action_flags(first, then_node) called from update_action() */
	enum pe_graph_flags changed = pe_graph_none;
	enum pe_action_flags then_flags = then->flags;
	enum pe_action_flags first_flags = first->flags;

	crm_trace( "Testing %s on %s (0x%.6x) with %s 0x%.6x",
	first->uuid, first->node ? first->node->details->uname : "[none]",
	first->flags, then->uuid, then->flags);

	if (type & pe_order_asymmetrical) {
	pe_resource_t *then_rsc = then->rsc;
	enum rsc_role_e then_rsc_role = then_rsc ? then_rsc->fns->state(then_rsc, TRUE) : 0;

	if (!then_rsc) {
	/* ignore */
	} else if ((then_rsc_role == RSC_ROLE_STOPPED) && pcmk__str_eq(then->task, RSC_STOP, pcmk__str_casei)) {
	/* ignore... if 'then' is supposed to be stopped after 'first', but
	* then is already stopped, there is nothing to be done when non-symmetrical. */
	} else if ((then_rsc_role >= RSC_ROLE_STARTED)
	&& pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)
	&& pcmk_is_set(then->flags, pe_action_optional)
	&& then->node
	&& pcmk__list_of_1(then_rsc->running_on)
	&& then->node->details == ((pe_node_t *) then_rsc->running_on->data)->details) {
	/* Ignore. If 'then' is supposed to be started after 'first', but
	* 'then' is already started, there is nothing to be done when
	* asymmetrical -- unless the start is mandatory, which indicates
	* the resource is restarting, and the ordering is still needed.
	*/
	} else if (!(first->flags & pe_action_runnable)) {
	/* prevent 'then' action from happening if 'first' is not runnable and
	* 'then' has not yet occurred. */
	pe_action_implies(then, first, pe_action_optional);
	pe_action_implies(then, first, pe_action_runnable);

	pe_rsc_trace(then->rsc, "Unset optional and runnable on %s", then->uuid);
	} else {
	/* ignore... then is allowed to start/stop if it wants to. */
	}
	}

	if (type & pe_order_implies_first) {
	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(flags /* Should be then_flags? */, pe_action_optional)) {
	// Needs pcmk_is_set(first_flags, pe_action_optional) too?
	pe_rsc_trace(first->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_optional);
	}

	if (pcmk_is_set(flags, pe_action_migrate_runnable) &&
	!pcmk_is_set(then->flags, pe_action_migrate_runnable) &&
	!pcmk_is_set(then->flags, pe_action_optional)) {

	pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s",
	first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_migrate_runnable);
	}
	}

	if (type & pe_order_implies_first_master) {
	if ((filter & pe_action_optional) &&
	((then->flags & pe_action_optional) == FALSE) &&
	then->rsc && (then->rsc->role == RSC_ROLE_MASTER)) {
	pe_action_implies(first, then, pe_action_optional);

	if (pcmk_is_set(first->flags, pe_action_migrate_runnable) &&
	!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {

	pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_migrate_runnable);
	}
	pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
	}
	}

	if ((type & pe_order_implies_first_migratable)
	&& pcmk_is_set(filter, pe_action_optional)) {

	if (((then->flags & pe_action_migrate_runnable) == FALSE) \|\|
	((then->flags & pe_action_runnable) == FALSE)) {

	pe_rsc_trace(then->rsc, "Unset runnable on %s because %s is neither runnable or migratable", first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_runnable);
	}

	if ((then->flags & pe_action_optional) == 0) {
	pe_rsc_trace(then->rsc, "Unset optional on %s because %s is not optional", first->uuid, then->uuid);
	pe_action_implies(first, then, pe_action_optional);
	}
	}

	if ((type & pe_order_pseudo_left)
	&& pcmk_is_set(filter, pe_action_optional)) {

	if ((first->flags & pe_action_runnable) == FALSE) {
	pe_action_implies(then, first, pe_action_migrate_runnable);
	pe__clear_action_flags(then, pe_action_pseudo);
	pe_rsc_trace(then->rsc, "Unset pseudo on %s because %s is not runnable", then->uuid, first->uuid);
	}

	}

	if (pcmk_is_set(type, pe_order_runnable_left)
	&& pcmk_is_set(filter, pe_action_runnable)
	&& pcmk_is_set(then->flags, pe_action_runnable)
	&& !pcmk_is_set(flags, pe_action_runnable)) {
	pe_rsc_trace(then->rsc, "Unset runnable on %s because of %s", then->uuid, first->uuid);
	pe_action_implies(then, first, pe_action_runnable);
	pe_action_implies(then, first, pe_action_migrate_runnable);
	}

	if (pcmk_is_set(type, pe_order_implies_then)
	&& pcmk_is_set(filter, pe_action_optional)
	&& pcmk_is_set(then->flags, pe_action_optional)
	&& !pcmk_is_set(flags, pe_action_optional)) {

	/* in this case, treat migrate_runnable as if first is optional */
	if (!pcmk_is_set(first->flags, pe_action_migrate_runnable)) {
	pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", then->uuid, first->uuid);
	pe_action_implies(then, first, pe_action_optional);
	}
	}

	if (pcmk_is_set(type, pe_order_restart)) {
	handle_restart_ordering(first, then, filter);
	}

	if (then_flags != then->flags) {
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	pe_rsc_trace(then->rsc,
	"Then: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
	then->uuid, then->node ? then->node->details->uname : "[none]", then->flags,
	then_flags, first->uuid, first->flags);

	if(then->rsc && then->rsc->parent) {
	/* "X_stop then X_start" doesn't get handled for cloned groups unless we do this */
	update_action(then, data_set);
	}
	}

	if (first_flags != first->flags) {
	pe__set_graph_flags(changed, first, pe_graph_updated_first);
	pe_rsc_trace(first->rsc,
	"First: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
	first->uuid, first->node ? first->node->details->uname : "[none]",
	first->flags, first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	void
	native_rsc_location(pe_resource_t rsc, pe__location_t constraint)
	{
	GListPtr gIter = NULL;
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (constraint == NULL) {
	pe_err("Constraint is NULL");
	return;

	} else if (rsc == NULL) {
	pe_err("LHS of rsc_to_node (%s) is NULL", constraint->id);
	return;
	}

	pe_rsc_trace(rsc, "Applying %s (%s) to %s", constraint->id,
	role2text(constraint->role_filter), rsc->id);

	/* take "lifetime" into account */
	if (constraint->role_filter > RSC_ROLE_UNKNOWN && constraint->role_filter != rsc->next_role) {
	pe_rsc_debug(rsc, "Constraint (%s) is not active (role : %s vs. %s)",
	constraint->id, role2text(constraint->role_filter), role2text(rsc->next_role));
	return;
	}

	if (constraint->node_list_rh == NULL) {
	pe_rsc_trace(rsc, "RHS of constraint %s is NULL", constraint->id);
	return;
	}

	for (gIter = constraint->node_list_rh; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;
	pe_node_t *other_node = NULL;

	other_node = (pe_node_t *) pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (other_node != NULL) {
	pe_rsc_trace(rsc, "%s + %s: %d + %d",
	node->details->uname,
	other_node->details->uname, node->weight, other_node->weight);
	other_node->weight = pe__add_scores(other_node->weight,
	node->weight);

	} else {
	other_node = pe__copy_node(node);

	pe_rsc_trace(rsc, "%s: %d (insert %d)", other_node->details->uname, other_node->weight, constraint->discover_mode);
	g_hash_table_insert(rsc->allowed_nodes, (gpointer) other_node->details->id, other_node);
	}

	if (other_node->rsc_discover_mode < constraint->discover_mode) {
	if (constraint->discover_mode == pe_discover_exclusive) {
	rsc->exclusive_discover = TRUE;
	}
	/* exclusive > never > always... always is default */
	other_node->rsc_discover_mode = constraint->discover_mode;
	}
	}

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	pe_rsc_trace(rsc, "%s + %s : %d", rsc->id, node->details->uname, node->weight);
	}
	}

	void
	native_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Processing actions from %s", rsc->id);

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	crm_trace("processing action %d for rsc=%s", action->id, rsc->id);
	graph_element_from_action(action, data_set);
	}

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->expand(child_rsc, data_set);
	}
	}

	#define log_change(a, fmt, args...) do { \
	if(a && a->reason && terminal) { \
	printf(" * "fmt" \tdue to %s\n", ##args, a->reason); \
	} else if(a && a->reason) { \
	crm_notice(fmt" \tdue to %s", ##args, a->reason); \
	} else if(terminal) { \
	printf(" * "fmt"\n", ##args); \
	} else { \
	crm_notice(fmt, ##args); \
	} \
	} while(0)

	#define STOP_SANITY_ASSERT(lineno) do { \
	if(current && current->details->unclean) { \
	/* It will be a pseudo op */ \
	} else if(stop == NULL) { \
	crm_err("%s:%d: No stop action exists for %s", \
	__func__, lineno, rsc->id); \
	CRM_ASSERT(stop != NULL); \
	} else if (pcmk_is_set(stop->flags, pe_action_optional)) { \
	crm_err("%s:%d: Action %s is still optional", \
	__func__, lineno, stop->uuid); \
	CRM_ASSERT(!pcmk_is_set(stop->flags, pe_action_optional)); \
	} \
	} while(0)

	static void
	LogAction(const char change, pe_resource_t rsc, pe_node_t origin, pe_node_t destination, pe_action_t action, pe_action_t source, gboolean terminal)
	{
	int len = 0;
	char *reason = NULL;
	char *details = NULL;
	bool same_host = FALSE;
	bool same_role = FALSE;
	bool need_role = FALSE;

	static int rsc_width = 5;
	static int detail_width = 5;

	CRM_ASSERT(action);
	CRM_ASSERT(destination != NULL \|\| origin != NULL);

	if(source == NULL) {
	source = action;
	}

	len = strlen(rsc->id);
	if(len > rsc_width) {
	rsc_width = len + 2;
	}

	if(rsc->role > RSC_ROLE_STARTED \|\| rsc->next_role > RSC_ROLE_SLAVE) {
	need_role = TRUE;
	}

	if(origin != NULL && destination != NULL && origin->details == destination->details) {
	same_host = TRUE;
	}

	if(rsc->role == rsc->next_role) {
	same_role = TRUE;
	}

	if (need_role && (origin == NULL)) {
	/* Starting and promoting a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), destination->details->uname);

	} else if (origin == NULL) {
	/* Starting a resource */
	details = crm_strdup_printf("%s", destination->details->uname);

	} else if (need_role && (destination == NULL)) {
	/* Stopping a promotable clone instance */
	details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname);

	} else if (destination == NULL) {
	/* Stopping a resource */
	details = crm_strdup_printf("%s", origin->details->uname);

	} else if (need_role && same_role && same_host) {
	/* Recovering, restarting or re-promoting a promotable clone instance */
	details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname);

	} else if (same_role && same_host) {
	/* Recovering or Restarting a normal resource */
	details = crm_strdup_printf("%s", origin->details->uname);

	} else if (need_role && same_role) {
	/* Moving a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", origin->details->uname, destination->details->uname, role2text(rsc->role));

	} else if (same_role) {
	/* Moving a normal resource */
	details = crm_strdup_printf("%s -> %s", origin->details->uname, destination->details->uname);

	} else if (same_host) {
	/* Promoting or demoting a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), origin->details->uname);

	} else {
	/* Moving and promoting/demoting */
	details = crm_strdup_printf("%s %s -> %s %s", role2text(rsc->role), origin->details->uname, role2text(rsc->next_role), destination->details->uname);
	}

	len = strlen(details);
	if(len > detail_width) {
	detail_width = len;
	}

	if(source->reason && !pcmk_is_set(action->flags, pe_action_runnable)) {
	reason = crm_strdup_printf(" due to %s (blocked)", source->reason);

	} else if(source->reason) {
	reason = crm_strdup_printf(" due to %s", source->reason);

	} else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	reason = strdup(" blocked");

	} else {
	reason = strdup("");
	}

	if(terminal) {
	printf(" * %-8s %-s ( %s ) %s\n", change, rsc_width, rsc->id, detail_width, details, reason);
	} else {
	crm_notice(" * %-8s %-s ( %s ) %s", change, rsc_width, rsc->id, detail_width, details, reason);
	}

	free(details);
	free(reason);
	}


	void
	LogActions(pe_resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
	{
	pe_node_t *next = NULL;
	pe_node_t *current = NULL;
	pe_node_t *start_node = NULL;

	pe_action_t *stop = NULL;
	pe_action_t *start = NULL;
	pe_action_t *demote = NULL;
	pe_action_t *promote = NULL;

	char *key = NULL;
	gboolean moving = FALSE;
	GListPtr possible_matches = NULL;

	if(rsc->variant == pe_container) {
	pcmk__bundle_log_actions(rsc, data_set, terminal);
	return;
	}

	if (rsc->children) {
	GListPtr gIter = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	LogActions(child_rsc, data_set, terminal);
	}
	return;
	}

	next = rsc->allocated_to;
	if (rsc->running_on) {
	current = pe__current_node(rsc);
	if (rsc->role == RSC_ROLE_STOPPED) {
	/*
	* This can occur when resources are being recovered
	* We fiddle with the current role in native_create_actions()
	*/
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	if ((current == NULL) && pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	/* Don't log stopped orphans */
	return;
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)
	\|\| (current == NULL && next == NULL)) {
	pe_rsc_info(rsc, "Leave %s\t(%s%s)",
	rsc->id, role2text(rsc->role),
	!pcmk_is_set(rsc->flags, pe_rsc_managed)? " unmanaged" : "");
	return;
	}

	if (current != NULL && next != NULL && !pcmk__str_eq(current->details->id, next->details->id, pcmk__str_casei)) {
	moving = TRUE;
	}

	possible_matches = pe__resource_actions(rsc, next, RSC_START, FALSE);
	if (possible_matches) {
	start = possible_matches->data;
	g_list_free(possible_matches);
	}

	if ((start == NULL) \|\| !pcmk_is_set(start->flags, pe_action_runnable)) {
	start_node = NULL;
	} else {
	start_node = current;
	}
	possible_matches = pe__resource_actions(rsc, start_node, RSC_STOP, FALSE);
	if (possible_matches) {
	stop = possible_matches->data;
	g_list_free(possible_matches);
	}

	possible_matches = pe__resource_actions(rsc, next, RSC_PROMOTE, FALSE);
	if (possible_matches) {
	promote = possible_matches->data;
	g_list_free(possible_matches);
	}

	possible_matches = pe__resource_actions(rsc, next, RSC_DEMOTE, FALSE);
	if (possible_matches) {
	demote = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (rsc->role == rsc->next_role) {
	pe_action_t *migrate_op = NULL;

	possible_matches = pe__resource_actions(rsc, next, RSC_MIGRATED, FALSE);
	if (possible_matches) {
	migrate_op = possible_matches->data;
	}

	CRM_CHECK(next != NULL,);
	if (next == NULL) {
	} else if ((migrate_op != NULL) && (current != NULL)
	&& pcmk_is_set(migrate_op->flags, pe_action_runnable)) {
	LogAction("Migrate", rsc, current, next, start, NULL, terminal);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_reload)) {
	LogAction("Reload", rsc, current, next, start, NULL, terminal);


	} else if (start == NULL \|\| pcmk_is_set(start->flags, pe_action_optional)) {
	if ((demote != NULL) && (promote != NULL)
	&& !pcmk_is_set(demote->flags, pe_action_optional)
	&& !pcmk_is_set(promote->flags, pe_action_optional)) {
	LogAction("Re-promote", rsc, current, next, promote, demote,
	terminal);
	} else {
	pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id,
	role2text(rsc->role), next->details->uname);
	}

	} else if (!pcmk_is_set(start->flags, pe_action_runnable)) {
	LogAction("Stop", rsc, current, NULL, stop,
	(stop && stop->reason)? stop : start, terminal);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (moving && current) {
	LogAction(pcmk_is_set(rsc->flags, pe_rsc_failed)? "Recover" : "Move",
	rsc, current, next, stop, NULL, terminal);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	LogAction("Recover", rsc, current, NULL, stop, NULL, terminal);
	STOP_SANITY_ASSERT(__LINE__);

	} else {
	LogAction("Restart", rsc, current, next, start, NULL, terminal);
	/* STOP_SANITY_ASSERT(__LINE__); False positive for migrate-fail-7 */
	}

	g_list_free(possible_matches);
	return;
	}

	if(stop
	&& (rsc->next_role == RSC_ROLE_STOPPED
	\|\| (start && !pcmk_is_set(start->flags, pe_action_runnable)))) {

	GListPtr gIter = NULL;

	key = stop_key(rsc);
	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;
	pe_action_t *stop_op = NULL;

	possible_matches = find_actions(rsc->actions, key, node);
	if (possible_matches) {
	stop_op = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (stop_op && (stop_op->flags & pe_action_runnable)) {
	STOP_SANITY_ASSERT(__LINE__);
	}

	LogAction("Stop", rsc, node, NULL, stop_op,
	(stop_op && stop_op->reason)? stop_op : start, terminal);
	}

	free(key);

	} else if ((stop != NULL)
	&& pcmk_all_flags_set(rsc->flags, pe_rsc_failed\|pe_rsc_stop)) {
	/* 'stop' may be NULL if the failure was ignored */
	LogAction("Recover", rsc, current, next, stop, start, terminal);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (moving) {
	LogAction("Move", rsc, current, next, stop, NULL, terminal);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_reload)) {
	LogAction("Reload", rsc, current, next, start, NULL, terminal);

	} else if (stop != NULL && !pcmk_is_set(stop->flags, pe_action_optional)) {
	LogAction("Restart", rsc, current, next, start, NULL, terminal);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (rsc->role == RSC_ROLE_MASTER) {
	CRM_LOG_ASSERT(current != NULL);
	LogAction("Demote", rsc, current, next, demote, NULL, terminal);

	} else if(rsc->next_role == RSC_ROLE_MASTER) {
	CRM_LOG_ASSERT(next);
	LogAction("Promote", rsc, current, next, promote, NULL, terminal);

	} else if (rsc->role == RSC_ROLE_STOPPED && rsc->next_role > RSC_ROLE_STOPPED) {
	LogAction("Start", rsc, current, next, start, NULL, terminal);
	}
	}

	gboolean
	StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t current = (pe_node_t ) gIter->data;
	pe_action_t *stop;

	if (rsc->partial_migration_target) {
	if (rsc->partial_migration_target->details == current->details) {
	pe_rsc_trace(rsc, "Filtered %s -> %s %s", current->details->uname,
	next->details->uname, rsc->id);
	continue;
	} else {
	pe_rsc_trace(rsc, "Forced on %s %s", current->details->uname, rsc->id);
	optional = FALSE;
	}
	}

	pe_rsc_trace(rsc, "%s on %s", rsc->id, current->details->uname);
	stop = stop_action(rsc, current, optional);

	if(rsc->allocated_to == NULL) {
	pe_action_set_reason(stop, "node availability", TRUE);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	update_action_flags(stop, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	}

	if (pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, current, optional, data_set);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
	pe_action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, FALSE, data_set);

	order_actions(stop, unfence, pe_order_implies_first);
	if (!node_has_been_unfenced(current)) {
	pe_proc_err("Stopping %s until %s can be unfenced", rsc->id, current->details->uname);
	}
	}
	}

	return TRUE;
	}

	static void
	order_after_unfencing(pe_resource_t rsc, pe_node_t node, pe_action_t *action,
	enum pe_ordering order, pe_working_set_t *data_set)
	{
	/* When unfencing is in use, we order unfence actions before any probe or
	* start of resources that require unfencing, and also of fence devices.
	*
	* This might seem to violate the principle that fence devices require
	* only quorum. However, fence agents that unfence often don't have enough
	* information to even probe or start unless the node is first unfenced.
	*/
	if (is_unfence_device(rsc, data_set)
	\|\| pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {

	/* Start with an optional ordering. Requiring unfencing would result in
	* the node being unfenced, and all its resources being stopped,
	* whenever a new resource is added -- which would be highly suboptimal.
	*/
	pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);

	order_actions(unfence, action, order);

	if (!node_has_been_unfenced(node)) {
	// But unfencing is required if it has never been done
	char *reason = crm_strdup_printf("required by %s %s",
	rsc->id, action->task);

	trigger_unfencing(NULL, node, reason, NULL, data_set);
	free(reason);
	}
	}
	}

	gboolean
	StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	pe_action_t *start = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s on %s %d %d", rsc->id, next ? next->details->uname : "N/A", optional, next ? next->weight : 0);
	start = start_action(rsc, next, TRUE);

	order_after_unfencing(rsc, next, start, pe_order_implies_then, data_set);

	if (pcmk_is_set(start->flags, pe_action_runnable) && !optional) {
	update_action_flags(start, pe_action_optional \| pe_action_clear,
	__func__, __LINE__);
	}


	return TRUE;
	}

	gboolean
	PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	gboolean runnable = TRUE;
	GListPtr action_list = NULL;

	CRM_ASSERT(rsc);
	CRM_CHECK(next != NULL, return FALSE);
	pe_rsc_trace(rsc, "%s on %s", rsc->id, next->details->uname);

	action_list = pe__resource_actions(rsc, next, RSC_START, TRUE);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t start = (pe_action_t ) gIter->data;

	if (!pcmk_is_set(start->flags, pe_action_runnable)) {
	runnable = FALSE;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	promote_action(rsc, next, optional);
	return TRUE;
	}

	pe_rsc_debug(rsc, "%s\tPromote %s (canceled)", next->details->uname, rsc->id);

	action_list = pe__resource_actions(rsc, next, RSC_PROMOTE, TRUE);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t promote = (pe_action_t ) gIter->data;

	update_action_flags(promote, pe_action_runnable \| pe_action_clear,
	__func__, __LINE__);
	}

	g_list_free(action_list);
	return TRUE;
	}

	gboolean
	DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);

	/* CRM_CHECK(rsc->next_role == RSC_ROLE_SLAVE, return FALSE); */
	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t current = (pe_node_t ) gIter->data;

	pe_rsc_trace(rsc, "%s on %s", rsc->id, next ? next->details->uname : "N/A");
	demote_action(rsc, current, optional);
	}
	return TRUE;
	}

	gboolean
	RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	CRM_ASSERT(rsc);
	crm_err("%s on %s", rsc->id, next ? next->details->uname : "N/A");
	CRM_CHECK(FALSE, return FALSE);
	return FALSE;
	}

	gboolean
	NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);
	return FALSE;
	}

	gboolean
	DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set)
	{
	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_trace(rsc, "Resource %s not deleted from %s: failed", rsc->id, node->details->uname);
	return FALSE;

	} else if (node == NULL) {
	pe_rsc_trace(rsc, "Resource %s not deleted: NULL node", rsc->id);
	return FALSE;

	} else if (node->details->unclean \|\| node->details->online == FALSE) {
	pe_rsc_trace(rsc, "Resource %s not deleted from %s: unrunnable", rsc->id,
	node->details->uname);
	return FALSE;
	}

	crm_notice("Removing %s from %s", rsc->id, node->details->uname);

	delete_action(rsc, node, optional);

	new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE,
	optional ? pe_order_implies_then : pe_order_optional, data_set);

	new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START,
	optional ? pe_order_implies_then : pe_order_optional, data_set);

	return TRUE;
	}

	gboolean
	native_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
	gboolean force, pe_working_set_t * data_set)
	{
	enum pe_ordering flags = pe_order_optional;
	char *key = NULL;
	pe_action_t *probe = NULL;
	pe_node_t *running = NULL;
	pe_node_t *allowed = NULL;
	pe_resource_t *top = uber_parent(rsc);

	static const char *rc_master = NULL;
	static const char *rc_inactive = NULL;

	if (rc_inactive == NULL) {
	rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING);
	rc_master = crm_itoa(PCMK_OCF_RUNNING_MASTER);
	}

	CRM_CHECK(node != NULL, return FALSE);
	if (!force && !pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
	pe_rsc_trace(rsc, "Skipping active resource detection for %s", rsc->id);
	return FALSE;
	}

	if (pe__is_guest_or_remote_node(node)) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because Pacemaker Remote nodes cannot run stonith agents",
	rsc->id, node->details->id);
	return FALSE;
	} else if (pe__is_guest_node(node)
	&& pe__resource_contains_guest_node(data_set, rsc)) {
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because guest nodes cannot run resources containing guest nodes",
	rsc->id, node->details->id);
	return FALSE;
	} else if (rsc->is_remote_node) {
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because Pacemaker Remote nodes cannot host remote connections",
	rsc->id, node->details->id);
	return FALSE;
	}
	}

	if (rsc->children) {
	GListPtr gIter = NULL;
	gboolean any_created = FALSE;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	any_created = child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set)
	\|\| any_created;
	}

	return any_created;

	} else if ((rsc->container) && (!rsc->is_remote_node)) {
	pe_rsc_trace(rsc, "Skipping %s: it is within container %s", rsc->id, rsc->container->id);
	return FALSE;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_rsc_trace(rsc, "Skipping orphan: %s", rsc->id);
	return FALSE;
	}

	// Check whether resource is already known on node
	if (!force && g_hash_table_lookup(rsc->known_on, node->details->id)) {
	pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname);
	return FALSE;
	}

	allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (rsc->exclusive_discover \|\| top->exclusive_discover) {
	if (allowed == NULL) {
	/* exclusive discover is enabled and this node is not in the allowed list. */
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, A", rsc->id, node->details->id);
	return FALSE;
	} else if (allowed->rsc_discover_mode != pe_discover_exclusive) {
	/* exclusive discover is enabled and this node is not marked
	* as a node this resource should be discovered on */
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, B", rsc->id, node->details->id);
	return FALSE;
	}
	}

	if(allowed == NULL && node->rsc_discover_mode == pe_discover_never) {
	/* If this node was allowed to host this resource it would
	* have been explicitly added to the 'allowed_nodes' list.
	* However it wasn't and the node has discovery disabled, so
	* no need to probe for this resource.
	*/
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, C", rsc->id, node->details->id);
	return FALSE;
	}

	if (allowed && allowed->rsc_discover_mode == pe_discover_never) {
	/* this resource is marked as not needing to be discovered on this node */
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, discovery mode", rsc->id, node->details->id);
	return FALSE;
	}

	if (pe__is_guest_node(node)) {
	pe_resource_t *remote = node->details->remote_rsc->container;

	if(remote->role == RSC_ROLE_STOPPED) {
	/* If the container is stopped, then we know anything that
	* might have been inside it is also stopped and there is
	* no need to probe.
	*
	* If we don't know the container's state on the target
	* either:
	*
	* - the container is running, the transition will abort
	* and we'll end up in a different case next time, or
	*
	* - the container is stopped
	*
	* Either way there is no need to probe.
	*
	*/
	if(remote->allocated_to
	&& g_hash_table_lookup(remote->known_on, remote->allocated_to->details->id) == NULL) {
	/* For safety, we order the 'rsc' start after 'remote'
	* has been probed.
	*
	* Using 'top' helps for groups, but we may need to
	* follow the start's ordering chain backwards.
	*/
	custom_action_order(remote,
	pcmk__op_key(remote->id, RSC_STATUS, 0),
	NULL, top,
	pcmk__op_key(top->id, RSC_START, 0), NULL,
	pe_order_optional, data_set);
	}
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopped",
	rsc->id, node->details->id, remote->id);
	return FALSE;

	/* Here we really we want to check if remote->stop is required,
	* but that information doesn't exist yet
	*/
	} else if(node->details->remote_requires_reset
	\|\| node->details->unclean
	\|\| pcmk_is_set(remote->flags, pe_rsc_failed)
	\|\| remote->next_role == RSC_ROLE_STOPPED
	\|\| (remote->allocated_to
	&& pe_find_node(remote->running_on, remote->allocated_to->details->uname) == NULL)
	) {
	/* The container is stopping or restarting, don't start
	* 'rsc' until 'remote' stops as this also implies that
	* 'rsc' is stopped - avoiding the need to probe
	*/
	custom_action_order(remote, pcmk__op_key(remote->id, RSC_STOP, 0),
	NULL, top, pcmk__op_key(top->id, RSC_START, 0),
	NULL, pe_order_optional, data_set);
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopping, restarting or moving",
	rsc->id, node->details->id, remote->id);
	return FALSE;
	/* } else {
	* The container is running so there is no problem probing it
	*/
	}
	}

	key = pcmk__op_key(rsc->id, RSC_STATUS, 0);
	probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
	update_action_flags(probe, pe_action_optional \| pe_action_clear, __func__,
	__LINE__);

	order_after_unfencing(rsc, node, probe, pe_order_optional, data_set);

	/*
	* We need to know if it's running_on (not just known_on) this node
	* to correctly determine the target rc.
	*/
	running = pe_find_node_id(rsc->running_on, node->details->id);
	if (running == NULL) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);

	} else if (rsc->role == RSC_ROLE_MASTER) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_master);
	}

	crm_debug("Probing %s on %s (%s) %d %p", rsc->id, node->details->uname, role2text(rsc->role),
	pcmk_is_set(probe->flags, pe_action_runnable), rsc->running_on);

	if (is_unfence_device(rsc, data_set) \|\| !pe_rsc_is_clone(top)) {
	top = rsc;
	} else {
	crm_trace("Probing %s on %s (%s) as %s", rsc->id, node->details->uname, role2text(rsc->role), top->id);
	}

	if (!pcmk_is_set(probe->flags, pe_action_runnable)
	&& (rsc->running_on == NULL)) {
	/* Prevent the start from occurring if rsc isn't active, but
	* don't cause it to stop if it was active already
	*/
	pe__set_order_flags(flags, pe_order_runnable_left);
	}

	custom_action_order(rsc, NULL, probe,
	top, pcmk__op_key(top->id, RSC_START, 0), NULL,
	flags, data_set);

	/* Before any reloads, if they exist */
	custom_action_order(rsc, NULL, probe,
	top, reload_key(rsc), NULL,
	pe_order_optional, data_set);

	#if 0
	// complete is always null currently
	if (!is_unfence_device(rsc, data_set)) {
	/* Normally rsc.start depends on probe complete which depends
	* on rsc.probe. But this can't be the case for fence devices
	* with unfencing, as it would create graph loops.
	*
	* So instead we explicitly order 'rsc.probe then rsc.start'
	*/
	order_actions(probe, complete, pe_order_implies_then);
	}
	#endif
	return TRUE;
	}

	/*!
	* \internal
	* \brief Check whether a resource is known on a particular node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return TRUE if resource (or parent if an anonymous clone) is known
	*/
	static bool
	rsc_is_known_on(pe_resource_t rsc, const pe_node_t node)
	{
	if (pe_hash_table_lookup(rsc->known_on, node->details->id)) {
	return TRUE;

	} else if ((rsc->variant == pe_native)
	&& pe_rsc_is_anon_clone(rsc->parent)
	&& pe_hash_table_lookup(rsc->parent->known_on, node->details->id)) {
	/* We check only the parent, not the uber-parent, because we cannot
	* assume that the resource is known if it is in an anonymously cloned
	* group (which may be only partially known).
	*/
	return TRUE;
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Order a resource's start and promote actions relative to fencing
	*
	* \param[in] rsc Resource to be ordered
	* \param[in] stonith_op Fence action
	* \param[in] data_set Cluster information
	*/
	static void
	native_start_constraints(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
	{
	pe_node_t *target;
	GListPtr gIter = NULL;

	CRM_CHECK(stonith_op && stonith_op->node, return);
	target = stonith_op->node;

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	switch (action->needs) {
	case rsc_req_nothing:
	// Anything other than start or promote requires nothing
	break;

	case rsc_req_stonith:
	order_actions(stonith_op, action, pe_order_optional);
	break;

	case rsc_req_quorum:
	if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
	&& pe_hash_table_lookup(rsc->allowed_nodes, target->details->id)
	&& !rsc_is_known_on(rsc, target)) {

	/* If we don't know the status of the resource on the node
	* we're about to shoot, we have to assume it may be active
	* there. Order the resource start after the fencing. This
	* is analogous to waiting for all the probes for a resource
	* to complete before starting it.
	*
	* The most likely explanation is that the DC died and took
	* its status with it.
	*/
	pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid,
	target->details->uname);
	order_actions(stonith_op, action,
	pe_order_optional \| pe_order_runnable_left);
	}
	break;
	}
	}
	}

	static void
	native_stop_constraints(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	GListPtr action_list = NULL;
	bool order_implicit = false;

	pe_resource_t *top = uber_parent(rsc);
	pe_action_t *parent_stop = NULL;
	pe_node_t *target;

	CRM_CHECK(stonith_op && stonith_op->node, return);
	target = stonith_op->node;

	/* Get a list of stop actions potentially implied by the fencing */
	action_list = pe__resource_actions(rsc, target, RSC_STOP, FALSE);

	/* If resource requires fencing, implicit actions must occur after fencing.
	*
	* Implied stops and demotes of resources running on guest nodes are always
	* ordered after fencing, even if the resource does not require fencing,
	* because guest node "fencing" is actually just a resource stop.
	*/
	if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)
	\|\| pe__is_guest_node(target)) {

	order_implicit = true;
	}

	if (action_list && order_implicit) {
	parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
	}

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	// The stop would never complete, so convert it into a pseudo-action.
	update_action_flags(action, pe_action_pseudo\|pe_action_runnable,
	__func__, __LINE__);

	if (order_implicit) {
	update_action_flags(action, pe_action_implied_by_stonith,
	__func__, __LINE__);

	/* Order the stonith before the parent stop (if any).
	*
	* Also order the stonith before the resource stop, unless the
	* resource is inside a bundle -- that would cause a graph loop.
	* We can rely on the parent stop's ordering instead.
	*
	* User constraints must not order a resource in a guest node
	* relative to the guest node container resource. The
	* pe_order_preserve flag marks constraints as generated by the
	* cluster and thus immune to that check (and is irrelevant if
	* target is not a guest).
	*/
	if (!pe_rsc_is_bundled(rsc)) {
	order_actions(stonith_op, action, pe_order_preserve);
	}
	order_actions(stonith_op, parent_stop, pe_order_preserve);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	crm_notice("Stop of failed resource %s is implicit %s %s is fenced",
	rsc->id, (order_implicit? "after" : "because"),
	target->details->uname);
	} else {
	crm_info("%s is implicit %s %s is fenced",
	action->uuid, (order_implicit? "after" : "because"),
	target->details->uname);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_notify)) {
	/* Create a second notification that will be delivered
	* immediately after the node is fenced
	*
	* Basic problem:
	* - C is a clone active on the node to be shot and stopping on another
	* - R is a resource that depends on C
	*
	* + C.stop depends on R.stop
	* + C.stopped depends on STONITH
	* + C.notify depends on C.stopped
	* + C.healthy depends on C.notify
	* + R.stop depends on C.healthy
	*
	* The extra notification here changes
	* + C.healthy depends on C.notify
	* into:
	* + C.healthy depends on C.notify'
	* + C.notify' depends on STONITH'
	* thus breaking the loop
	*/
	create_secondary_notification(action, rsc, stonith_op, data_set);
	}

	/* From Bug #1601, successful fencing must be an input to a failed resources stop action.

	However given group(rA, rB) running on nodeX and B.stop has failed,
	A := stop healthy resource (rA.stop)
	B := stop failed resource (pseudo operation B.stop)
	C := stonith nodeX
	A requires B, B requires C, C requires A
	This loop would prevent the cluster from making progress.

	This block creates the "C requires A" dependency and therefore must (at least
	for now) be disabled.

	Instead, run the block above and treat all resources on nodeX as B would be
	(marked as a pseudo op depending on the STONITH).

	TODO: Break the "A requires B" dependency in update_action() and re-enable this block

	} else if(is_stonith == FALSE) {
	crm_info("Moving healthy resource %s"
	" off %s before fencing",
	rsc->id, node->details->uname);

	* stop healthy resources before the
	* stonith op
	*
	custom_action_order(
	rsc, stop_key(rsc), NULL,
	NULL,strdup(CRM_OP_FENCE),stonith_op,
	pe_order_optional, data_set);
	*/
	}

	g_list_free(action_list);

	/* Get a list of demote actions potentially implied by the fencing */
	action_list = pe__resource_actions(rsc, target, RSC_DEMOTE, FALSE);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (action->node->details->online == FALSE \|\| action->node->details->unclean == TRUE
	\|\| pcmk_is_set(rsc->flags, pe_rsc_failed)) {

	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_info(rsc,
	"Demote of failed resource %s is implicit after %s is fenced",
	rsc->id, target->details->uname);
	} else {
	pe_rsc_info(rsc, "%s is implicit after %s is fenced",
	action->uuid, target->details->uname);
	}

	/* The demote would never complete and is now implied by the
	* fencing, so convert it into a pseudo-action.
	*/
	update_action_flags(action, pe_action_pseudo\|pe_action_runnable,
	__func__, __LINE__);

	if (pe_rsc_is_bundled(rsc)) {
	/* Do nothing, let the recovery be ordered after the parent's implied stop */

	} else if (order_implicit) {
	order_actions(stonith_op, action, pe_order_preserve\|pe_order_optional);
	}
	}
	}

	g_list_free(action_list);
	}

	void
	rsc_stonith_ordering(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
	{
	if (rsc->children) {
	GListPtr gIter = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	rsc_stonith_ordering(child_rsc, stonith_op, data_set);
	}

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id);

	} else {
	native_start_constraints(rsc, stonith_op, data_set);
	native_stop_constraints(rsc, stonith_op, data_set);
	}
	}

	void
	ReloadRsc(pe_resource_t * rsc, pe_node_t node, pe_working_set_t data_set)
	{
	GListPtr gIter = NULL;
	pe_action_t *reload = NULL;

	if (rsc->children) {
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	ReloadRsc(child_rsc, node, data_set);
	}
	return;

	} else if (rsc->variant > pe_native) {
	/* Complex resource with no children */
	return;

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "%s: unmanaged", rsc->id);
	return;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	/* We don't need to specify any particular actions here, normal failure
	* recovery will apply.
	*/
	pe_rsc_trace(rsc, "%s: preventing reload because failed", rsc->id);
	return;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
	/* If a resource's configuration changed while a start was pending,
	* force a full restart.
	*/
	pe_rsc_trace(rsc, "%s: preventing reload because start pending", rsc->id);
	stop_action(rsc, node, FALSE);
	return;

	} else if (node == NULL) {
	pe_rsc_trace(rsc, "%s: not active", rsc->id);
	return;
	}

	pe_rsc_trace(rsc, "Processing %s", rsc->id);
	pe__set_resource_flags(rsc, pe_rsc_reload);

	reload = custom_action(
	rsc, reload_key(rsc), CRMD_ACTION_RELOAD, node, FALSE, TRUE, data_set);
	pe_action_set_reason(reload, "resource definition change", FALSE);

	custom_action_order(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
	pe_order_optional\|pe_order_then_cancels_first,
	data_set);
	custom_action_order(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
	pe_order_optional\|pe_order_then_cancels_first,
	data_set);
	}

	void
	native_append_meta(pe_resource_t * rsc, xmlNode * xml)
	{
	char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
	pe_resource_t *parent;

	if (value) {
	char *name = NULL;

	name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
	crm_xml_add(xml, name, value);
	free(name);
	}

	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
	if (value) {
	char *name = NULL;

	name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	for (parent = rsc; parent != NULL; parent = parent->parent) {
	if (parent->container) {
	crm_xml_add(xml, CRM_META"_"XML_RSC_ATTR_CONTAINER, parent->container->id);
	}
	}
	}
	diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
	index 8ed4eafa6f..5c33a0bf49 100644
	--- a/lib/pengine/utils.c
	+++ b/lib/pengine/utils.c
	@@ -1,2834 +1,2832 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>

	extern xmlNode get_object_root(const char object_type, xmlNode * the_root);
	void print_str_str(gpointer key, gpointer value, gpointer user_data);
	gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);
	static void unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
	pe_working_set_t * data_set, guint interval_ms);
	static xmlNode find_rsc_op_entry_helper(pe_resource_t rsc, const char *key,
	gboolean include_disabled);

	#if ENABLE_VERSIONED_ATTRS
	pe_rsc_action_details_t *
	pe_rsc_action_details(pe_action_t *action)
	{
	pe_rsc_action_details_t *details;

	CRM_CHECK(action != NULL, return NULL);

	if (action->action_details == NULL) {
	action->action_details = calloc(1, sizeof(pe_rsc_action_details_t));
	CRM_CHECK(action->action_details != NULL, return NULL);
	}

	details = (pe_rsc_action_details_t *) action->action_details;
	if (details->versioned_parameters == NULL) {
	details->versioned_parameters = create_xml_node(NULL,
	XML_TAG_OP_VER_ATTRS);
	}
	if (details->versioned_meta == NULL) {
	details->versioned_meta = create_xml_node(NULL, XML_TAG_OP_VER_META);
	}
	return details;
	}

	static void
	pe_free_rsc_action_details(pe_action_t *action)
	{
	pe_rsc_action_details_t *details;

	if ((action == NULL) \|\| (action->action_details == NULL)) {
	return;
	}

	details = (pe_rsc_action_details_t *) action->action_details;

	if (details->versioned_parameters) {
	free_xml(details->versioned_parameters);
	}
	if (details->versioned_meta) {
	free_xml(details->versioned_meta);
	}

	action->action_details = NULL;
	}
	#endif

	/*!
	* \internal
	* \brief Check whether we can fence a particular node
	*
	* \param[in] data_set Working set for cluster
	* \param[in] node Name of node to check
	*
	* \return true if node can be fenced, false otherwise
	*/
	bool
	pe_can_fence(pe_working_set_t data_set, pe_node_t node)
	{
	if (pe__is_guest_node(node)) {
	/* Guest nodes are fenced by stopping their container resource. We can
	* do that if the container's host is either online or fenceable.
	*/
	pe_resource_t *rsc = node->details->remote_rsc->container;

	for (GList *n = rsc->running_on; n != NULL; n = n->next) {
	pe_node_t *container_node = n->data;

	if (!container_node->details->online
	&& !pe_can_fence(data_set, container_node)) {
	return false;
	}
	}
	return true;

	} else if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	return false; /* Turned off */

	} else if (!pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {
	return false; /* No devices */

	} else if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	return true;

	} else if (data_set->no_quorum_policy == no_quorum_ignore) {
	return true;

	} else if(node == NULL) {
	return false;

	} else if(node->details->online) {
	crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname);
	return true;
	}

	crm_trace("Cannot fence %s", node->details->uname);
	return false;
	}

	/*!
	* \internal
	* \brief Copy a node object
	*
	* \param[in] this_node Node object to copy
	*
	* \return Newly allocated shallow copy of this_node
	* \note This function asserts on errors and is guaranteed to return non-NULL.
	*/
	pe_node_t *
	pe__copy_node(const pe_node_t *this_node)
	{
	pe_node_t *new_node = NULL;

	CRM_ASSERT(this_node != NULL);

	new_node = calloc(1, sizeof(pe_node_t));
	CRM_ASSERT(new_node != NULL);

	new_node->rsc_discover_mode = this_node->rsc_discover_mode;
	new_node->weight = this_node->weight;
	new_node->fixed = this_node->fixed;
	new_node->details = this_node->details;

	return new_node;
	}

	/* any node in list1 or list2 and not in the other gets a score of -INFINITY */
	void
	node_list_exclude(GHashTable * hash, GListPtr list, gboolean merge_scores)
	{
	GHashTable *result = hash;
	pe_node_t *other_node = NULL;
	GListPtr gIter = list;

	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, hash);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {

	other_node = pe_find_node_id(list, node->details->id);
	if (other_node == NULL) {
	node->weight = -INFINITY;
	} else if (merge_scores) {
	node->weight = pe__add_scores(node->weight, other_node->weight);
	}
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	other_node = pe_hash_table_lookup(result, node->details->id);

	if (other_node == NULL) {
	pe_node_t *new_node = pe__copy_node(node);

	new_node->weight = -INFINITY;
	g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
	}
	}
	}

	/*!
	* \internal
	* \brief Create a node hash table from a node list
	*
	* \param[in] list Node list
	*
	* \return Hash table equivalent of node list
	*/
	GHashTable *
	pe__node_list2table(GList *list)
	{
	GHashTable *result = NULL;

	result = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free);
	for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pe_node_t new_node = pe__copy_node((pe_node_t ) gIter->data);

	g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
	}
	return result;
	}

	gint
	sort_node_uname(gconstpointer a, gconstpointer b)
	{
	return pcmk_numeric_strcasecmp(((const pe_node_t *) a)->details->uname,
	((const pe_node_t *) b)->details->uname);
	}

	/*!
	* \internal
	* \brief Output node weights to stdout
	*
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	*/
	static void
	pe__output_node_weights(pe_resource_t rsc, const char comment,
	GHashTable *nodes)
	{
	char score[128]; // Stack-allocated since this is called frequently

	// Sort the nodes so the output is consistent for regression tests
	GList *list = g_list_sort(g_hash_table_get_values(nodes), sort_node_uname);

	for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	score2char_stack(node->weight, score, sizeof(score));
	if (rsc) {
	printf("%s: %s allocation score on %s: %s\n",
	comment, rsc->id, node->details->uname, score);
	} else {
	printf("%s: %s = %s\n", comment, node->details->uname, score);
	}
	}
	g_list_free(list);
	}

	/*!
	* \internal
	* \brief Log node weights at trace level
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	*/
	static void
	pe__log_node_weights(const char file, const char function, int line,
	pe_resource_t rsc, const char comment, GHashTable *nodes)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	char score[128]; // Stack-allocated since this is called frequently

	// Don't waste time if we're not tracing at this point
	pcmk__log_else(LOG_TRACE, return);

	g_hash_table_iter_init(&iter, nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	score2char_stack(node->weight, score, sizeof(score));
	if (rsc) {
	qb_log_from_external_source(function, file,
	"%s: %s allocation score on %s: %s",
	LOG_TRACE, line, 0,
	comment, rsc->id,
	node->details->uname, score);
	} else {
	qb_log_from_external_source(function, file, "%s: %s = %s",
	LOG_TRACE, line, 0,
	comment, node->details->uname,
	score);
	}
	}
	}

	/*!
	* \internal
	* \brief Log or output node weights
	*
	* \param[in] file Caller's filename
	* \param[in] function Caller's function name
	* \param[in] line Caller's line number
	* \param[in] to_log Log if true, otherwise output
	* \param[in] rsc Use allowed nodes for this resource
	* \param[in] comment Text description to prefix lines with
	* \param[in] nodes If rsc is not specified, use these nodes
	*/
	void
	pe__show_node_weights_as(const char file, const char function, int line,
	bool to_log, pe_resource_t rsc, const char comment,
	GHashTable *nodes)
	{
	if (rsc != NULL) {
	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	// Don't show allocation scores for orphans
	return;
	}
	nodes = rsc->allowed_nodes;
	}
	if (nodes == NULL) {
	// Nothing to show
	return;
	}

	if (to_log) {
	pe__log_node_weights(file, function, line, rsc, comment, nodes);
	} else {
	pe__output_node_weights(rsc, comment, nodes);
	}

	// If this resource has children, repeat recursively for each
	if (rsc && rsc->children) {
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	pe__show_node_weights_as(file, function, line, to_log, child,
	comment, nodes);
	}
	}
	}

	static void
	append_dump_text(gpointer key, gpointer value, gpointer user_data)
	{
	char **dump_text = user_data;
	char *new_text = crm_strdup_printf("%s %s=%s",
	dump_text, (char )key, (char *)value);

	free(*dump_text);
	*dump_text = new_text;
	}

	void
	dump_node_capacity(int level, const char comment, pe_node_t node)
	{
	char *dump_text = crm_strdup_printf("%s: %s capacity:",
	comment, node->details->uname);

	g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);

	if (level == LOG_STDOUT) {
	fprintf(stdout, "%s\n", dump_text);
	} else {
	crm_trace("%s", dump_text);
	}

	free(dump_text);
	}

	void
	dump_rsc_utilization(int level, const char comment, pe_resource_t rsc, pe_node_t * node)
	{
	char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
	comment, rsc->id, node->details->uname);

	g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
	switch (level) {
	case LOG_STDOUT:
	fprintf(stdout, "%s\n", dump_text);
	break;
	case LOG_NEVER:
	break;
	default:
	crm_trace("%s", dump_text);
	}
	free(dump_text);
	}

	gint
	sort_rsc_index(gconstpointer a, gconstpointer b)
	{
	const pe_resource_t resource1 = (const pe_resource_t )a;
	const pe_resource_t resource2 = (const pe_resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->sort_index > resource2->sort_index) {
	return -1;
	}

	if (resource1->sort_index < resource2->sort_index) {
	return 1;
	}

	return 0;
	}

	gint
	sort_rsc_priority(gconstpointer a, gconstpointer b)
	{
	const pe_resource_t resource1 = (const pe_resource_t )a;
	const pe_resource_t resource2 = (const pe_resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->priority > resource2->priority) {
	return -1;
	}

	if (resource1->priority < resource2->priority) {
	return 1;
	}

	return 0;
	}

	static enum pe_quorum_policy
	effective_quorum_policy(pe_resource_t rsc, pe_working_set_t data_set)
	{
	enum pe_quorum_policy policy = data_set->no_quorum_policy;

	if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	policy = no_quorum_ignore;

	} else if (data_set->no_quorum_policy == no_quorum_demote) {
	switch (rsc->role) {
	case RSC_ROLE_MASTER:
	case RSC_ROLE_SLAVE:
	if (rsc->next_role > RSC_ROLE_SLAVE) {
	rsc->next_role = RSC_ROLE_SLAVE;
	}
	policy = no_quorum_ignore;
	break;
	default:
	policy = no_quorum_stop;
	break;
	}
	}
	return policy;
	}

	pe_action_t *
	custom_action(pe_resource_t * rsc, char key, const char task,
	pe_node_t * on_node, gboolean optional, gboolean save_action,
	pe_working_set_t * data_set)
	{
	pe_action_t *action = NULL;
	GListPtr possible_matches = NULL;

	CRM_CHECK(key != NULL, return NULL);
	CRM_CHECK(task != NULL, free(key); return NULL);

	if (save_action && rsc != NULL) {
	possible_matches = find_actions(rsc->actions, key, on_node);
	} else if(save_action) {
	#if 0
	action = g_hash_table_lookup(data_set->singletons, key);
	#else
	/* More expensive but takes 'node' into account */
	possible_matches = find_actions(data_set->actions, key, on_node);
	#endif
	}

	if(data_set->singletons == NULL) {
	data_set->singletons = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
	}

	if (possible_matches != NULL) {
	if (pcmk__list_of_multiple(possible_matches)) {
	pe_warn("Action %s for %s on %s exists %d times",
	task, rsc ? rsc->id : "<NULL>",
	on_node ? on_node->details->uname : "<NULL>", g_list_length(possible_matches));
	}

	action = g_list_nth_data(possible_matches, 0);
	- pe_rsc_trace(rsc, "Found existing action %d (%s) for %s (%s) on %s",
	- action->id, action->uuid,
	- (rsc? rsc->id : "no resource"), task,
	+ pe_rsc_trace(rsc, "Found action %d: %s for %s (%s) on %s",
	+ action->id, task, (rsc? rsc->id : "no resource"),
	+ action->uuid,
	(on_node? on_node->details->uname : "no node"));
	g_list_free(possible_matches);
	}

	if (action == NULL) {
	if (save_action) {
	- pe_rsc_trace(rsc, "Creating %s action %d: %s for %s (%s) on %s",
	- (optional? "optional" : "mandatory"),
	- data_set->action_id, key,
	- (rsc? rsc->id : "no resource"), task,
	+ pe_rsc_trace(rsc, "Creating action %d (%s): %s for %s (%s) on %s",
	+ data_set->action_id,
	+ (optional? "optional" : "required"),
	+ task, (rsc? rsc->id : "no resource"), key,
	(on_node? on_node->details->uname : "no node"));
	}

	action = calloc(1, sizeof(pe_action_t));
	if (save_action) {
	action->id = data_set->action_id++;
	} else {
	action->id = 0;
	}
	action->rsc = rsc;
	- CRM_ASSERT(task != NULL);
	action->task = strdup(task);
	if (on_node) {
	action->node = pe__copy_node(on_node);
	}
	action->uuid = strdup(key);

	if (pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	// Resource history deletion for a node can be done on the DC
	pe__set_action_flags(action, pe_action_dc);
	}

	pe__set_action_flags(action, pe_action_runnable);
	if (optional) {
	pe__set_action_flags(action, pe_action_optional);
	} else {
	pe__clear_action_flags(action, pe_action_optional);
	}

	action->extra = crm_str_table_new();
	action->meta = crm_str_table_new();

	if (save_action) {
	data_set->actions = g_list_prepend(data_set->actions, action);
	if(rsc == NULL) {
	g_hash_table_insert(data_set->singletons, action->uuid, action);
	}
	}

	if (rsc != NULL) {
	guint interval_ms = 0;

	action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE);
	parse_op_key(key, NULL, NULL, &interval_ms);

	unpack_operation(action, action->op_entry, rsc->container, data_set,
	interval_ms);

	if (save_action) {
	rsc->actions = g_list_prepend(rsc->actions, action);
	}
	}
	-
	- if (save_action) {
	- pe_rsc_trace(rsc, "Action %d created", action->id);
	- }
	}

	if (!optional && pcmk_is_set(action->flags, pe_action_optional)) {
	- pe_rsc_trace(rsc, "Unset optional on action %d", action->id);
	pe__clear_action_flags(action, pe_action_optional);
	}

	if (rsc != NULL) {
	enum action_tasks a_task = text2task(action->task);
	enum pe_quorum_policy quorum_policy = effective_quorum_policy(rsc, data_set);
	int warn_level = LOG_TRACE;

	if (save_action) {
	warn_level = LOG_WARNING;
	}

	if (!pcmk_is_set(action->flags, pe_action_have_node_attrs)
	&& action->node != NULL && action->op_entry != NULL) {
	pe_rule_eval_data_t rule_data = {
	.node_hash = action->node->details->attrs,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe__set_action_flags(action, pe_action_have_node_attrs);
	pe__unpack_dataset_nvpairs(action->op_entry, XML_TAG_ATTR_SETS,
	&rule_data, action->extra, NULL,
	FALSE, data_set);
	}

	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	/* leave untouched */

	} else if (action->node == NULL) {
	- pe_rsc_trace(rsc, "Unset runnable on %s", action->uuid);
	+ pe_rsc_trace(rsc, "%s is unrunnable (unallocated)",
	+ action->uuid);
	pe__clear_action_flags(action, pe_action_runnable);

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)
	&& g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS) == NULL) {
	- crm_debug("Action %s (unmanaged)", action->uuid);
	- pe_rsc_trace(rsc, "Set optional on %s", action->uuid);
	+ pe_rsc_debug(rsc, "%s on %s is optional (%s is unmanaged)",
	+ action->uuid, action->node->details->uname, rsc->id);
	pe__set_action_flags(action, pe_action_optional);
	//pe__clear_action_flags(action, pe_action_runnable);

	} else if (!pcmk_is_set(action->flags, pe_action_dc)
	&& !(action->node->details->online)
	&& (!pe__is_guest_node(action->node)
	\|\| action->node->details->remote_requires_reset)) {
	pe__clear_action_flags(action, pe_action_runnable);
	- do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)",
	+ do_crm_log(warn_level,
	+ "%s on %s is unrunnable (node is offline)",
	action->uuid, action->node->details->uname);
	if (pcmk_is_set(action->rsc->flags, pe_rsc_managed)
	&& save_action && a_task == stop_rsc
	&& action->node->details->unclean == FALSE) {
	pe_fence_node(data_set, action->node, "resource actions are unrunnable", FALSE);
	}

	} else if (!pcmk_is_set(action->flags, pe_action_dc)
	&& action->node->details->pending) {
	pe__clear_action_flags(action, pe_action_runnable);
	- do_crm_log(warn_level, "Action %s on %s is unrunnable (pending)",
	+ do_crm_log(warn_level,
	+ "Action %s on %s is unrunnable (node is pending)",
	action->uuid, action->node->details->uname);

	} else if (action->needs == rsc_req_nothing) {
	- pe_rsc_trace(rsc, "Action %s does not require anything", action->uuid);
	pe_action_set_reason(action, NULL, TRUE);
	if (pe__is_guest_node(action->node)
	&& !pe_can_fence(data_set, action->node)) {
	/* An action that requires nothing usually does not require any
	* fencing in order to be runnable. However, there is an
	* exception: an action cannot be completed if it is on a guest
	* node whose host is unclean and cannot be fenced.
	*/
	+ pe_rsc_debug(rsc, "%s on %s is unrunnable "
	+ "(node's host cannot be fenced)",
	+ action->uuid, action->node->details->uname);
	pe__clear_action_flags(action, pe_action_runnable);
	- crm_debug("%s\t%s (cancelled : host cannot be fenced)",
	- action->node->details->uname, action->uuid);
	} else {
	+ pe_rsc_trace(rsc, "%s on %s does not require fencing or quorum",
	+ action->uuid, action->node->details->uname);
	pe__set_action_flags(action, pe_action_runnable);
	}
	#if 0
	/*
	* No point checking this
	* - if we don't have quorum we can't stonith anyway
	*/
	} else if (action->needs == rsc_req_stonith) {
	crm_trace("Action %s requires only stonith", action->uuid);
	action->runnable = TRUE;
	#endif
	} else if (quorum_policy == no_quorum_stop) {
	+ pe_rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
	+ action->uuid, action->node->details->uname);
	pe_action_set_flag_reason(__func__, __LINE__, action, NULL,
	"no quorum", pe_action_runnable, TRUE);
	- crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid);

	} else if (quorum_policy == no_quorum_freeze) {
	- pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role));
	if (rsc->fns->active(rsc, TRUE) == FALSE \|\| rsc->next_role > rsc->role) {
	+ pe_rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
	+ action->uuid, action->node->details->uname);
	pe_action_set_flag_reason(__func__, __LINE__, action, NULL,
	"quorum freeze", pe_action_runnable,
	TRUE);
	- pe_rsc_debug(rsc, "%s\t%s (cancelled : quorum freeze)",
	- action->node->details->uname, action->uuid);
	}

	- } else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	- pe_rsc_trace(rsc, "Action %s is runnable", action->uuid);
	+ } else {
	//pe_action_set_reason(action, NULL, TRUE);
	pe__set_action_flags(action, pe_action_runnable);
	}

	if (save_action) {
	switch (a_task) {
	case stop_rsc:
	pe__set_resource_flags(rsc, pe_rsc_stopping);
	break;
	case start_rsc:
	pe__clear_resource_flags(rsc, pe_rsc_starting);
	if (pcmk_is_set(action->flags, pe_action_runnable)) {
	pe__set_resource_flags(rsc, pe_rsc_starting);
	}
	break;
	default:
	break;
	}
	}
	}

	free(key);
	return action;
	}

	static bool
	valid_stop_on_fail(const char *value)
	{
	return !pcmk__strcase_any_of(value, "standby", "demote", "stop", NULL);
	}

	static const char *
	unpack_operation_on_fail(pe_action_t * action)
	{

	const char *name = NULL;
	const char *role = NULL;
	const char *on_fail = NULL;
	const char *interval_spec = NULL;
	const char *enabled = NULL;
	const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);

	if (pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)
	&& !valid_stop_on_fail(value)) {

	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
	"action to default value because '%s' is not "
	"allowed for stop", action->rsc->id, value);
	return NULL;

	} else if (pcmk__str_eq(action->task, CRMD_ACTION_DEMOTE, pcmk__str_casei) && !value) {
	/* demote on_fail defaults to master monitor value if present */
	xmlNode *operation = NULL;

	CRM_CHECK(action->rsc != NULL, return NULL);

	for (operation = __xml_first_child_element(action->rsc->ops_xml);
	operation && !value; operation = __xml_next_element(operation)) {

	if (!pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	continue;
	}
	name = crm_element_value(operation, "name");
	role = crm_element_value(operation, "role");
	on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
	enabled = crm_element_value(operation, "enabled");
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (!on_fail) {
	continue;
	} else if (enabled && !crm_is_true(enabled)) {
	continue;
	} else if (!pcmk__str_eq(name, "monitor", pcmk__str_casei) \|\| !pcmk__str_eq(role, "Master", pcmk__str_casei)) {
	continue;
	} else if (crm_parse_interval_spec(interval_spec) == 0) {
	continue;
	} else if (pcmk__str_eq(on_fail, "demote", pcmk__str_casei)) {
	continue;
	}

	value = on_fail;
	}
	} else if (pcmk__str_eq(action->task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	value = "ignore";

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	name = crm_element_value(action->op_entry, "name");
	role = crm_element_value(action->op_entry, "role");
	on_fail = crm_element_value(action->op_entry, XML_OP_ATTR_ON_FAIL);
	interval_spec = crm_element_value(action->op_entry,
	XML_LRM_ATTR_INTERVAL);

	if (!pcmk__str_eq(name, CRMD_ACTION_PROMOTE, pcmk__str_casei)
	&& (!pcmk__str_eq(name, CRMD_ACTION_STATUS, pcmk__str_casei)
	\|\| !pcmk__str_eq(role, "Master", pcmk__str_casei)
	\|\| (crm_parse_interval_spec(interval_spec) == 0))) {
	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s "
	"action to default value because 'demote' is not "
	"allowed for it", action->rsc->id, name);
	return NULL;
	}
	}

	return value;
	}

	static xmlNode *
	find_min_interval_mon(pe_resource_t * rsc, gboolean include_disabled)
	{
	guint interval_ms = 0;
	guint min_interval_ms = G_MAXUINT;
	const char *name = NULL;
	const char *value = NULL;
	const char *interval_spec = NULL;
	xmlNode *op = NULL;
	xmlNode *operation = NULL;

	for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
	operation = __xml_next_element(operation)) {

	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	name = crm_element_value(operation, "name");
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	value = crm_element_value(operation, "enabled");
	if (!include_disabled && value && crm_is_true(value) == FALSE) {
	continue;
	}

	if (!pcmk__str_eq(name, RSC_STATUS, pcmk__str_casei)) {
	continue;
	}

	interval_ms = crm_parse_interval_spec(interval_spec);

	if (interval_ms && (interval_ms < min_interval_ms)) {
	min_interval_ms = interval_ms;
	op = operation;
	}
	}
	}

	return op;
	}

	static int
	unpack_start_delay(const char value, GHashTable meta)
	{
	int start_delay = 0;

	if (value != NULL) {
	start_delay = crm_get_msec(value);

	if (start_delay < 0) {
	start_delay = 0;
	}

	if (meta) {
	g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY), crm_itoa(start_delay));
	}
	}

	return start_delay;
	}

	// true if value contains valid, non-NULL interval origin for recurring op
	static bool
	unpack_interval_origin(const char value, xmlNode xml_obj, guint interval_ms,
	crm_time_t now, long long start_delay)
	{
	long long result = 0;
	guint interval_sec = interval_ms / 1000;
	crm_time_t *origin = NULL;

	// Ignore unspecified values and non-recurring operations
	if ((value == NULL) \|\| (interval_ms == 0) \|\| (now == NULL)) {
	return false;
	}

	// Parse interval origin from text
	origin = crm_time_new(value);
	if (origin == NULL) {
	pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
	"'%s' because '%s' is not valid",
	(ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
	return false;
	}

	// Get seconds since origin (negative if origin is in the future)
	result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
	crm_time_free(origin);

	// Calculate seconds from closest interval to now
	result = result % interval_sec;

	// Calculate seconds remaining until next interval
	result = ((result <= 0)? 0 : interval_sec) - result;
	crm_info("Calculated a start delay of %llds for operation '%s'",
	result,
	(ID(xml_obj)? ID(xml_obj) : "(unspecified)"));

	if (start_delay != NULL) {
	start_delay = result 1000; // milliseconds
	}
	return true;
	}

	static int
	unpack_timeout(const char *value)
	{
	int timeout_ms = crm_get_msec(value);

	if (timeout_ms < 0) {
	timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
	}
	return timeout_ms;
	}

	int
	pe_get_configured_timeout(pe_resource_t rsc, const char action, pe_working_set_t *data_set)
	{
	xmlNode *child = NULL;
	GHashTable *action_meta = NULL;
	const char *timeout_spec = NULL;
	int timeout_ms = 0;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	child != NULL; child = crm_next_same_xml(child)) {
	if (pcmk__str_eq(action, crm_element_value(child, XML_NVPAIR_ATTR_NAME),
	pcmk__str_casei)) {
	timeout_spec = crm_element_value(child, XML_ATTR_TIMEOUT);
	break;
	}
	}

	if (timeout_spec == NULL && data_set->op_defaults) {
	action_meta = crm_str_table_new();
	pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS,
	&rule_data, action_meta, NULL, FALSE, data_set);
	timeout_spec = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
	}

	// @TODO check meta-attributes (including versioned meta-attributes)
	// @TODO maybe use min-interval monitor timeout as default for monitors

	timeout_ms = crm_get_msec(timeout_spec);
	if (timeout_ms < 0) {
	timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
	}

	if (action_meta != NULL) {
	g_hash_table_destroy(action_meta);
	}
	return timeout_ms;
	}

	#if ENABLE_VERSIONED_ATTRS
	static void
	unpack_versioned_meta(xmlNode versioned_meta, xmlNode xml_obj,
	guint interval_ms, crm_time_t *now)
	{
	xmlNode *attrs = NULL;
	xmlNode *attr = NULL;

	for (attrs = __xml_first_child_element(versioned_meta); attrs != NULL;
	attrs = __xml_next_element(attrs)) {

	for (attr = __xml_first_child_element(attrs); attr != NULL;
	attr = __xml_next_element(attr)) {

	const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
	const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);

	if (pcmk__str_eq(name, XML_OP_ATTR_START_DELAY, pcmk__str_casei)) {
	int start_delay = unpack_start_delay(value, NULL);

	crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
	} else if (pcmk__str_eq(name, XML_OP_ATTR_ORIGIN, pcmk__str_casei)) {
	long long start_delay = 0;

	if (unpack_interval_origin(value, xml_obj, interval_ms, now,
	&start_delay)) {
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME,
	XML_OP_ATTR_START_DELAY);
	crm_xml_add_ll(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
	}
	} else if (pcmk__str_eq(name, XML_ATTR_TIMEOUT, pcmk__str_casei)) {
	int timeout_ms = unpack_timeout(value);

	crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, timeout_ms);
	}
	}
	}
	}
	#endif

	/*!
	* \brief Unpack operation XML into an action structure
	*
	* Unpack an operation's meta-attributes (normalizing the interval, timeout,
	* and start delay values as integer milliseconds), requirements, and
	* failure policy.
	*
	* \param[in,out] action Action to unpack into
	* \param[in] xml_obj Operation XML (or NULL if all defaults)
	* \param[in] container Resource that contains affected resource, if any
	* \param[in] data_set Cluster state
	* \param[in] interval_ms How frequently to perform the operation
	*/
	static void
	unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
	pe_working_set_t * data_set, guint interval_ms)
	{
	int timeout_ms = 0;
	const char *value = NULL;
	#if ENABLE_VERSIONED_ATTRS
	pe_rsc_action_details_t *rsc_details = NULL;
	#endif

	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(action->rsc->xml, XML_EXPR_ATTR_TYPE)
	};

	pe_op_eval_data_t op_rule_data = {
	.op_name = action->task,
	.interval = interval_ms
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = &op_rule_data
	};

	CRM_CHECK(action && action->rsc, return);

	// Cluster-wide <op_defaults> <meta_attributes>
	pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, &rule_data,
	action->meta, NULL, FALSE, data_set);

	// Determine probe default timeout differently
	if (pcmk__str_eq(action->task, RSC_STATUS, pcmk__str_casei)
	&& (interval_ms == 0)) {
	xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE);

	if (min_interval_mon) {
	value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT);
	if (value) {
	crm_trace("\t%s: Setting default timeout to minimum-interval "
	"monitor's timeout '%s'", action->uuid, value);
	g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
	strdup(value));
	}
	}
	}

	if (xml_obj) {
	xmlAttrPtr xIter = NULL;

	// <op> <meta_attributes> take precedence over defaults
	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_META_SETS, &rule_data,
	action->meta, NULL, TRUE, data_set);

	#if ENABLE_VERSIONED_ATTRS
	rsc_details = pe_rsc_action_details(action);

	pe_eval_versioned_attributes(data_set->input, xml_obj,
	XML_TAG_ATTR_SETS, &rule_data,
	rsc_details->versioned_parameters,
	NULL);
	pe_eval_versioned_attributes(data_set->input, xml_obj,
	XML_TAG_META_SETS, &rule_data,
	rsc_details->versioned_meta,
	NULL);
	#endif

	/* Anything set as an <op> XML property has highest precedence.
	* This ensures we use the name and interval from the <op> tag.
	*/
	for (xIter = xml_obj->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = crm_element_value(xml_obj, prop_name);

	g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value));
	}
	}

	g_hash_table_remove(action->meta, "id");

	// Normalize interval to milliseconds
	if (interval_ms > 0) {
	g_hash_table_replace(action->meta, strdup(XML_LRM_ATTR_INTERVAL),
	crm_strdup_printf("%u", interval_ms));
	} else {
	g_hash_table_remove(action->meta, XML_LRM_ATTR_INTERVAL);
	}

	/*
	* Timeout order of precedence:
	* 1. pcmk_monitor_timeout (if rsc has pcmk_ra_cap_fence_params
	* and task is start or a probe; pcmk_monitor_timeout works
	* by default for a recurring monitor)
	* 2. explicit op timeout on the primitive
	* 3. default op timeout
	* a. if probe, then min-interval monitor's timeout
	* b. else, in XML_CIB_TAG_OPCONFIG
	* 4. CRM_DEFAULT_OP_TIMEOUT_S
	*
	* #1 overrides general rule of <op> XML property having highest
	* precedence.
	*/
	if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
	pcmk_ra_cap_fence_params)
	&& (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
	\|\| (pcmk__str_eq(action->task, RSC_STATUS, pcmk__str_casei)
	&& (interval_ms == 0)))
	&& action->rsc->parameters) {

	value = g_hash_table_lookup(action->rsc->parameters,
	"pcmk_monitor_timeout");

	if (value) {
	crm_trace("\t%s: Setting timeout to pcmk_monitor_timeout '%s', "
	"overriding default", action->uuid, value);
	g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
	strdup(value));
	}
	}

	// Normalize timeout to positive milliseconds
	value = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
	timeout_ms = unpack_timeout(value);
	g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
	crm_itoa(timeout_ms));

	if (!pcmk__strcase_any_of(action->task, RSC_START, RSC_PROMOTE, NULL)) {
	action->needs = rsc_req_nothing;
	- value = "nothing (not start/promote)";
	+ value = "nothing (not start or promote)";

	} else if (pcmk_is_set(action->rsc->flags, pe_rsc_needs_fencing)) {
	action->needs = rsc_req_stonith;
	- value = "fencing (resource)";
	+ value = "fencing";

	} else if (pcmk_is_set(action->rsc->flags, pe_rsc_needs_quorum)) {
	action->needs = rsc_req_quorum;
	- value = "quorum (resource)";
	+ value = "quorum";

	} else {
	action->needs = rsc_req_nothing;
	- value = "nothing (resource)";
	+ value = "nothing";
	}
	-
	- pe_rsc_trace(action->rsc, "\tAction %s requires: %s", action->uuid, value);
	+ pe_rsc_trace(action->rsc, "%s requires %s", action->uuid, value);

	value = unpack_operation_on_fail(action);

	if (value == NULL) {

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	action->on_fail = action_fail_block;
	g_hash_table_insert(action->meta, strdup(XML_OP_ATTR_ON_FAIL), strdup("block"));
	value = "block"; // The above could destroy the original string

	} else if (pcmk__str_eq(value, "fence", pcmk__str_casei)) {
	action->on_fail = action_fail_fence;
	value = "node fencing";

	if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
	"operation '%s' to 'stop' because 'fence' is not "
	"valid when fencing is disabled", action->uuid);
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop resource";
	}

	} else if (pcmk__str_eq(value, "standby", pcmk__str_casei)) {
	action->on_fail = action_fail_standby;
	value = "node standby";

	} else if (pcmk__strcase_any_of(value, "ignore", "nothing", NULL)) {
	action->on_fail = action_fail_ignore;
	value = "ignore";

	} else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
	action->on_fail = action_fail_migrate;
	value = "force migration";

	} else if (pcmk__str_eq(value, "stop", pcmk__str_casei)) {
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop resource";

	} else if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	action->on_fail = action_fail_recover;
	value = "restart (and possibly migrate)";

	} else if (pcmk__str_eq(value, "restart-container", pcmk__str_casei)) {
	if (container) {
	action->on_fail = action_fail_restart_container;
	value = "restart container (and possibly migrate)";

	} else {
	value = NULL;
	}

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	action->on_fail = action_fail_demote;
	value = "demote instance";

	} else {
	pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
	value = NULL;
	}

	/* defaults */
	if (value == NULL && container) {
	action->on_fail = action_fail_restart_container;
	value = "restart container (and possibly migrate) (default)";

	/* For remote nodes, ensure that any failure that results in dropping an
	* active connection to the node results in fencing of the node.
	*
	* There are only two action failures that don't result in fencing.
	* 1. probes - probe failures are expected.
	* 2. start - a start failure indicates that an active connection does not already
	* exist. The user can set op on-fail=fence if they really want to fence start
	* failures. */
	} else if (((value == NULL) \|\| !pcmk_is_set(action->rsc->flags, pe_rsc_managed))
	&& pe__resource_is_remote_conn(action->rsc, data_set)
	&& !(pcmk__str_eq(action->task, CRMD_ACTION_STATUS, pcmk__str_casei)
	&& (interval_ms == 0))
	&& !pcmk__str_eq(action->task, CRMD_ACTION_START, pcmk__str_casei)) {

	if (!pcmk_is_set(action->rsc->flags, pe_rsc_managed)) {
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop unmanaged remote node (enforcing default)";

	} else {
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	value = "fence remote node (default)";
	} else {
	value = "recover remote node connection (default)";
	}

	if (action->rsc->remote_reconnect_ms) {
	action->fail_role = RSC_ROLE_STOPPED;
	}
	action->on_fail = action_fail_reset_remote;
	}

	} else if (value == NULL && pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)) {
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	action->on_fail = action_fail_fence;
	value = "resource fence (default)";

	} else {
	action->on_fail = action_fail_block;
	value = "resource block (default)";
	}

	} else if (value == NULL) {
	action->on_fail = action_fail_recover;
	value = "restart (and possibly migrate) (default)";
	}

	- pe_rsc_trace(action->rsc, "\t%s failure handling: %s", action->task, value);
	+ pe_rsc_trace(action->rsc, "%s failure handling: %s",
	+ action->uuid, value);

	value = NULL;
	if (xml_obj != NULL) {
	value = g_hash_table_lookup(action->meta, "role_after_failure");
	if (value) {
	pe_warn_once(pe_wo_role_after,
	"Support for role_after_failure is deprecated and will be removed in a future release");
	}
	}
	if (value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) {
	action->fail_role = text2role(value);
	}
	/* defaults */
	if (action->fail_role == RSC_ROLE_UNKNOWN) {
	if (pcmk__str_eq(action->task, CRMD_ACTION_PROMOTE, pcmk__str_casei)) {
	action->fail_role = RSC_ROLE_SLAVE;
	} else {
	action->fail_role = RSC_ROLE_STARTED;
	}
	}
	- pe_rsc_trace(action->rsc, "\t%s failure results in: %s", action->task,
	- role2text(action->fail_role));
	+ pe_rsc_trace(action->rsc, "%s failure results in: %s",
	+ action->uuid, role2text(action->fail_role));

	value = g_hash_table_lookup(action->meta, XML_OP_ATTR_START_DELAY);
	if (value) {
	unpack_start_delay(value, action->meta);
	} else {
	long long start_delay = 0;

	value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN);
	if (unpack_interval_origin(value, xml_obj, interval_ms, data_set->now,
	&start_delay)) {
	g_hash_table_replace(action->meta, strdup(XML_OP_ATTR_START_DELAY),
	crm_strdup_printf("%lld", start_delay));
	}
	}

	#if ENABLE_VERSIONED_ATTRS
	unpack_versioned_meta(rsc_details->versioned_meta, xml_obj, interval_ms,
	data_set->now);
	#endif
	}

	static xmlNode *
	find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key, gboolean include_disabled)
	{
	guint interval_ms = 0;
	gboolean do_retry = TRUE;
	char *local_key = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *interval_spec = NULL;
	char *match_key = NULL;
	xmlNode *op = NULL;
	xmlNode *operation = NULL;

	retry:
	for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
	operation = __xml_next_element(operation)) {
	if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
	name = crm_element_value(operation, "name");
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	value = crm_element_value(operation, "enabled");
	if (!include_disabled && value && crm_is_true(value) == FALSE) {
	continue;
	}

	interval_ms = crm_parse_interval_spec(interval_spec);
	match_key = pcmk__op_key(rsc->id, name, interval_ms);
	if (pcmk__str_eq(key, match_key, pcmk__str_casei)) {
	op = operation;
	}
	free(match_key);

	if (rsc->clone_name) {
	match_key = pcmk__op_key(rsc->clone_name, name, interval_ms);
	if (pcmk__str_eq(key, match_key, pcmk__str_casei)) {
	op = operation;
	}
	free(match_key);
	}

	if (op != NULL) {
	free(local_key);
	return op;
	}
	}
	}

	free(local_key);
	if (do_retry == FALSE) {
	return NULL;
	}

	do_retry = FALSE;
	if (strstr(key, CRMD_ACTION_MIGRATE) \|\| strstr(key, CRMD_ACTION_MIGRATED)) {
	local_key = pcmk__op_key(rsc->id, "migrate", 0);
	key = local_key;
	goto retry;

	} else if (strstr(key, "_notify_")) {
	local_key = pcmk__op_key(rsc->id, "notify", 0);
	key = local_key;
	goto retry;
	}

	return NULL;
	}

	xmlNode *
	find_rsc_op_entry(pe_resource_t * rsc, const char *key)
	{
	return find_rsc_op_entry_helper(rsc, key, FALSE);
	}

	void
	print_node(const char pre_text, pe_node_t node, gboolean details)
	{
	if (node == NULL) {
	crm_trace("%s%s: <NULL>", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
	return;
	}

	CRM_ASSERT(node->details);
	crm_trace("%s%s%sNode %s: (weight=%d, fixed=%s)",
	pre_text == NULL ? "" : pre_text,
	pre_text == NULL ? "" : ": ",
	node->details->online ? "" : "Unavailable/Unclean ",
	node->details->uname, node->weight, node->fixed ? "True" : "False");

	if (details) {
	int log_level = LOG_TRACE;

	char *pe_mutable = strdup("\t\t");
	GListPtr gIter = node->details->running_rsc;

	crm_trace("\t\t===Node Attributes");
	g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable);
	free(pe_mutable);

	crm_trace("\t\t=== Resources");

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	rsc->fns->print(rsc, "\t\t", pe_print_log\|pe_print_pending,
	&log_level);
	}
	}
	}

	/*
	* Used by the HashTable for-loop
	*/
	void
	print_str_str(gpointer key, gpointer value, gpointer user_data)
	{
	crm_trace("%s%s %s ==> %s",
	user_data == NULL ? "" : (char *)user_data,
	user_data == NULL ? "" : ": ", (char )key, (char )value);
	}

	void
	pe_free_action(pe_action_t * action)
	{
	if (action == NULL) {
	return;
	}
	g_list_free_full(action->actions_before, free); /* pe_action_wrapper_t* */
	g_list_free_full(action->actions_after, free); /* pe_action_wrapper_t* */
	if (action->extra) {
	g_hash_table_destroy(action->extra);
	}
	if (action->meta) {
	g_hash_table_destroy(action->meta);
	}
	#if ENABLE_VERSIONED_ATTRS
	if (action->rsc) {
	pe_free_rsc_action_details(action);
	}
	#endif
	free(action->cancel_task);
	free(action->reason);
	free(action->task);
	free(action->uuid);
	free(action->node);
	free(action);
	}

	GListPtr
	find_recurring_actions(GListPtr input, pe_node_t * not_on_node)
	{
	const char *value = NULL;
	GListPtr result = NULL;
	GListPtr gIter = input;

	CRM_CHECK(input != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL_MS);
	if (value == NULL) {
	/* skip */
	} else if (pcmk__str_eq(value, "0", pcmk__str_casei)) {
	/* skip */
	} else if (pcmk__str_eq(CRMD_ACTION_CANCEL, action->task, pcmk__str_casei)) {
	/* skip */
	} else if (not_on_node == NULL) {
	crm_trace("(null) Found: %s", action->uuid);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	/* skip */
	} else if (action->node->details != not_on_node->details) {
	crm_trace("Found: %s", action->uuid);
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	enum action_tasks
	get_complex_task(pe_resource_t * rsc, const char *name, gboolean allow_non_atomic)
	{
	enum action_tasks task = text2task(name);

	if (rsc == NULL) {
	return task;

	} else if (allow_non_atomic == FALSE \|\| rsc->variant == pe_native) {
	switch (task) {
	case stopped_rsc:
	case started_rsc:
	case action_demoted:
	case action_promoted:
	crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id);
	return task - 1;
	default:
	break;
	}
	}
	return task;
	}

	pe_action_t *
	find_first_action(GListPtr input, const char uuid, const char task, pe_node_t * on_node)
	{
	GListPtr gIter = NULL;

	CRM_CHECK(uuid \|\| task, return NULL);

	for (gIter = input; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	return action;

	} else if (action->node == NULL) {
	continue;

	} else if (on_node->details == action->node->details) {
	return action;
	}
	}

	return NULL;
	}

	GListPtr
	find_actions(GListPtr input, const char key, const pe_node_t on_node)
	{
	GListPtr gIter = input;
	GListPtr result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
	crm_trace("%s does not match action %s", key, action->uuid);
	continue;

	} else if (on_node == NULL) {
	crm_trace("Action %s matches (ignoring node)", key);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	crm_trace("Action %s matches (unallocated, assigning to %s)",
	key, on_node->details->uname);

	action->node = pe__copy_node(on_node);
	result = g_list_prepend(result, action);

	} else if (on_node->details == action->node->details) {
	crm_trace("Action %s on %s matches", key, on_node->details->uname);
	result = g_list_prepend(result, action);

	} else {
	crm_trace("Action %s on node %s does not match requested node %s",
	key, action->node->details->uname,
	on_node->details->uname);
	}
	}

	return result;
	}

	GList *
	find_actions_exact(GList input, const char key, const pe_node_t *on_node)
	{
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	if (on_node == NULL) {
	crm_trace("Not searching for action %s because node not specified",
	key);
	return NULL;
	}

	for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (action->node == NULL) {
	crm_trace("Skipping comparison of %s vs action %s without node",
	key, action->uuid);

	} else if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
	crm_trace("Desired action %s doesn't match %s", key, action->uuid);

	} else if (!pcmk__str_eq(on_node->details->id, action->node->details->id, pcmk__str_casei)) {
	crm_trace("Action %s desired node ID %s doesn't match %s",
	key, on_node->details->id, action->node->details->id);

	} else {
	crm_trace("Action %s matches", key);
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	/*!
	* \brief Find all actions of given type for a resource
	*
	* \param[in] rsc Resource to search
	* \param[in] node Find only actions scheduled on this node
	* \param[in] task Action name to search for
	* \param[in] require_node If TRUE, NULL node or action node will not match
	*
	* \return List of actions found (or NULL if none)
	* \note If node is not NULL and require_node is FALSE, matching actions
	* without a node will be assigned to node.
	*/
	GList *
	pe__resource_actions(const pe_resource_t rsc, const pe_node_t node,
	const char *task, bool require_node)
	{
	GList *result = NULL;
	char *key = pcmk__op_key(rsc->id, task, 0);

	if (require_node) {
	result = find_actions_exact(rsc->actions, key, node);
	} else {
	result = find_actions(rsc->actions, key, node);
	}
	free(key);
	return result;
	}

	static void
	resource_node_score(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag)
	{
	pe_node_t *match = NULL;

	if ((rsc->exclusive_discover \|\| (node->rsc_discover_mode == pe_discover_never))
	&& pcmk__str_eq(tag, "symmetric_default", pcmk__str_casei)) {
	/* This string comparision may be fragile, but exclusive resources and
	* exclusive nodes should not have the symmetric_default constraint
	* applied to them.
	*/
	return;

	} else if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	resource_node_score(child_rsc, node, score, tag);
	}
	}

	pe_rsc_trace(rsc, "Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score);
	match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match == NULL) {
	match = pe__copy_node(node);
	g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
	}
	match->weight = pe__add_scores(match->weight, score);
	}

	void
	resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
	pe_working_set_t * data_set)
	{
	if (node != NULL) {
	resource_node_score(rsc, node, score, tag);

	} else if (data_set != NULL) {
	GListPtr gIter = data_set->nodes;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node_iter = (pe_node_t ) gIter->data;

	resource_node_score(rsc, node_iter, score, tag);
	}

	} else {
	GHashTableIter iter;
	pe_node_t *node_iter = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
	resource_node_score(rsc, node_iter, score, tag);
	}
	}

	if (node == NULL && score == -INFINITY) {
	if (rsc->allocated_to) {
	crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname);
	free(rsc->allocated_to);
	rsc->allocated_to = NULL;
	}
	}
	}

	#define sort_return(an_int, why) do { \
	free(a_uuid); \
	free(b_uuid); \
	crm_trace("%s (%d) %c %s (%d) : %s", \
	a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
	b_xml_id, b_call_id, why); \
	return an_int; \
	} while(0)

	gint
	sort_op_by_callid(gconstpointer a, gconstpointer b)
	{
	int a_call_id = -1;
	int b_call_id = -1;

	char *a_uuid = NULL;
	char *b_uuid = NULL;

	const xmlNode *xml_a = a;
	const xmlNode *xml_b = b;

	const char *a_xml_id = crm_element_value(xml_a, XML_ATTR_ID);
	const char *b_xml_id = crm_element_value(xml_b, XML_ATTR_ID);

	if (pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_casei)) {
	/* We have duplicate lrm_rsc_op entries in the status
	* section which is unlikely to be a good thing
	* - we can handle it easily enough, but we need to get
	* to the bottom of why it's happening.
	*/
	pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
	sort_return(0, "duplicate");
	}

	crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
	crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);

	if (a_call_id == -1 && b_call_id == -1) {
	/* both are pending ops so it doesn't matter since
	* stops are never pending
	*/
	sort_return(0, "pending");

	} else if (a_call_id >= 0 && a_call_id < b_call_id) {
	sort_return(-1, "call id");

	} else if (b_call_id >= 0 && a_call_id > b_call_id) {
	sort_return(1, "call id");

	} else if (b_call_id >= 0 && a_call_id == b_call_id) {
	/*
	* The op and last_failed_op are the same
	* Order on last-rc-change
	*/
	time_t last_a = -1;
	time_t last_b = -1;

	crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
	crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);

	crm_trace("rc-change: %lld vs %lld",
	(long long) last_a, (long long) last_b);
	if (last_a >= 0 && last_a < last_b) {
	sort_return(-1, "rc-change");

	} else if (last_b >= 0 && last_a > last_b) {
	sort_return(1, "rc-change");
	}
	sort_return(0, "rc-change");

	} else {
	/* One of the inputs is a pending operation
	* Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
	*/

	int a_id = -1;
	int b_id = -1;

	const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
	const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);

	CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
	if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic a");
	}
	if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic b");
	}
	/* try to determine the relative age of the operation...
	* some pending operations (e.g. a start) may have been superseded
	* by a subsequent stop
	*
	* [a\|b]_id == -1 means it's a shutdown operation and _always_ comes last
	*/
	if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) \|\| a_id == b_id) {
	/*
	* some of the logic in here may be redundant...
	*
	* if the UUID from the TE doesn't match then one better
	* be a pending operation.
	* pending operations don't survive between elections and joins
	* because we query the LRM directly
	*/

	if (b_call_id == -1) {
	sort_return(-1, "transition + call");

	} else if (a_call_id == -1) {
	sort_return(1, "transition + call");
	}

	} else if ((a_id >= 0 && a_id < b_id) \|\| b_id == -1) {
	sort_return(-1, "transition");

	} else if ((b_id >= 0 && a_id > b_id) \|\| a_id == -1) {
	sort_return(1, "transition");
	}
	}

	/* we should never end up here */
	CRM_CHECK(FALSE, sort_return(0, "default"));

	}

	time_t
	get_effective_time(pe_working_set_t * data_set)
	{
	if(data_set) {
	if (data_set->now == NULL) {
	crm_trace("Recording a new 'now'");
	data_set->now = crm_time_new(NULL);
	}
	return crm_time_get_seconds_since_epoch(data_set->now);
	}

	crm_trace("Defaulting to 'now'");
	return time(NULL);
	}

	gboolean
	get_target_role(pe_resource_t * rsc, enum rsc_role_e * role)
	{
	enum rsc_role_e local_role = RSC_ROLE_UNKNOWN;
	const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);

	CRM_CHECK(role != NULL, return FALSE);

	if (pcmk__str_eq(value, "started", pcmk__str_null_matches \| pcmk__str_casei)
	\|\| pcmk__str_eq("default", value, pcmk__str_casei)) {
	return FALSE;
	}

	local_role = text2role(value);
	if (local_role == RSC_ROLE_UNKNOWN) {
	pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
	"because '%s' is not valid", rsc->id, value);
	return FALSE;

	} else if (local_role > RSC_ROLE_STARTED) {
	if (pcmk_is_set(uber_parent(rsc)->flags, pe_rsc_promotable)) {
	if (local_role > RSC_ROLE_SLAVE) {
	/* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
	return FALSE;
	}

	} else {
	pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
	"because '%s' only makes sense for promotable "
	"clones", rsc->id, value);
	return FALSE;
	}
	}

	*role = local_role;
	return TRUE;
	}

	gboolean
	order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order)
	{
	GListPtr gIter = NULL;
	pe_action_wrapper_t *wrapper = NULL;
	GListPtr list = NULL;

	if (order == pe_order_none) {
	return FALSE;
	}

	if (lh_action == NULL \|\| rh_action == NULL) {
	return FALSE;
	}

	crm_trace("Ordering Action %s before %s", lh_action->uuid, rh_action->uuid);

	/* Ensure we never create a dependency on ourselves... it's happened */
	CRM_ASSERT(lh_action != rh_action);

	/* Filter dups, otherwise update_action_states() has too much work to do */
	gIter = lh_action->actions_after;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_wrapper_t after = (pe_action_wrapper_t ) gIter->data;

	if (after->action == rh_action && (after->type & order)) {
	return FALSE;
	}
	}

	wrapper = calloc(1, sizeof(pe_action_wrapper_t));
	wrapper->action = rh_action;
	wrapper->type = order;
	list = lh_action->actions_after;
	list = g_list_prepend(list, wrapper);
	lh_action->actions_after = list;

	wrapper = calloc(1, sizeof(pe_action_wrapper_t));
	wrapper->action = lh_action;
	wrapper->type = order;
	list = rh_action->actions_before;
	list = g_list_prepend(list, wrapper);
	rh_action->actions_before = list;
	return TRUE;
	}

	pe_action_t *
	get_pseudo_op(const char name, pe_working_set_t data_set)
	{
	pe_action_t *op = NULL;

	if(data_set->singletons) {
	op = g_hash_table_lookup(data_set->singletons, name);
	}
	if (op == NULL) {
	op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);
	}

	return op;
	}

	void
	destroy_ticket(gpointer data)
	{
	pe_ticket_t *ticket = data;

	if (ticket->state) {
	g_hash_table_destroy(ticket->state);
	}
	free(ticket->id);
	free(ticket);
	}

	pe_ticket_t *
	ticket_new(const char ticket_id, pe_working_set_t data_set)
	{
	pe_ticket_t *ticket = NULL;

	if (pcmk__str_empty(ticket_id)) {
	return NULL;
	}

	if (data_set->tickets == NULL) {
	data_set->tickets =
	g_hash_table_new_full(crm_str_hash, g_str_equal, free,
	destroy_ticket);
	}

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {

	ticket = calloc(1, sizeof(pe_ticket_t));
	if (ticket == NULL) {
	crm_err("Cannot allocate ticket '%s'", ticket_id);
	return NULL;
	}

	crm_trace("Creaing ticket entry for %s", ticket_id);

	ticket->id = strdup(ticket_id);
	ticket->granted = FALSE;
	ticket->last_granted = -1;
	ticket->standby = FALSE;
	ticket->state = crm_str_table_new();

	g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket);
	}

	return ticket;
	}

	static void
	filter_parameters(xmlNode * param_set, const char *param_string, bool need_present)
	{
	if (param_set && param_string) {
	xmlAttrPtr xIter = param_set->properties;

	while (xIter) {
	const char prop_name = (const char )xIter->name;
	char *name = crm_strdup_printf(" %s ", prop_name);
	char *match = strstr(param_string, name);

	free(name);

	// Do now, because current entry might get removed below
	xIter = xIter->next;

	if (need_present && match == NULL) {
	crm_trace("%s not found in %s", prop_name, param_string);
	xml_remove_prop(param_set, prop_name);

	} else if (need_present == FALSE && match) {
	crm_trace("%s found in %s", prop_name, param_string);
	xml_remove_prop(param_set, prop_name);
	}
	}
	}
	}

	#if ENABLE_VERSIONED_ATTRS
	static void
	append_versioned_params(xmlNode versioned_params, const char ra_version, xmlNode *params)
	{
	GHashTable *hash = pe_unpack_versioned_parameters(versioned_params, ra_version);
	char *key = NULL;
	char *value = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, hash);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	crm_xml_add(params, key, value);
	}
	g_hash_table_destroy(hash);
	}
	#endif

	/*!
	* \internal
	* \brief Calculate action digests and store in node's digest cache
	*
	* \param[in] rsc Resource that action was for
	* \param[in] task Name of action performed
	* \param[in] key Action's task key
	* \param[in] node Node action was performed on
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] calc_secure Whether to calculate secure digest
	* \param[in] data_set Cluster working set
	*
	* \return Pointer to node's digest cache entry
	*/
	static op_digest_cache_t *
	rsc_action_digest(pe_resource_t rsc, const char task, const char *key,
	pe_node_t node, xmlNode xml_op, bool calc_secure,
	pe_working_set_t *data_set)
	{
	op_digest_cache_t *data = NULL;

	data = g_hash_table_lookup(node->details->digest_cache, key);
	if (data == NULL) {
	GHashTable *local_rsc_params = crm_str_table_new();
	pe_action_t *action = custom_action(rsc, strdup(key), task, node, TRUE, FALSE, data_set);
	#if ENABLE_VERSIONED_ATTRS
	xmlNode *local_versioned_params = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);
	const char *ra_version = NULL;
	#endif

	const char *op_version;
	const char *restart_list = NULL;
	const char *secure_list = " passwd password ";

	data = calloc(1, sizeof(op_digest_cache_t));
	CRM_ASSERT(data != NULL);

	get_rsc_attributes(local_rsc_params, rsc, node, data_set);
	#if ENABLE_VERSIONED_ATTRS
	pe_get_versioned_attributes(local_versioned_params, rsc, node, data_set);
	#endif

	data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);

	// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
	if (pe__add_bundle_remote_name(rsc, data->params_all,
	XML_RSC_ATTR_REMOTE_RA_ADDR)) {
	crm_trace("Set address for bundle connection %s (on %s)",
	rsc->id, node->details->uname);
	}

	g_hash_table_foreach(local_rsc_params, hash2field, data->params_all);
	g_hash_table_foreach(action->extra, hash2field, data->params_all);
	g_hash_table_foreach(rsc->parameters, hash2field, data->params_all);
	g_hash_table_foreach(action->meta, hash2metafield, data->params_all);

	if(xml_op) {
	secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
	restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);

	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	#if ENABLE_VERSIONED_ATTRS
	ra_version = crm_element_value(xml_op, XML_ATTR_RA_VERSION);
	#endif

	} else {
	op_version = CRM_FEATURE_SET;
	}

	#if ENABLE_VERSIONED_ATTRS
	append_versioned_params(local_versioned_params, ra_version, data->params_all);
	append_versioned_params(rsc->versioned_parameters, ra_version, data->params_all);

	{
	pe_rsc_action_details_t *details = pe_rsc_action_details(action);
	append_versioned_params(details->versioned_parameters, ra_version, data->params_all);
	}
	#endif

	pcmk__filter_op_for_digest(data->params_all);

	g_hash_table_destroy(local_rsc_params);
	pe_free_action(action);

	data->digest_all_calc = calculate_operation_digest(data->params_all, op_version);

	if (calc_secure) {
	data->params_secure = copy_xml(data->params_all);
	if(secure_list) {
	filter_parameters(data->params_secure, secure_list, FALSE);
	}
	data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version);
	}

	if(xml_op && crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) != NULL) {
	data->params_restart = copy_xml(data->params_all);
	if (restart_list) {
	filter_parameters(data->params_restart, restart_list, TRUE);
	}
	data->digest_restart_calc = calculate_operation_digest(data->params_restart, op_version);
	}

	g_hash_table_insert(node->details->digest_cache, strdup(key), data);
	}

	return data;
	}

	op_digest_cache_t *
	rsc_action_digest_cmp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * node,
	pe_working_set_t * data_set)
	{
	op_digest_cache_t *data = NULL;

	char *key = NULL;
	guint interval_ms = 0;

	const char *op_version;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *digest_all;
	const char *digest_restart;

	CRM_ASSERT(node != NULL);

	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
	digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	key = pcmk__op_key(rsc->id, task, interval_ms);
	data = rsc_action_digest(rsc, task, key, node, xml_op,
	pcmk_is_set(data_set->flags, pe_flag_sanitized),
	data_set);

	data->rc = RSC_DIGEST_MATCH;
	if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) {
	pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (restart:%s) %s",
	key, node->details->uname,
	crm_str(digest_restart), data->digest_restart_calc,
	op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	data->rc = RSC_DIGEST_RESTART;

	} else if (digest_all == NULL) {
	/* it is unknown what the previous op digest was */
	data->rc = RSC_DIGEST_UNKNOWN;

	} else if (strcmp(digest_all, data->digest_all_calc) != 0) {
	pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (%s:%s) %s",
	key, node->details->uname,
	crm_str(digest_all), data->digest_all_calc,
	(interval_ms > 0)? "reschedule" : "reload",
	op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	data->rc = RSC_DIGEST_ALL;
	}

	free(key);
	return data;
	}

	/*!
	* \internal
	* \brief Create an unfencing summary for use in special node attribute
	*
	* Create a string combining a fence device's resource ID, agent type, and
	* parameter digest (whether for all parameters or just non-private parameters).
	* This can be stored in a special node attribute, allowing us to detect changes
	* in either the agent type or parameters, to know whether unfencing must be
	* redone or can be safely skipped when the device's history is cleaned.
	*
	* \param[in] rsc_id Fence device resource ID
	* \param[in] agent_type Fence device agent
	* \param[in] param_digest Fence device parameter digest
	*
	* \return Newly allocated string with unfencing digest
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	create_unfencing_summary(const char rsc_id, const char agent_type,
	const char *param_digest)
	{
	return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest);
	}

	/*!
	* \internal
	* \brief Check whether a node can skip unfencing
	*
	* Check whether a fence device's current definition matches a node's
	* stored summary of when it was last unfenced by the device.
	*
	* \param[in] rsc_id Fence device's resource ID
	* \param[in] agent Fence device's agent type
	* \param[in] digest_calc Fence device's current parameter digest
	* \param[in] node_summary Value of node's special unfencing node attribute
	* (a comma-separated list of unfencing summaries for
	* all devices that have unfenced this node)
	*
	* \return TRUE if digest matches, FALSE otherwise
	*/
	static bool
	unfencing_digest_matches(const char rsc_id, const char agent,
	const char digest_calc, const char node_summary)
	{
	bool matches = FALSE;

	if (rsc_id && agent && digest_calc && node_summary) {
	char *search_secure = create_unfencing_summary(rsc_id, agent,
	digest_calc);

	/* The digest was calculated including the device ID and agent,
	* so there is no risk of collision using strstr().
	*/
	matches = (strstr(node_summary, search_secure) != NULL);
	crm_trace("Calculated unfencing digest '%s' %sfound in '%s'",
	search_secure, matches? "" : "not ", node_summary);
	free(search_secure);
	}
	return matches;
	}

	/* Magic string to use as action name for digest cache entries used for
	* unfencing checks. This is not a real action name (i.e. "on"), so
	* check_action_definition() won't confuse these entries with real actions.
	*/
	#define STONITH_DIGEST_TASK "stonith-on"

	/*!
	* \internal
	* \brief Calculate fence device digests and digest comparison result
	*
	* \param[in] rsc Fence device resource
	* \param[in] agent Fence device's agent type
	* \param[in] node Node with digest cache to use
	* \param[in] data_set Cluster working set
	*
	* \return Node's digest cache entry
	*/
	static op_digest_cache_t *
	fencing_action_digest_cmp(pe_resource_t rsc, const char agent,
	pe_node_t node, pe_working_set_t data_set)
	{
	const char *node_summary = NULL;

	// Calculate device's current parameter digests
	char *key = pcmk__op_key(rsc->id, STONITH_DIGEST_TASK, 0);
	op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key,
	node, NULL, TRUE, data_set);

	free(key);

	// Check whether node has special unfencing summary node attribute
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL);
	if (node_summary == NULL) {
	data->rc = RSC_DIGEST_UNKNOWN;
	return data;
	}

	// Check whether full parameter digest matches
	if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc,
	node_summary)) {
	data->rc = RSC_DIGEST_MATCH;
	return data;
	}

	// Check whether secure parameter digest matches
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE);
	if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc,
	node_summary)) {
	data->rc = RSC_DIGEST_MATCH;
	if (pcmk_is_set(data_set->flags, pe_flag_stdout)) {
	printf("Only 'private' parameters to %s for unfencing %s changed\n",
	rsc->id, node->details->uname);
	}
	return data;
	}

	// Parameters don't match
	data->rc = RSC_DIGEST_ALL;
	if (pcmk_is_set(data_set->flags, (pe_flag_sanitized\|pe_flag_stdout))
	&& data->digest_secure_calc) {
	char *digest = create_unfencing_summary(rsc->id, agent,
	data->digest_secure_calc);

	printf("Parameters to %s for unfencing %s changed, try '%s'\n",
	rsc->id, node->details->uname, digest);
	free(digest);
	}
	return data;
	}

	const char rsc_printable_id(pe_resource_t rsc)
	{
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	return ID(rsc->xml);
	}
	return rsc->id;
	}

	void
	pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags)
	{
	pe__clear_resource_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__clear_resource_flags_recursive((pe_resource_t *) gIter->data, flags);
	}
	}

	void
	pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags)
	{
	pe__set_resource_flags(rsc, flags);
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe__set_resource_flags_recursive((pe_resource_t *) gIter->data, flags);
	}
	}

	static GListPtr
	find_unfencing_devices(GListPtr candidates, GListPtr matches)
	{
	for (GListPtr gIter = candidates; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *candidate = gIter->data;
	const char *provides = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_PROVIDES);
	const char *requires = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_REQUIRES);

	if(candidate->children) {
	matches = find_unfencing_devices(candidate->children, matches);
	} else if (!pcmk_is_set(candidate->flags, pe_rsc_fence_device)) {
	continue;

	} else if (pcmk__str_eq(provides, "unfencing", pcmk__str_casei) \|\| pcmk__str_eq(requires, "unfencing", pcmk__str_casei)) {
	matches = g_list_prepend(matches, candidate);
	}
	}
	return matches;
	}

	static int
	node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set)
	{
	int member_count = 0;
	int online_count = 0;
	int top_priority = 0;
	int lowest_priority = 0;
	GListPtr gIter = NULL;

	// `priority-fencing-delay` is disabled
	if (data_set->priority_fencing_delay <= 0) {
	return 0;
	}

	/* No need to request a delay if the fencing target is not a normal cluster
	* member, for example if it's a remote node or a guest node. */
	if (node->details->type != node_member) {
	return 0;
	}

	// No need to request a delay if the fencing target is in our partition
	if (node->details->online) {
	return 0;
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t *n = gIter->data;

	if (n->details->type != node_member) {
	continue;
	}

	member_count ++;

	if (n->details->online) {
	online_count++;
	}

	if (member_count == 1
	\|\| n->details->priority > top_priority) {
	top_priority = n->details->priority;
	}

	if (member_count == 1
	\|\| n->details->priority < lowest_priority) {
	lowest_priority = n->details->priority;
	}
	}

	// No need to delay if we have more than half of the cluster members
	if (online_count > member_count / 2) {
	return 0;
	}

	/* All the nodes have equal priority.
	* Any configured corresponding `pcmk_delay_base/max` will be applied. */
	if (lowest_priority == top_priority) {
	return 0;
	}

	if (node->details->priority < top_priority) {
	return 0;
	}

	return data_set->priority_fencing_delay;
	}

	pe_action_t *
	pe_fence_op(pe_node_t * node, const char op, bool optional, const char reason,
	bool priority_delay, pe_working_set_t * data_set)
	{
	char *op_key = NULL;
	pe_action_t *stonith_op = NULL;

	if(op == NULL) {
	op = data_set->stonith_action;
	}

	op_key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);

	if(data_set->singletons) {
	stonith_op = g_hash_table_lookup(data_set->singletons, op_key);
	}

	if(stonith_op == NULL) {
	stonith_op = custom_action(NULL, op_key, CRM_OP_FENCE, node, TRUE, TRUE, data_set);

	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
	add_hash_param(stonith_op->meta, "stonith_action", op);

	if (pe__is_guest_or_remote_node(node)
	&& pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)) {
	/* Extra work to detect device changes on remotes
	*
	* We may do this for all nodes in the future, but for now
	* the check_action_definition() based stuff works fine.
	*/
	long max = 1024;
	long digests_all_offset = 0;
	long digests_secure_offset = 0;

	char *digests_all = calloc(max, sizeof(char));
	char *digests_secure = calloc(max, sizeof(char));
	GListPtr matches = find_unfencing_devices(data_set->resources, NULL);

	for (GListPtr gIter = matches; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *match = gIter->data;
	const char *agent = g_hash_table_lookup(match->meta,
	XML_ATTR_TYPE);
	op_digest_cache_t *data = NULL;

	data = fencing_action_digest_cmp(match, agent, node, data_set);
	if(data->rc == RSC_DIGEST_ALL) {
	optional = FALSE;
	crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id);
	if (pcmk_is_set(data_set->flags, pe_flag_stdout)) {
	fprintf(stdout, " notice: Unfencing %s (remote): because the definition of %s changed\n", node->details->uname, match->id);
	}
	}

	digests_all_offset += snprintf(
	digests_all+digests_all_offset, max-digests_all_offset,
	"%s:%s:%s,", match->id, agent, data->digest_all_calc);

	digests_secure_offset += snprintf(
	digests_secure+digests_secure_offset, max-digests_secure_offset,
	"%s:%s:%s,", match->id, agent, data->digest_secure_calc);
	}
	g_hash_table_insert(stonith_op->meta,
	strdup(XML_OP_ATTR_DIGESTS_ALL),
	digests_all);
	g_hash_table_insert(stonith_op->meta,
	strdup(XML_OP_ATTR_DIGESTS_SECURE),
	digests_secure);
	}

	} else {
	free(op_key);
	}

	if (data_set->priority_fencing_delay > 0

	/* It's a suitable case where `priority-fencing-delay` applies.
	* At least add `priority-fencing-delay` field as an indicator. */
	&& (priority_delay

	/* Re-calculate priority delay for the suitable case when
	* pe_fence_op() is called again by stage6() after node priority has
	* been actually calculated with native_add_running() */
	\|\| g_hash_table_lookup(stonith_op->meta,
	XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) {

	/* Add `priority-fencing-delay` to the fencing op even if it's 0 for
	* the targeting node. So that it takes precedence over any possible
	* `pcmk_delay_base/max`.
	*/
	char *delay_s = crm_itoa(node_priority_fencing_delay(node, data_set));

	g_hash_table_insert(stonith_op->meta,
	strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY),
	delay_s);
	}

	if(optional == FALSE && pe_can_fence(data_set, node)) {
	pe_action_required(stonith_op, NULL, reason);
	} else if(reason && stonith_op->reason == NULL) {
	stonith_op->reason = strdup(reason);
	}

	return stonith_op;
	}

	void
	trigger_unfencing(
	pe_resource_t * rsc, pe_node_t node, const char reason, pe_action_t dependency, pe_working_set_t data_set)
	{
	if (!pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)) {
	/* No resources require it */
	return;

	} else if ((rsc != NULL)
	&& !pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
	/* Wasn't a stonith device */
	return;

	} else if(node
	&& node->details->online
	&& node->details->unclean == FALSE
	&& node->details->shutdown == FALSE) {
	pe_action_t *unfence = pe_fence_op(node, "on", FALSE, reason, FALSE, data_set);

	if(dependency) {
	order_actions(unfence, dependency, pe_order_optional);
	}

	} else if(rsc) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
	trigger_unfencing(rsc, node, reason, dependency, data_set);
	}
	}
	}
	}

	gboolean
	add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref)
	{
	pe_tag_t *tag = NULL;
	GListPtr gIter = NULL;
	gboolean is_existing = FALSE;

	CRM_CHECK(tags && tag_name && obj_ref, return FALSE);

	tag = g_hash_table_lookup(tags, tag_name);
	if (tag == NULL) {
	tag = calloc(1, sizeof(pe_tag_t));
	if (tag == NULL) {
	return FALSE;
	}
	tag->id = strdup(tag_name);
	tag->refs = NULL;
	g_hash_table_insert(tags, strdup(tag_name), tag);
	}

	for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
	const char existing_ref = (const char ) gIter->data;

	if (pcmk__str_eq(existing_ref, obj_ref, pcmk__str_none)){
	is_existing = TRUE;
	break;
	}
	}

	if (is_existing == FALSE) {
	tag->refs = g_list_append(tag->refs, strdup(obj_ref));
	crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
	}

	return TRUE;
	}

	void pe_action_set_flag_reason(const char *function, long line,
	pe_action_t action, pe_action_t reason, const char *text,
	enum pe_action_flags flags, bool overwrite)
	{
	bool unset = FALSE;
	bool update = FALSE;
	const char *change = NULL;

	if (pcmk_is_set(flags, pe_action_runnable)) {
	unset = TRUE;
	change = "unrunnable";
	} else if (pcmk_is_set(flags, pe_action_optional)) {
	unset = TRUE;
	change = "required";
	} else if (pcmk_is_set(flags, pe_action_migrate_runnable)) {
	unset = TRUE;
	overwrite = TRUE;
	change = "unrunnable";
	} else if (pcmk_is_set(flags, pe_action_dangle)) {
	change = "dangling";
	} else if (pcmk_is_set(flags, pe_action_requires_any)) {
	change = "required";
	} else {
	crm_err("Unknown flag change to %x by %s: 0x%s",
	flags, action->uuid, (reason? reason->uuid : "0"));
	}

	if(unset) {
	if (pcmk_is_set(action->flags, flags)) {
	pe__clear_action_flags_as(function, line, action, flags);
	update = TRUE;
	}

	} else {
	if (!pcmk_is_set(action->flags, flags)) {
	pe__set_action_flags_as(function, line, action, flags);
	update = TRUE;
	}
	}

	if((change && update) \|\| text) {
	char *reason_text = NULL;
	if(reason == NULL) {
	pe_action_set_reason(action, text, overwrite);

	} else if(reason->rsc == NULL) {
	reason_text = crm_strdup_printf("%s %s%c %s", change, reason->task, text?':':0, text?text:"");
	} else {
	reason_text = crm_strdup_printf("%s %s %s%c %s", change, reason->rsc->id, reason->task, text?':':0, text?text:"NA");
	}

	if(reason_text && action->rsc != reason->rsc) {
	pe_action_set_reason(action, reason_text, overwrite);
	}
	free(reason_text);
	}
	}

	void pe_action_set_reason(pe_action_t action, const char reason, bool overwrite)
	{
	if (action->reason != NULL && overwrite) {
	pe_rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
	action->uuid, action->reason, crm_str(reason));
	free(action->reason);
	} else if (action->reason == NULL) {
	pe_rsc_trace(action->rsc, "Set %s reason to '%s'",
	action->uuid, crm_str(reason));
	} else {
	// crm_assert(action->reason != NULL && !overwrite);
	return;
	}

	if (reason != NULL) {
	action->reason = strdup(reason);
	} else {
	action->reason = NULL;
	}
	}

	/*!
	* \internal
	* \brief Check whether shutdown has been requested for a node
	*
	* \param[in] node Node to check
	*
	* \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
	* \note This differs from simply using node->details->shutdown in that it can
	* be used before that has been determined (and in fact to determine it),
	* and it can also be used to distinguish requested shutdown from implicit
	* shutdown of remote nodes by virtue of their connection stopping.
	*/
	bool
	pe__shutdown_requested(pe_node_t *node)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);

	return !pcmk__str_eq(shutdown, "0", pcmk__str_null_matches);
	}

	/*!
	* \internal
	* \brief Update a data set's "recheck by" time
	*
	* \param[in] recheck Epoch time when recheck should happen
	* \param[in,out] data_set Current working set
	*/
	void
	pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set)
	{
	if ((recheck > get_effective_time(data_set))
	&& ((data_set->recheck_by == 0)
	\|\| (data_set->recheck_by > recheck))) {
	data_set->recheck_by = recheck;
	}
	}

	/*!
	* \internal
	* \brief Wrapper for pe_unpack_nvpairs() using a cluster working set
	*/
	void
	pe__unpack_dataset_nvpairs(xmlNode xml_obj, const char set_name,
	pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	pe_working_set_t *data_set)
	{
	crm_time_t *next_change = crm_time_new_undefined();

	pe_eval_nvpairs(data_set->input, xml_obj, set_name, rule_data, hash,
	always_first, overwrite, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, data_set);
	}
	crm_time_free(next_change);
	}

	bool
	pe__resource_is_disabled(pe_resource_t *rsc)
	{
	const char *target_role = NULL;

	CRM_CHECK(rsc != NULL, return false);
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	if (target_role) {
	enum rsc_role_e target_role_e = text2role(target_role);

	if ((target_role_e == RSC_ROLE_STOPPED)
	\|\| ((target_role_e == RSC_ROLE_SLAVE)
	&& pcmk_is_set(uber_parent(rsc)->flags, pe_rsc_promotable))) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Create an action to clear a resource's history from CIB
	*
	* \param[in] rsc Resource to clear
	* \param[in] node Node to clear history on
	*
	* \return New action to clear resource history
	*/
	pe_action_t *
	pe__clear_resource_history(pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set)
	{
	char *key = NULL;

	CRM_ASSERT(rsc && node);
	key = pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0);
	return custom_action(rsc, key, CRM_OP_LRM_DELETE, node, FALSE, TRUE,
	data_set);
	}

	bool
	pe__rsc_running_on_any_node_in_list(pe_resource_t *rsc, GListPtr node_list)
	{
	for (GListPtr ele = rsc->running_on; ele; ele = ele->next) {
	pe_node_t node = (pe_node_t ) ele->data;
	if (pcmk__str_in_list(node_list, node->details->uname)) {
	return true;
	}
	}

	return false;
	}

	bool
	pcmk__rsc_filtered_by_node(pe_resource_t *rsc, GListPtr only_node)
	{
	return (rsc->fns->active(rsc, FALSE) && !pe__rsc_running_on_any_node_in_list(rsc, only_node));
	}

	GListPtr
	pe__filter_rsc_list(GListPtr rscs, GListPtr filter)
	{
	GListPtr retval = NULL;

	for (GListPtr gIter = rscs; gIter; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* I think the second condition is safe here for all callers of this
	* function. If not, it needs to move into pe__node_text.
	*/
	if (pcmk__str_in_list(filter, rsc_printable_id(rsc)) \|\|
	(rsc->parent && pcmk__str_in_list(filter, rsc_printable_id(rsc->parent)))) {
	retval = g_list_prepend(retval, rsc);
	}
	}

	return retval;
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 10:03 AM (1 d, 12 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1018554
Default Alt Text: (236 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions