No OneTemporary
Actions

Size

112 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index 1c0f3bd3a0..8aeb302809 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,1713 +1,1713 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <crm_internal.h>

	#include <lrm/lrm_api.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/msg.h>
	#include <crm/common/util.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/rules.h>
	#include <utils.h>
	#include <unpack.h>

	#define set_config_flag(data_set, option, flag) do { \
	const char *tmp = pe_pref(data_set->config_hash, option); \
	if(tmp) { \
	if(crm_is_true(tmp)) { \
	set_bit_inplace(data_set->flags, flag); \
	} else { \
	clear_bit_inplace(data_set->flags, flag); \
	} \
	} \
	} while(0)

	gboolean unpack_rsc_op(
	resource_t rsc, node_t node, xmlNode *xml_op,
	enum action_fail_response failed, pe_working_set_t data_set);


	gboolean
	unpack_config(xmlNode config, pe_working_set_t data_set)
	{
	const char *value = NULL;
	GHashTable *config_hash = g_hash_table_new_full(
	g_str_hash,g_str_equal, g_hash_destroy_str,g_hash_destroy_str);

	data_set->config_hash = config_hash;

	unpack_instance_attributes(
	data_set->input, config, XML_CIB_TAG_PROPSET, NULL, config_hash,
	CIB_OPTIONS_FIRST, FALSE, data_set->now);

	verify_pe_options(data_set->config_hash);

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	data_set->stonith_timeout = crm_get_msec(value);
	crm_debug("STONITH timeout: %d", data_set->stonith_timeout);

	set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
	crm_debug("STONITH of failed nodes is %s",
	is_set(data_set->flags, pe_flag_stonith_enabled)?"enabled":"disabled");

	data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
	crm_debug_2("STONITH will %s nodes", data_set->stonith_action);

	set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
	crm_debug("Stop all active resources: %s",
	is_set(data_set->flags, pe_flag_stop_everything)?"true":"false");

	set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
	if(is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	crm_debug("Cluster is symmetric"
	" - resources can run anywhere by default");
	}

	value = pe_pref(data_set->config_hash, "default-resource-stickiness");
	data_set->default_resource_stickiness = char2score(value);
	crm_debug("Default stickiness: %d",
	data_set->default_resource_stickiness);

	value = pe_pref(data_set->config_hash, "no-quorum-policy");

	if(safe_str_eq(value, "ignore")) {
	data_set->no_quorum_policy = no_quorum_ignore;

	} else if(safe_str_eq(value, "freeze")) {
	data_set->no_quorum_policy = no_quorum_freeze;

	} else if(safe_str_eq(value, "suicide")) {
	gboolean do_panic = FALSE;
	crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC, &do_panic);

	if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE){
	crm_config_err("Setting no-quorum-policy=suicide makes no sense if stonith-enabled=false");
	}

	if(do_panic && is_set(data_set->flags, pe_flag_stonith_enabled)) {
	data_set->no_quorum_policy = no_quorum_suicide;

	} else if(is_set(data_set->flags, pe_flag_have_quorum) == FALSE && do_panic == FALSE) {
	crm_notice("Resetting no-quorum-policy to 'stop': The cluster has never had quorum");
	data_set->no_quorum_policy = no_quorum_stop;
	}

	} else {
	data_set->no_quorum_policy = no_quorum_stop;
	}

	switch (data_set->no_quorum_policy) {
	case no_quorum_freeze:
	crm_debug("On loss of CCM Quorum: Freeze resources");
	break;
	case no_quorum_stop:
	crm_debug("On loss of CCM Quorum: Stop ALL resources");
	break;
	case no_quorum_suicide:
	crm_notice("On loss of CCM Quorum: Fence all remaining nodes");
	break;
	case no_quorum_ignore:
	crm_notice("On loss of CCM Quorum: Ignore");
	break;
	}

	set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
	crm_debug_2("Orphan resources are %s",
	is_set(data_set->flags, pe_flag_stop_rsc_orphans)?"stopped":"ignored");

	set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
	crm_debug_2("Orphan resource actions are %s",
	is_set(data_set->flags, pe_flag_stop_action_orphans)?"stopped":"ignored");

	set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
	crm_debug_2("Stopped resources are removed from the status section: %s",
	is_set(data_set->flags, pe_flag_remove_after_stop)?"true":"false");

	set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
	crm_debug_2("Maintenance mode: %s",
	is_set(data_set->flags, pe_flag_maintenance_mode)?"true":"false");

	if(is_set(data_set->flags, pe_flag_maintenance_mode)) {
	clear_bit(data_set->flags, pe_flag_is_managed_default);
	} else {
	set_config_flag(data_set, "is-managed-default", pe_flag_is_managed_default);
	}
	crm_debug_2("By default resources are %smanaged",
	is_set(data_set->flags, pe_flag_is_managed_default)?"":"not ");

	set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
	crm_debug_2("Start failures are %s",
	is_set(data_set->flags, pe_flag_start_failure_fatal)?"always fatal":"handled by failcount");

	node_score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
	node_score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
	node_score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));

	crm_info("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
	score2char(node_score_red),score2char(node_score_yellow),
	score2char(node_score_green));

	return TRUE;
	}

	gboolean
	unpack_nodes(xmlNode * xml_nodes, pe_working_set_t *data_set)
	{
	node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	gboolean unseen_are_unclean = TRUE;
	const char *blind_faith = pe_pref(
	data_set->config_hash, "startup-fencing");

	if(crm_is_true(blind_faith) == FALSE) {
	unseen_are_unclean = FALSE;
	crm_warn("Blind faith: not fencing unseen nodes");
	}

	xml_child_iter_filter(
	xml_nodes, xml_obj, XML_CIB_TAG_NODE,

	new_node = NULL;

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
	type = crm_element_value(xml_obj, XML_ATTR_TYPE);
	crm_debug_3("Processing node %s/%s", uname, id);

	if(id == NULL) {
	crm_config_err("Must specify id tag in <node>");
	continue;
	}
	if(type == NULL) {
	crm_config_err("Must specify type tag in <node>");
	continue;
	}
	if(pe_find_node(data_set->nodes, uname) != NULL) {
	crm_config_warn("Detected multiple node entries with uname=%s"
	" - this is rarely intended", uname);
	}

	crm_malloc0(new_node, sizeof(node_t));
	if(new_node == NULL) {
	return FALSE;
	}

	new_node->weight = 0;
	new_node->fixed = FALSE;
	crm_malloc0(new_node->details,
	sizeof(struct node_shared_s));

	if(new_node->details == NULL) {
	crm_free(new_node);
	return FALSE;
	}

	crm_debug_3("Creaing node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->type = node_ping;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->running_rsc = NULL;
	new_node->details->attrs = g_hash_table_new_full(
	g_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	/* if(data_set->have_quorum == FALSE */
	/* && data_set->no_quorum_policy == no_quorum_stop) { */
	/* /\* start shutting resources down \/ /
	/* new_node->weight = -INFINITY; */
	/* } */

	if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE \|\| unseen_are_unclean == FALSE) {
	/* blind faith... */
	new_node->details->unclean = FALSE;

	} else {
	/* all nodes are unclean until we've seen their
	* status entry
	*/
	new_node->details->unclean = TRUE;
	}

	if(type == NULL
	\|\| safe_str_eq(type, "member")
	\|\| safe_str_eq(type, NORMALNODE)) {
	new_node->details->type = node_member;
	}

	add_node_attrs(xml_obj, new_node, FALSE, data_set);
	data_set->nodes = g_list_append(data_set->nodes, new_node);
	crm_debug_3("Done with node %s",
	crm_element_value(xml_obj, XML_ATTR_UNAME));
	);

	return TRUE;
	}

	gboolean
	unpack_resources(xmlNode * xml_resources, pe_working_set_t *data_set)
	{
	xml_child_iter(
	xml_resources, xml_obj,

	resource_t *new_rsc = NULL;
	crm_debug_3("Begining unpack... <%s id=%s... >",
	crm_element_name(xml_obj), ID(xml_obj));
	if(common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
	data_set->resources = g_list_append(
	data_set->resources, new_rsc);

	print_resource(LOG_DEBUG_3, "Added", new_rsc, FALSE);

	} else {
	crm_config_err("Failed unpacking %s %s",
	crm_element_name(xml_obj),
	crm_element_value(xml_obj, XML_ATTR_ID));
	if(new_rsc != NULL && new_rsc->fns != NULL) {
	new_rsc->fns->free(new_rsc);
	}
	}
	);

	data_set->resources = g_list_sort(
	data_set->resources, sort_rsc_priority);

	if(is_set(data_set->flags, pe_flag_stonith_enabled) && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
	- crm_config_err("No STONITH resources have been defined");
	+ crm_config_err("Resource start-up disabled since no STONITH resources have been defined");
	crm_config_err("Either configure some or disable STONITH with the stonith-enabled option");
	crm_config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}


	/* remove nodes that are down, stopping */
	/* create +ve rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode * status, pe_working_set_t *data_set)
	{
	const char *id = NULL;
	const char *uname = NULL;

	xmlNode * lrm_rsc = NULL;
	xmlNode * attrs = NULL;
	node_t *this_node = NULL;

	crm_debug_3("Begining unpack");
	xml_child_iter_filter(
	status, node_state, XML_CIB_TAG_STATE,

	id = crm_element_value(node_state, XML_ATTR_ID);
	uname = crm_element_value(node_state, XML_ATTR_UNAME);
	attrs = find_xml_node(
	node_state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);

	lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);

	crm_debug_3("Processing node %s", uname);
	this_node = pe_find_node_id(data_set->nodes, id);

	if(uname == NULL) {
	/* error */
	continue;

	} else if(this_node == NULL) {
	crm_config_warn("Node %s in status section no longer exists",
	uname);
	continue;
	}

	/* Mark the node as provisionally clean
	* - at least we have seen it in the current cluster's lifetime
	*/
	this_node->details->unclean = FALSE;
	add_node_attrs(attrs, this_node, TRUE, data_set);

	if(crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
	crm_info("Node %s is in standby-mode",
	this_node->details->uname);
	this_node->details->standby = TRUE;
	}

	crm_debug_3("determining node state");
	determine_online_status(node_state, this_node, data_set);

	if(this_node->details->online
	&& data_set->no_quorum_policy == no_quorum_suicide) {
	/* Everything else should flow from this automatically
	* At least until the PE becomes able to migrate off healthy resources
	*/
	crm_notice("Marking node %s for STONITH: The cluster does not have quorum",
	this_node->details->uname);
	this_node->details->unclean = TRUE;
	}

	if(this_node->details->online \|\| is_set(data_set->flags, pe_flag_stonith_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	crm_debug_3("Processing lrm resource entries");
	unpack_lrm_resources(this_node, lrm_rsc, data_set);
	}
	);

	return TRUE;

	}

	static gboolean
	determine_online_status_no_fencing(xmlNode * node_state, node_t *this_node)
	{
	gboolean online = FALSE;
	const char *join_state = crm_element_value(node_state, XML_CIB_ATTR_JOINSTATE);
	const char *crm_state = crm_element_value(node_state, XML_CIB_ATTR_CRMDSTATE);
	const char *ccm_state = crm_element_value(node_state, XML_CIB_ATTR_INCCM);
	const char *ha_state = crm_element_value(node_state, XML_CIB_ATTR_HASTATE);
	const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE);

	if(ha_state == NULL) {
	ha_state = DEADSTATUS;
	}

	if(!crm_is_true(ccm_state) \|\| safe_str_eq(ha_state, DEADSTATUS)){
	crm_debug_2("Node is down: ha_state=%s, ccm_state=%s",
	crm_str(ha_state), crm_str(ccm_state));

	} else if(!crm_is_true(ccm_state)
	\|\| safe_str_eq(ha_state, DEADSTATUS)) {

	} else if(safe_str_eq(crm_state, ONLINESTATUS)) {
	if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) {
	online = TRUE;
	} else {
	crm_debug("Node is not ready to run resources: %s", join_state);
	}

	} else if(this_node->details->expected_up == FALSE) {
	crm_debug_2("CRMd is down: ha_state=%s, ccm_state=%s",
	crm_str(ha_state), crm_str(ccm_state));
	crm_debug_2("\tcrm_state=%s, join_state=%s, expected=%s",
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));

	} else {
	/* mark it unclean */
	this_node->details->unclean = TRUE;

	crm_warn("Node %s is partially & un-expectedly down",
	this_node->details->uname);
	crm_info("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));
	}
	return online;
	}

	static gboolean
	determine_online_status_fencing(xmlNode * node_state, node_t *this_node)
	{
	gboolean online = FALSE;
	gboolean do_terminate = FALSE;
	const char *join_state = crm_element_value(node_state, XML_CIB_ATTR_JOINSTATE);
	const char *crm_state = crm_element_value(node_state, XML_CIB_ATTR_CRMDSTATE);
	const char *ccm_state = crm_element_value(node_state, XML_CIB_ATTR_INCCM);
	const char *ha_state = crm_element_value(node_state, XML_CIB_ATTR_HASTATE);
	const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE);
	const char *terminate = g_hash_table_lookup(this_node->details->attrs, "terminate");

	if(ha_state == NULL) {
	ha_state = DEADSTATUS;
	}

	if(crm_is_true(terminate)) {
	do_terminate = TRUE;

	} else if(terminate != NULL && strlen(terminate) > 0) {
	/* could be a time() value */
	char t = terminate[0];
	if(t != '0' && isdigit(t)) {
	do_terminate = TRUE;
	}
	}

	if(crm_is_true(ccm_state)
	&& safe_str_eq(ha_state, ACTIVESTATUS)
	&& safe_str_eq(crm_state, ONLINESTATUS)) {

	if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) {
	online = TRUE;
	if(do_terminate) {
	crm_notice("Forcing node %s to be terminated", this_node->details->uname);
	this_node->details->unclean = TRUE;
	this_node->details->shutdown = TRUE;
	}

	} else if(join_state == exp_state /* == NULL */) {
	crm_info("Node %s is coming up", this_node->details->uname);
	crm_debug("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));

	} else if(safe_str_eq(join_state, CRMD_JOINSTATE_PENDING)) {
	crm_info("Node %s is not ready to run resources",
	this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;
	online = TRUE;

	} else if(safe_str_eq(join_state, CRMD_JOINSTATE_NACK)) {
	crm_warn("Node %s is not part of the cluster",
	this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;
	online = TRUE;

	} else {
	crm_warn("Node %s (%s) is un-expectedly down",
	this_node->details->uname, this_node->details->id);
	crm_info("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));
	this_node->details->unclean = TRUE;

	}

	} else if(crm_is_true(ccm_state) == FALSE
	&& safe_str_eq(ha_state, DEADSTATUS)
	&& safe_str_eq(crm_state, OFFLINESTATUS)
	&& this_node->details->expected_up == FALSE) {
	crm_debug("Node %s is down: join_state=%s, expected=%s",
	this_node->details->uname,
	crm_str(join_state), crm_str(exp_state));

	#if 0
	/* While a nice optimization, it causes the cluster to block until the node
	* comes back online. Which is a serious problem if the cluster software
	* is not configured to start at boot or stonith is configured to merely
	* stop the node instead of restart it.
	* Easily triggered by setting terminate=true for the DC
	*/
	} else if(do_terminate) {
	crm_info("Node %s is %s after forced termination",
	this_node->details->uname, crm_is_true(ccm_state)?"coming up":"going down");
	crm_debug("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));

	if(crm_is_true(ccm_state) == FALSE) {
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;
	online = TRUE;
	}
	#endif

	} else if(this_node->details->expected_up) {
	/* mark it unclean */
	this_node->details->unclean = TRUE;

	crm_warn("Node %s (%s) is un-expectedly down",
	this_node->details->uname, this_node->details->id);
	crm_info("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));

	} else {
	crm_info("Node %s is down", this_node->details->uname);
	crm_debug("\tha_state=%s, ccm_state=%s,"
	" crm_state=%s, join_state=%s, expected=%s",
	crm_str(ha_state), crm_str(ccm_state),
	crm_str(crm_state), crm_str(join_state),
	crm_str(exp_state));
	}
	return online;
	}

	gboolean
	determine_online_status(
	xmlNode * node_state, node_t this_node, pe_working_set_t data_set)
	{
	gboolean online = FALSE;
	const char *shutdown = NULL;
	const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE);

	if(this_node == NULL) {
	crm_config_err("No node to check");
	return online;
	}

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;
	shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN);

	if(shutdown != NULL && safe_str_neq("0", shutdown)) {
	this_node->details->shutdown = TRUE;

	} else if(safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
	this_node->details->expected_up = TRUE;
	}

	if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
	online = determine_online_status_no_fencing(
	node_state, this_node);

	} else {
	online = determine_online_status_fencing(
	node_state, this_node);
	}

	if(online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if(online && this_node->details->shutdown) {
	/* dont run resources here */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if(this_node->details->unclean) {
	pe_proc_warn("Node %s is unclean", this_node->details->uname);

	} else if(this_node->details->online) {
	crm_info("Node %s is %s", this_node->details->uname,
	this_node->details->shutdown?"shutting down":
	this_node->details->pending?"pending":
	this_node->details->standby?"standby":"online");

	} else {
	crm_debug_2("Node %s is offline", this_node->details->uname);
	}

	return online;
	}

	#define set_char(x) last_rsc_id[lpc] = x; complete = TRUE;

	static char *
	clone_zero(const char *last_rsc_id)
	{
	int lpc = 0;
	char *zero = NULL;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	if(last_rsc_id != NULL) {
	lpc = strlen(last_rsc_id);
	}

	while(--lpc > 0) {
	switch(last_rsc_id[lpc]) {
	case 0:
	return NULL;
	break;
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	crm_malloc0(zero, lpc + 3);
	memcpy(zero, last_rsc_id, lpc);
	zero[lpc] = ':';
	zero[lpc+1] = '0';
	zero[lpc+2] = 0;
	return zero;
	}
	}
	return NULL;
	}


	static char *
	increment_clone(char *last_rsc_id)
	{
	int lpc = 0;
	int len = 0;
	char *tmp = NULL;
	gboolean complete = FALSE;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	if(last_rsc_id != NULL) {
	len = strlen(last_rsc_id);
	}

	lpc = len-1;
	while(complete == FALSE && lpc > 0) {
	switch (last_rsc_id[lpc]) {
	case 0:
	lpc--;
	break;
	case '0':
	set_char('1');
	break;
	case '1':
	set_char('2');
	break;
	case '2':
	set_char('3');
	break;
	case '3':
	set_char('4');
	break;
	case '4':
	set_char('5');
	break;
	case '5':
	set_char('6');
	break;
	case '6':
	set_char('7');
	break;
	case '7':
	set_char('8');
	break;
	case '8':
	set_char('9');
	break;
	case '9':
	last_rsc_id[lpc] = '0';
	lpc--;
	break;
	case ':':
	tmp = last_rsc_id;
	crm_malloc0(last_rsc_id, len + 2);
	memcpy(last_rsc_id, tmp, len);
	last_rsc_id[++lpc] = '1';
	last_rsc_id[len] = '0';
	last_rsc_id[len+1] = 0;
	complete = TRUE;
	crm_free(tmp);
	break;
	default:
	crm_err("Unexpected char: %c (%d)",
	last_rsc_id[lpc], lpc);
	break;
	}
	}
	return last_rsc_id;
	}

	static resource_t *
	create_fake_resource(const char rsc_id, xmlNode rsc_entry, pe_working_set_t *data_set)
	{
	resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
	crm_log_xml_info(xml_rsc, "Orphan resource");

	common_unpack(xml_rsc, &rsc, NULL, data_set);
	set_bit(rsc->flags, pe_rsc_orphan);

	data_set->resources = g_list_append(data_set->resources, rsc);
	return rsc;
	}

	extern resource_t create_child_clone(resource_t rsc, int sub_id, pe_working_set_t *data_set);

	static resource_t find_clone(pe_working_set_t data_set, node_t node, resource_t parent, const char *rsc_id)
	{
	int len = 0;
	resource_t *rsc = NULL;
	char *base = clone_zero(rsc_id);
	char *alt_rsc_id = crm_strdup(rsc_id);

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(parent->variant == pe_clone \|\| parent->variant == pe_master);

	if(base) {
	len = strlen(base);
	}
	if(len > 0) {
	base[len-1] = 0;
	}

	crm_debug_3("Looking for %s on %s in %s %d",
	rsc_id, node->details->uname, parent->id, is_set(parent->flags, pe_rsc_unique));

	if(is_set(parent->flags, pe_rsc_unique)) {
	crm_debug_3("Looking for %s", rsc_id);
	rsc = parent->fns->find_rsc(parent, rsc_id, FALSE, FALSE, NULL, TRUE);

	} else {
	rsc = parent->fns->find_rsc(parent, base, FALSE, TRUE, node, TRUE);
	if(rsc != NULL && rsc->running_on) {
	rsc = NULL;
	crm_debug_3("Looking for an existing orphan for %s: %s on %s", parent->id, rsc_id, node->details->uname);

	/* There is already an instance of this _anonymous_ clone active on "node".
	*
	* If there is a partially active orphan (only applies to clone groups) on
	* the same node, use that.
	* Otherwise create a new (orphaned) instance at "orphan_check:".
	*/
	slist_iter(child, resource_t, parent->children, lpc,
	node_t *loc = child->fns->location(child, NULL, TRUE);
	if(loc && loc->details == node->details) {
	resource_t *tmp = child->fns->find_rsc(child, base, FALSE, TRUE, NULL, TRUE);
	if(tmp && tmp->running_on == NULL) {
	rsc = tmp;
	break;
	}
	}
	);

	goto orphan_check;
	}

	while(rsc == NULL) {
	crm_debug_3("Trying %s", alt_rsc_id);
	rsc = parent->fns->find_rsc(parent, alt_rsc_id, FALSE, FALSE, NULL, TRUE);
	if(rsc == NULL) {
	break;

	} else if(rsc->running_on == NULL) {
	break;
	}
	alt_rsc_id = increment_clone(alt_rsc_id);
	rsc = NULL;
	}
	}

	orphan_check:
	if(rsc == NULL) {
	/* Create an extra orphan */
	resource_t *top = create_child_clone(parent, -1, data_set);
	crm_debug("Created orphan for %s: %s on %s", parent->id, rsc_id, node->details->uname);
	rsc = top->fns->find_rsc(top, base, FALSE, TRUE, NULL, TRUE);
	CRM_ASSERT(rsc != NULL);
	}

	crm_free(rsc->clone_name); rsc->clone_name = NULL;
	if(safe_str_neq(rsc_id, rsc->id)) {
	crm_info("Internally renamed %s on %s to %s%s",
	rsc_id, node->details->uname, rsc->id,
	is_set(rsc->flags, pe_rsc_orphan)?" (ORPHAN)":"");
	rsc->clone_name = crm_strdup(rsc_id);
	}

	crm_free(alt_rsc_id);
	crm_free(base);
	return rsc;
	}

	static resource_t *
	unpack_find_resource(
	pe_working_set_t data_set, node_t node, const char rsc_id, xmlNode rsc_entry)
	{
	resource_t *rsc = NULL;
	resource_t *clone_parent = NULL;
	char *alt_rsc_id = crm_strdup(rsc_id);

	crm_debug_2("looking for %s", rsc_id);

	rsc = pe_find_resource(data_set->resources, alt_rsc_id);
	/* no match */
	if(rsc == NULL) {
	/* Even when clone-max=0, we still create a single :0 orphan to match against */
	char *tmp = clone_zero(alt_rsc_id);
	resource_t *clone0 = pe_find_resource(data_set->resources, tmp);
	clone_parent = uber_parent(clone0);
	crm_free(tmp);

	crm_debug_2("%s not found: %s", alt_rsc_id, clone_parent?clone_parent->id:"orphan");

	} else {
	clone_parent = uber_parent(rsc);
	}

	if(clone_parent && clone_parent->variant > pe_group) {
	rsc = find_clone(data_set, node, clone_parent, rsc_id);
	CRM_ASSERT(rsc != NULL);
	}

	crm_free(alt_rsc_id);
	return rsc;
	}

	static resource_t *
	process_orphan_resource(xmlNode rsc_entry, node_t node, pe_working_set_t *data_set)
	{
	resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	crm_config_warn("Nothing known about resource %s running on %s",
	rsc_id, node->details->uname);
	rsc = create_fake_resource(rsc_id, rsc_entry, data_set);

	if(is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
	clear_bit(rsc->flags, pe_rsc_managed);

	} else {
	crm_info("Making sure orphan %s is stopped", rsc_id);

	print_resource(LOG_DEBUG_3, "Added orphan", rsc, FALSE);

	CRM_CHECK(rsc != NULL, return NULL);
	resource_location(rsc, NULL, -INFINITY, "__orphan_dont_run__", data_set);
	}
	return rsc;
	}

	static void
	process_rsc_state(resource_t rsc, node_t node,
	enum action_fail_response on_fail,
	xmlNode *migrate_op,
	pe_working_set_t *data_set)
	{
	if(on_fail == action_migrate_failure) {
	node_t *from = NULL;
	const char *uuid = crm_element_value(migrate_op, CRMD_ACTION_MIGRATED);

	on_fail = action_fail_recover;

	from = pe_find_node_id(data_set->nodes, uuid);
	if(from != NULL) {
	process_rsc_state(rsc, from, on_fail, NULL, data_set);
	} else {
	crm_log_xml_err(migrate_op, "Bad Op");
	}
	}

	crm_debug_2("Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role),
	node->details->uname, fail2text(on_fail));

	/* process current state */
	if(rsc->role != RSC_ROLE_UNKNOWN) {
	rsc->known_on = g_list_append(rsc->known_on, node);
	}

	if(node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = action_fail_ignore;
	}

	switch(on_fail) {
	case action_fail_ignore:
	/* nothing to do */
	break;

	case action_fail_fence:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	node->details->unclean = TRUE;
	break;

	case action_fail_standby:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case action_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	clear_bit(rsc->flags, pe_rsc_managed);
	break;

	case action_fail_migrate:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY,
	"__action_migration_auto__",data_set);
	break;

	case action_fail_stop:
	rsc->next_role = RSC_ROLE_STOPPED;
	break;

	case action_fail_recover:
	if(rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	set_bit(rsc->flags, pe_rsc_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case action_migrate_failure:
	/* anything extra? */
	break;
	}

	if(rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	native_add_running(rsc, node, data_set);
	if(on_fail != action_fail_ignore) {
	set_bit(rsc->flags, pe_rsc_failed);
	}

	} else if(rsc->clone_name) {
	crm_debug_2("Resetting clone_name %s for %s (stopped)",
	rsc->clone_name, rsc->id);
	crm_free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	char *key = stop_key(rsc);
	GListPtr possible_matches = find_actions(rsc->actions, key, node);
	slist_iter(stop, action_t, possible_matches, lpc,
	stop->optional = TRUE;
	);
	crm_free(key);
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(node_t node, resource_t rsc,
	int start_index, int stop_index,
	GListPtr sorted_op_list, pe_working_set_t *data_set)
	{
	const char *task = NULL;
	const char *status = NULL;

	crm_debug_3("%s: Start index %d, stop index = %d",
	rsc->id, start_index, stop_index);
	slist_iter(rsc_op, xmlNode, sorted_op_list, lpc,
	int interval = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);
	const char *interval_s = NULL;
	if(node->details->online == FALSE) {
	crm_debug_4("Skipping %s/%s: node is offline",
	rsc->id, node->details->uname);
	break;

	} else if(start_index < stop_index) {
	crm_debug_4("Skipping %s/%s: not active",
	rsc->id, node->details->uname);
	break;

	} else if(lpc <= start_index) {
	crm_debug_4("Skipping %s/%s: old",
	id, node->details->uname);
	continue;
	}

	interval_s = crm_element_value(rsc_op,XML_LRM_ATTR_INTERVAL);
	interval = crm_parse_int(interval_s, "0");
	if(interval == 0) {
	crm_debug_4("Skipping %s/%s: non-recurring",
	id, node->details->uname);
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if(safe_str_eq(status, "-1")) {
	crm_debug_4("Skipping %s/%s: status",
	id, node->details->uname);
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = generate_op_key(rsc->id, task, interval);
	crm_debug_3("Creating %s/%s", key, node->details->uname);
	custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
	);
	}

	void
	calculate_active_ops(GListPtr sorted_op_list, int start_index, int stop_index)
	{
	const char *task = NULL;
	const char *status = NULL;

	*stop_index = -1;
	*start_index = -1;

	slist_iter(
	rsc_op, xmlNode, sorted_op_list, lpc,

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if(safe_str_eq(task, CRMD_ACTION_STOP)
	&& safe_str_eq(status, "0")) {
	*stop_index = lpc;

	} else if(safe_str_eq(task, CRMD_ACTION_START)) {
	*start_index = lpc;

	} else if(start_index <= stop_index
	&& safe_str_eq(task, CRMD_ACTION_STATUS)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
	if(safe_str_eq(rc, "0") \|\| safe_str_eq(rc, "8")) {
	*start_index = lpc;
	}
	}
	);
	}

	static void
	unpack_lrm_rsc_state(
	node_t node, xmlNode rsc_entry, pe_working_set_t *data_set)
	{
	int stop_index = -1;
	int start_index = -1;

	const char *task = NULL;
	const char *value = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	resource_t *rsc = NULL;
	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;

	xmlNode *migrate_op = NULL;

	enum action_fail_response on_fail = FALSE;
	enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;

	crm_debug_3("[%s] Processing %s on %s",
	crm_element_name(rsc_entry), rsc_id, node->details->uname);

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	xml_child_iter_filter(
	rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP,
	op_list = g_list_append(op_list, rsc_op);
	);

	if(op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return;
	}

	/* find the resource */
	rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
	if(rsc == NULL) {
	rsc = process_orphan_resource(rsc_entry, node, data_set);
	}
	CRM_ASSERT(rsc != NULL);

	/* process operations */
	saved_role = rsc->role;
	on_fail = action_fail_ignore;
	rsc->role = RSC_ROLE_UNKNOWN;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	slist_iter(
	rsc_op, xmlNode, sorted_op_list, lpc,

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	if(safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	migrate_op = rsc_op;
	}

	unpack_rsc_op(rsc, node, rsc_op, &on_fail, data_set);
	);

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index,
	sorted_op_list, data_set);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail, migrate_op, data_set);

	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	if(value != NULL && safe_str_neq("default", value)) {
	enum rsc_role_e req_role = text2role(value);
	if(req_role != RSC_ROLE_UNKNOWN && req_role != rsc->next_role){
	if(rsc->next_role != RSC_ROLE_UNKNOWN) {
	crm_debug("%s: Overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role),
	role2text(req_role));
	}
	rsc->next_role = req_role;
	}
	}

	if(saved_role > rsc->role) {
	rsc->role = saved_role;
	}
	}

	gboolean
	unpack_lrm_resources(node_t node, xmlNode lrm_rsc_list, pe_working_set_t *data_set)
	{
	CRM_CHECK(node != NULL, return FALSE);

	crm_debug_3("Unpacking resources on %s", node->details->uname);

	xml_child_iter_filter(
	lrm_rsc_list, rsc_entry, XML_LRM_TAG_RESOURCE,
	unpack_lrm_rsc_state(node, rsc_entry, data_set);
	);

	return TRUE;
	}

	static void set_active(resource_t *rsc)
	{
	resource_t *top = uber_parent(rsc);
	if(top && top->variant == pe_master) {
	rsc->role = RSC_ROLE_SLAVE;
	} else {
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	gboolean
	unpack_rsc_op(resource_t rsc, node_t node, xmlNode *xml_op,
	enum action_fail_response on_fail, pe_working_set_t data_set)
	{
	const char *id = NULL;
	const char *key = NULL;
	const char *task = NULL;
	const char *magic = NULL;
	const char *task_id = NULL;
	const char *actual_rc = NULL;
	/* const char target_rc = NULL; /
	const char *task_status = NULL;
	const char *interval_s = NULL;
	const char *op_digest = NULL;
	const char *op_version = NULL;

	int interval = 0;
	int task_status_i = -2;
	int actual_rc_i = 0;
	int target_rc = -1;

	action_t *action = NULL;
	node_t *effective_node = NULL;
	resource_t *failed = NULL;

	gboolean expired = FALSE;
	gboolean is_probe = FALSE;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(node != NULL, return FALSE);
	CRM_CHECK(xml_op != NULL, return FALSE);

	id = ID(xml_op);
	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	task_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	task_status = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
	op_digest = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
	key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);

	CRM_CHECK(id != NULL, return FALSE);
	CRM_CHECK(task != NULL, return FALSE);
	CRM_CHECK(task_status != NULL, return FALSE);

	task_status_i = crm_parse_int(task_status, NULL);

	CRM_CHECK(task_status_i <= LRM_OP_ERROR, return FALSE);
	CRM_CHECK(task_status_i >= LRM_OP_PENDING, return FALSE);

	if(safe_str_eq(task, CRMD_ACTION_NOTIFY)) {
	/* safe to ignore these */
	return TRUE;
	}

	if(rsc->failure_timeout > 0) {
	int last_run = 0;

	if(crm_element_value_int(xml_op, "last-run", &last_run) == 0) {
	/* int last_change = crm_element_value_int(xml_op, "last_rc_change"); */
	time_t now = get_timet_now(data_set);
	if(now > (last_run + rsc->failure_timeout)) {
	expired = TRUE;
	}
	}
	}

	crm_debug_2("Unpacking task %s/%s (call_id=%s, status=%s) on %s (role=%s)",
	id, task, task_id, task_status, node->details->uname,
	role2text(rsc->role));

	interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
	interval = crm_parse_int(interval_s, "0");

	if(interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	is_probe = TRUE;
	}

	if(node->details->unclean) {
	crm_debug_2("Node %s (where %s is running) is unclean."
	" Further action depends on the value of the stop's on-fail attribue",
	node->details->uname, rsc->id);
	}

	actual_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
	CRM_CHECK(actual_rc != NULL, return FALSE);
	actual_rc_i = crm_parse_int(actual_rc, NULL);

	if(key) {
	int dummy = 0;
	char *dummy_string = NULL;
	decode_transition_key(key, &dummy_string, &dummy, &dummy, &target_rc);
	crm_free(dummy_string);
	}

	if(task_status_i == LRM_OP_DONE && target_rc >= 0) {
	if(target_rc == actual_rc_i) {
	task_status_i = LRM_OP_DONE;

	} else {
	task_status_i = LRM_OP_ERROR;
	crm_info("%s on %s returned %d (%s) instead of the expected value: %d (%s)",
	id, node->details->uname,
	actual_rc_i, execra_code2string(actual_rc_i),
	target_rc, execra_code2string(target_rc));
	}

	} else if(task_status_i == LRM_OP_ERROR) {
	/* let us decide that */
	task_status_i = LRM_OP_DONE;
	}

	if(task_status_i == LRM_OP_NOTSUPPORTED) {
	actual_rc_i = EXECRA_UNIMPLEMENT_FEATURE;
	}

	if(expired
	&& actual_rc_i != EXECRA_NOT_RUNNING
	&& actual_rc_i != EXECRA_RUNNING_MASTER
	&& actual_rc_i != EXECRA_OK) {
	crm_notice("Ignoring expired failure %s (rc=%d, magic=%s) on %s",
	id, actual_rc_i, magic, node->details->uname);
	goto done;
	}


	/* we could clean this up significantly except for old LRMs and CRMs that
	* didnt include target_rc and liked to remap status
	*/
	switch(actual_rc_i) {
	case EXECRA_NOT_RUNNING:
	if(is_probe \|\| target_rc == actual_rc_i) {
	task_status_i = LRM_OP_DONE;
	rsc->role = RSC_ROLE_STOPPED;

	/* clear any previous failure actions */
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;

	} else if(safe_str_neq(task, CRMD_ACTION_STOP)) {
	task_status_i = LRM_OP_ERROR;
	}
	break;

	case EXECRA_RUNNING_MASTER:
	if(is_probe) {
	task_status_i = LRM_OP_DONE;
	crm_notice("Operation %s found resource %s active in master mode on %s",
	id, rsc->id, node->details->uname);

	} else if(target_rc == actual_rc_i) {
	/* nothing to do */

	} else if(target_rc >= 0) {
	task_status_i = LRM_OP_ERROR;

	/* legacy code for pre-0.6.5 operations */
	} else if(safe_str_neq(task, CRMD_ACTION_STATUS)
	\|\| rsc->role != RSC_ROLE_MASTER) {
	task_status_i = LRM_OP_ERROR;
	if(rsc->role != RSC_ROLE_MASTER) {
	crm_err("%s reported %s in master mode on %s",
	id, rsc->id,
	node->details->uname);
	}
	}
	rsc->role = RSC_ROLE_MASTER;
	break;

	case EXECRA_FAILED_MASTER:
	rsc->role = RSC_ROLE_MASTER;
	task_status_i = LRM_OP_ERROR;
	break;

	case EXECRA_UNIMPLEMENT_FEATURE:
	if(interval > 0) {
	task_status_i = LRM_OP_NOTSUPPORTED;
	break;
	}
	/* else: fall through */
	case EXECRA_INSUFFICIENT_PRIV:
	case EXECRA_NOT_INSTALLED:
	case EXECRA_INVALID_PARAM:
	effective_node = node;
	/* fall through */
	case EXECRA_NOT_CONFIGURED:
	failed = rsc;
	if(is_not_set(rsc->flags, pe_rsc_unique)) {
	failed = uber_parent(failed);
	}

	crm_err("Hard error - %s failed with rc=%d: Preventing %s from re-starting %s %s",
	id, actual_rc_i, failed->id,
	effective_node?"on":"anywhere",
	effective_node?effective_node->details->uname:"in the cluster");

	resource_location(failed, effective_node, -INFINITY, "hard-error", data_set);
	if(is_probe) {
	/* treat these like stops */
	task = CRMD_ACTION_STOP;
	task_status_i = LRM_OP_DONE;
	add_node_copy(data_set->failed, xml_op);
	}
	break;

	case EXECRA_OK:
	if(is_probe && target_rc == 7) {
	task_status_i = LRM_OP_DONE;
	crm_notice("Operation %s found resource %s active on %s",
	id, rsc->id, node->details->uname);

	/* legacy code for pre-0.6.5 operations */
	} else if(target_rc < 0
	&& interval > 0
	&& rsc->role == RSC_ROLE_MASTER) {
	/* catch status ops that return 0 instead of 8 while they
	* are supposed to be in master mode
	*/
	task_status_i = LRM_OP_ERROR;
	}

	break;

	default:
	if(task_status_i == LRM_OP_DONE) {
	crm_info("Remapping %s (rc=%d) on %s to an ERROR",
	id, actual_rc_i, node->details->uname);
	task_status_i = LRM_OP_ERROR;
	}
	}

	if(task_status_i == LRM_OP_ERROR
	\|\| task_status_i == LRM_OP_TIMEOUT
	\|\| task_status_i == LRM_OP_NOTSUPPORTED) {
	action = custom_action(rsc, crm_strdup(id), task, NULL, TRUE, FALSE, data_set);
	if(expired) {
	crm_notice("Ignoring expired failure (calculated) %s (rc=%d, magic=%s) on %s",
	id, actual_rc_i, magic, node->details->uname);
	goto done;

	} else if(action->on_fail == action_fail_ignore) {
	crm_warn("Remapping %s (rc=%d) on %s to DONE: ignore",
	id, actual_rc_i, node->details->uname);
	task_status_i = LRM_OP_DONE;
	}
	}

	switch(task_status_i) {
	case LRM_OP_PENDING:
	if(safe_str_eq(task, CRMD_ACTION_START)) {
	set_bit(rsc->flags, pe_rsc_start_pending);
	set_active(rsc);

	} else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;
	}
	break;

	case LRM_OP_DONE:
	crm_debug_3("%s/%s completed on %s",
	rsc->id, task, node->details->uname);

	if(actual_rc_i == EXECRA_NOT_RUNNING) {
	/* nothing to do */

	} else if(safe_str_eq(task, CRMD_ACTION_STOP)) {
	rsc->role = RSC_ROLE_STOPPED;

	/* clear any previous failure actions */
	switch(*on_fail) {
	case action_fail_block:
	case action_fail_stop:
	case action_fail_fence:
	case action_fail_migrate:
	case action_fail_standby:
	crm_debug_2("%s.%s is not cleared by a completed stop",
	rsc->id, fail2text(*on_fail));
	break;

	case action_fail_ignore:
	case action_fail_recover:
	case action_migrate_failure:
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;
	}

	} else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;

	} else if(safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	rsc->role = RSC_ROLE_SLAVE;

	} else if(rsc->role < RSC_ROLE_STARTED) {
	crm_debug_3("%s active on %s",
	rsc->id, node->details->uname);
	set_active(rsc);
	}
	break;

	case LRM_OP_ERROR:
	case LRM_OP_TIMEOUT:
	case LRM_OP_NOTSUPPORTED:
	crm_warn("Processing failed op %s on %s: %s",
	id, node->details->uname,
	execra_code2string(actual_rc_i));
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	add_node_copy(data_set->failed, xml_op);

	if(*on_fail < action->on_fail) {
	*on_fail = action->on_fail;
	}

	if(safe_str_eq(task, CRMD_ACTION_STOP)) {
	resource_location(
	rsc, node, -INFINITY, "__stop_fail__", data_set);

	} else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;

	} else if(safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	/*
	* staying in role=master ends up putting the PE/TE into a loop
	* setting role=slave is not dangerous because no master will be
	* promoted until the failed resource has been fully stopped
	*/
	crm_warn("Forcing %s to stop after a failed demote action", rsc->id);
	rsc->next_role = RSC_ROLE_STOPPED;
	rsc->role = RSC_ROLE_SLAVE;

	} else if(compare_version("2.0", op_version) > 0
	&& safe_str_eq(task, CRMD_ACTION_START)) {
	crm_warn("Compatibility handling for failed op %s on %s",
	id, node->details->uname);
	resource_location(
	rsc, node, -INFINITY, "__legacy_start__", data_set);
	}

	if(rsc->role < RSC_ROLE_STARTED) {
	set_active(rsc);
	}

	crm_debug_2("Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
	rsc->id, role2text(rsc->role),
	node->details->unclean?"true":"false",
	fail2text(action->on_fail),
	role2text(action->fail_role));

	if(action->fail_role != RSC_ROLE_STARTED
	&& rsc->next_role < action->fail_role) {
	rsc->next_role = action->fail_role;
	}

	if(action->fail_role == RSC_ROLE_STOPPED) {
	crm_err("Making sure %s doesn't come up again", rsc->id);
	/* make sure it doesnt come up again */
	pe_free_shallow_adv(rsc->allowed_nodes, TRUE);
	rsc->allowed_nodes = node_list_dup(
	data_set->nodes, FALSE, FALSE);
	slist_iter(
	node, node_t, rsc->allowed_nodes, lpc,
	node->weight = -INFINITY;
	);
	}

	pe_free_action(action);
	action = NULL;
	break;
	case LRM_OP_CANCELLED:
	/* do nothing?? */
	pe_err("Dont know what to do for cancelled ops yet");
	break;
	}

	done:
	crm_debug_3("Resource %s after %s: role=%s",
	rsc->id, task, role2text(rsc->role));

	pe_free_action(action);

	return TRUE;
	}

	gboolean
	add_node_attrs(xmlNode xml_obj, node_t node, gboolean overwrite, pe_working_set_t *data_set)
	{
	g_hash_table_insert(node->details->attrs,
	crm_strdup("#"XML_ATTR_UNAME),
	crm_strdup(node->details->uname));
	g_hash_table_insert(node->details->attrs,
	crm_strdup("#"XML_ATTR_ID),
	crm_strdup(node->details->id));
	if(safe_str_eq(node->details->id, data_set->dc_uuid)) {
	data_set->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	crm_strdup("#"XML_ATTR_DC),
	crm_strdup(XML_BOOLEAN_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	crm_strdup("#"XML_ATTR_DC),
	crm_strdup(XML_BOOLEAN_FALSE));
	}

	unpack_instance_attributes(
	data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL,
	node->details->attrs, NULL, overwrite, data_set->now);

	return TRUE;
	}

	static GListPtr
	extract_operations(const char node, const char rsc, xmlNode *rsc_entry, gboolean active_filter)
	{
	int stop_index = -1;
	int start_index = -1;

	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	xml_child_iter_filter(
	rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP,
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
	op_list = g_list_append(op_list, rsc_op);
	);

	if(op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if(active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	slist_iter(rsc_op, xmlNode, sorted_op_list, lpc,
	if(start_index < stop_index) {
	crm_debug_4("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if(lpc < start_index) {
	crm_debug_4("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	);

	g_list_free(sorted_op_list);
	return op_list;
	}

	GListPtr find_operations(
	const char rsc, const char node, gboolean active_filter, pe_working_set_t *data_set)
	{
	GListPtr output = NULL;
	GListPtr intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);

	const char *uname = NULL;
	node_t *this_node = NULL;

	xml_child_iter_filter(
	status, node_state, XML_CIB_TAG_STATE,

	uname = crm_element_value(node_state, XML_ATTR_UNAME);
	if(node != NULL && safe_str_neq(uname, node)) {
	continue;
	}

	this_node = pe_find_node(data_set->nodes, uname);
	CRM_CHECK(this_node != NULL, continue);

	determine_online_status(node_state, this_node, data_set);

	if(this_node->details->online \|\| is_set(data_set->flags, pe_flag_stonith_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	xml_child_iter_filter(
	tmp, lrm_rsc, XML_LRM_TAG_RESOURCE,
	const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);
	if(rsc != NULL && safe_str_neq(rsc_id, rsc)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	);
	}
	);

	return output;
	}
	diff --git a/pengine/native.c b/pengine/native.c
	index 654e087813..65a15db4ef 100644
	--- a/pengine/native.c
	+++ b/pengine/native.c
	@@ -1,2116 +1,2121 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <crm_internal.h>

	#include <pengine.h>
	#include <crm/pengine/rules.h>
	#include <lib/pengine/utils.h>
	#include <crm/msg_xml.h>
	#include <allocate.h>
	#include <utils.h>

	#define DELETE_THEN_REFRESH 1 /* The crmd will remove the resource from the CIB itself, making this redundant */

	#define VARIANT_NATIVE 1
	#include <lib/pengine/variant.h>

	void native_rsc_colocation_rh_must(resource_t *rsc_lh, gboolean update_lh,
	resource_t *rsc_rh, gboolean update_rh);

	void native_rsc_colocation_rh_mustnot(resource_t *rsc_lh, gboolean update_lh,
	resource_t *rsc_rh, gboolean update_rh);

	void Recurring(resource_t rsc, action_t start, node_t *node,
	pe_working_set_t *data_set);
	void RecurringOp(resource_t rsc, action_t start, node_t *node,
	xmlNode operation, pe_working_set_t data_set);
	void pe_post_notify(
	resource_t rsc, node_t node, action_t *op,
	notify_data_t n_data, pe_working_set_t data_set);

	void NoRoleChange (resource_t rsc, node_t current, node_t next, pe_working_set_t data_set);
	gboolean DeleteRsc (resource_t rsc, node_t node, gboolean optional, pe_working_set_t *data_set);
	gboolean StopRsc (resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set);
	gboolean StartRsc (resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set);
	gboolean DemoteRsc (resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set);
	gboolean PromoteRsc(resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set);
	gboolean RoleError (resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set);
	gboolean NullOp (resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set);

	enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* Current State */
	/* Next State: Unknown Stopped Started Slave Master */
	/* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, },
	/* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, },
	/* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	/* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_UNKNOWN, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	/* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_UNKNOWN, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	};

	gboolean (rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(resource_t,node_t,gboolean,pe_working_set_t) = {
	/* Current State */
	/* Next State: Unknown Stopped Started Slave Master */
	/* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
	/* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
	/* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
	/* Slave */ { RoleError, StopRsc, RoleError, NullOp, PromoteRsc, },
	/* Master */ { RoleError, RoleError, RoleError, DemoteRsc, NullOp, },
	};


	static gboolean
	native_choose_node(resource_t *rsc)
	{
	/*
	1. Sort by weight
	2. color.chosen_node = the node (of those with the highest wieght)
	with the fewest resources
	3. remove color.chosen_node from all other colors
	*/
	GListPtr nodes = NULL;
	node_t *chosen = NULL;

	int lpc = 0;
	int multiple = 0;
	int length = g_list_length(rsc->allowed_nodes);

	if(is_not_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to?TRUE:FALSE;
	}

	crm_debug_3("Choosing node for %s from %d candidates",
	rsc->id, length);

	if(rsc->allowed_nodes) {
	rsc->allowed_nodes = g_list_sort(rsc->allowed_nodes, sort_node_weight);
	nodes = rsc->allowed_nodes;
	chosen = g_list_nth_data(nodes, 0);

	if(chosen
	&& chosen->weight > 0
	&& can_run_resources(chosen)) {
	node_t *running = g_list_nth_data(rsc->running_on, 0);
	if(can_run_resources(running) == FALSE) {
	running = NULL;
	}

	for(lpc = 1; lpc < length; lpc++) {
	node_t *tmp = g_list_nth_data(nodes, lpc);
	if(tmp->weight == chosen->weight) {
	multiple++;
	if(running && tmp->details == running->details) {
	/* prefer the existing node if scores are equal */
	chosen = tmp;
	}
	}
	}
	}
	}

	if(multiple > 1) {
	int log_level = LOG_INFO;
	char *score = score2char(chosen->weight);
	if(chosen->weight >= INFINITY) {
	log_level = LOG_WARNING;
	}

	do_crm_log(log_level, "%d nodes with equal score (%s) for"
	" running %s resources. Chose %s.",
	multiple, score, rsc->id, chosen->details->uname);
	crm_free(score);
	}


	return native_assign_node(rsc, nodes, chosen, FALSE);
	}

	int node_list_attr_score(GListPtr list, const char attr, const char value)
	{
	int best_score = -INFINITY;
	const char *best_node = NULL;

	if(attr == NULL) {
	attr = "#"XML_ATTR_UNAME;
	}

	slist_iter(node, node_t, list, lpc,
	int weight = node->weight;
	if(can_run_resources(node) == FALSE) {
	weight = -INFINITY;
	}
	if(weight > best_score \|\| best_node == NULL) {
	const char *tmp = g_hash_table_lookup(node->details->attrs, attr);
	if(safe_str_eq(value, tmp)) {
	best_score = weight;
	best_node = node->details->uname;
	}
	}
	);

	if(safe_str_neq(attr, "#"XML_ATTR_UNAME)) {
	crm_info("Best score for %s=%s was %s with %d",
	attr, value, best_node?best_node:"<none>", best_score);
	}

	return best_score;
	}


	static void
	node_list_update(GListPtr list1, GListPtr list2, const char *attr, int factor)
	{
	int score = 0;
	if(attr == NULL) {
	attr = "#"XML_ATTR_UNAME;
	}

	slist_iter(
	node, node_t, list1, lpc,

	CRM_CHECK(node != NULL, continue);
	score = node_list_attr_score(list2, attr, g_hash_table_lookup(node->details->attrs, attr));

	if(factor < 0 && score < 0) {
	/* Negative preference for a node with a negative score
	* should not become a positive preference
	*
	* TODO: Decide if we want to filter only if weight == -INFINITY
	*
	*/
	continue;
	}
	crm_debug_2("%s: %d + %d*%d",
	node->details->uname, node->weight, factor, score);
	node->weight = merge_weights(factor*score, node->weight);
	);
	}

	GListPtr
	native_merge_weights(
	resource_t rsc, const char rhs, GListPtr nodes, const char *attr, int factor, gboolean allow_rollback)
	{
	GListPtr archive = NULL;

	if(is_set(rsc->flags, pe_rsc_merging)) {
	crm_info("%s: Breaking dependancy loop at %s", rhs, rsc->id);
	return nodes;

	} else if(is_not_set(rsc->flags, pe_rsc_provisional)) {
	crm_debug_4("%s: not provisional", rsc->id);
	return nodes;
	}

	set_bit(rsc->flags, pe_rsc_merging);
	crm_debug_2("%s: Combining scores from %s", rhs, rsc->id);

	if(allow_rollback) {
	archive = node_list_dup(nodes, FALSE, FALSE);
	}

	node_list_update(nodes, rsc->allowed_nodes, attr, factor);

	if(can_run_any(nodes) == FALSE) {
	if(archive) {
	crm_info("%s: Rolling back scores from %s", rhs, rsc->id);
	pe_free_shallow_adv(nodes, TRUE);
	nodes = archive;
	}
	goto bail;
	}

	pe_free_shallow_adv(archive, TRUE);

	slist_iter(
	constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,

	nodes = constraint->rsc_lh->cmds->merge_weights(
	constraint->rsc_lh, rhs, nodes,
	constraint->node_attribute,
	constraint->score/INFINITY, allow_rollback);
	);

	bail:
	clear_bit(rsc->flags, pe_rsc_merging);
	return nodes;
	}

	node_t *
	native_color(resource_t rsc, pe_working_set_t data_set)
	{
	int alloc_details = scores_log_level+1;
	if(rsc->parent && is_not_set(rsc->parent->flags, pe_rsc_allocating)) {
	/* never allocate children on their own */
	crm_debug("Escalating allocation of %s to its parent: %s",
	rsc->id, rsc->parent->id);
	rsc->parent->cmds->color(rsc->parent, data_set);
	}

	if(is_not_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to;
	}

	if(is_set(rsc->flags, pe_rsc_allocating)) {
	crm_debug("Dependancy loop detected involving %s", rsc->id);
	return NULL;
	}

	set_bit(rsc->flags, pe_rsc_allocating);
	print_resource(alloc_details, "Allocating: ", rsc, FALSE);
	dump_node_scores(alloc_details, rsc, "Pre-allloc", rsc->allowed_nodes);

	slist_iter(
	constraint, rsc_colocation_t, rsc->rsc_cons, lpc,

	resource_t *rsc_rh = constraint->rsc_rh;
	crm_debug_2("%s: Pre-Processing %s (%s)",
	rsc->id, constraint->id, rsc_rh->id);
	rsc_rh->cmds->color(rsc_rh, data_set);
	rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint);
	);

	dump_node_scores(alloc_details, rsc, "Post-coloc", rsc->allowed_nodes);

	slist_iter(
	constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,

	rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights(
	constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute, constraint->score/INFINITY, TRUE);
	);

	print_resource(LOG_DEBUG_2, "Allocating: ", rsc, FALSE);
	if(rsc->next_role == RSC_ROLE_STOPPED) {
	crm_debug_2("Making sure %s doesn't get allocated", rsc->id);
	/* make sure it doesnt come up again */
	resource_location(
	rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);
	}

	dump_node_scores(show_scores?0:scores_log_level, rsc, __PRETTY_FUNCTION__, rsc->allowed_nodes);
	+ if(is_set(data_set->flags, pe_flag_stonith_enabled)
	+ && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
	+ clear_bit(rsc->flags, pe_rsc_managed);
	+ }
	+
	if(is_not_set(rsc->flags, pe_rsc_managed)) {
	const char *reason = NULL;
	node_t *assign_to = NULL;
	if(rsc->running_on == NULL) {
	reason = "inactive";
	} else if(rsc->role == RSC_ROLE_MASTER) {
	assign_to = rsc->running_on->data;
	reason = "master";
	} else if(is_set(rsc->flags, pe_rsc_failed)) {
	reason = "failed";
	} else {
	assign_to = rsc->running_on->data;
	reason = "active";
	}
	crm_info("Unmanaged resource %s allocated to %s: %s", rsc->id,
	assign_to?assign_to->details->uname:"'nowhere'", reason);
	native_assign_node(rsc, NULL, assign_to, TRUE);

	} else if(is_set(data_set->flags, pe_flag_stop_everything)) {
	crm_debug("Forcing %s to stop", rsc->id);
	native_assign_node(rsc, NULL, NULL, TRUE);

	} else if(is_set(rsc->flags, pe_rsc_provisional)
	&& native_choose_node(rsc) ) {
	crm_debug_3("Allocated resource %s to %s",
	rsc->id, rsc->allocated_to->details->uname);

	} else if(rsc->allocated_to == NULL) {
	if(is_not_set(rsc->flags, pe_rsc_orphan)) {
	pe_warn("Resource %s cannot run anywhere", rsc->id);
	} else if(rsc->running_on != NULL) {
	crm_info("Stopping orphan resource %s", rsc->id);
	}

	} else {
	crm_debug("Pre-Allocated resource %s to %s",
	rsc->id, rsc->allocated_to->details->uname);
	}

	clear_bit(rsc->flags, pe_rsc_allocating);
	print_resource(LOG_DEBUG_3, "Allocated ", rsc, TRUE);

	return rsc->allocated_to;
	}

	static gboolean is_op_dup(
	resource_t rsc, const char name, const char *interval)
	{
	gboolean dup = FALSE;
	const char *id = NULL;
	const char *value = NULL;
	xml_child_iter_filter(
	rsc->ops_xml, operation, "op",
	value = crm_element_value(operation, "name");
	if(safe_str_neq(value, name)) {
	continue;
	}

	value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if(value == NULL) {
	value = "0";
	}

	if(safe_str_neq(value, interval)) {
	continue;
	}

	if(id == NULL) {
	id = ID(operation);

	} else {
	crm_config_err("Operation %s is a duplicate of %s", ID(operation), id);
	crm_config_err("Do not use the same (name, interval) combination more than once per resource");
	dup = TRUE;
	}
	);

	return dup;
	}

	void
	RecurringOp(resource_t rsc, action_t start, node_t *node,
	xmlNode operation, pe_working_set_t data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *interval = NULL;
	const char *node_uname = NULL;

	unsigned long long interval_ms = 0;
	action_t *mon = NULL;
	gboolean is_optional = TRUE;
	GListPtr possible_matches = NULL;

	crm_debug_2("Creating recurring action %s for %s in role %s",
	ID(operation), rsc->id, role2text(rsc->next_role));

	if(node != NULL) {
	node_uname = node->details->uname;
	}

	interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_get_interval(interval);

	if(interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if(is_op_dup(rsc, name, interval)) {
	return;
	}

	key = generate_op_key(rsc->id, name, interval_ms);
	if(find_rsc_op_entry(rsc, key) == NULL) {
	/* disabled */
	return;
	}

	if(start != NULL) {
	crm_debug_3("Marking %s %s due to %s",
	key, start->optional?"optional":"manditory",
	start->uuid);
	is_optional = start->optional;
	} else {
	crm_debug_2("Marking %s optional", key);
	is_optional = TRUE;
	}

	/* start a monitor for an already active resource */
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if(possible_matches == NULL) {
	is_optional = FALSE;
	crm_debug_3("Marking %s manditory: not active", key);
	} else {
	g_list_free(possible_matches);
	}

	value = crm_element_value(operation, "role");
	if((rsc->next_role == RSC_ROLE_MASTER && value == NULL)
	\|\| (value != NULL && text2role(value) != rsc->next_role)) {
	int log_level = LOG_DEBUG_2;
	const char *result = "Ignoring";
	if(is_optional) {
	char *local_key = crm_strdup(key);
	log_level = LOG_INFO;
	result = "Cancelling";
	/* its running : cancel it */

	mon = custom_action(
	rsc, local_key, RSC_CANCEL, node,
	FALSE, TRUE, data_set);

	crm_free(mon->task);
	mon->task = crm_strdup(RSC_CANCEL);
	add_hash_param(mon->meta, XML_LRM_ATTR_INTERVAL, interval);
	add_hash_param(mon->meta, XML_LRM_ATTR_TASK, name);

	local_key = NULL;

	switch(rsc->role) {
	case RSC_ROLE_SLAVE:
	case RSC_ROLE_STARTED:
	if(rsc->next_role == RSC_ROLE_MASTER) {
	local_key = promote_key(rsc);

	} else if(rsc->next_role == RSC_ROLE_STOPPED) {
	local_key = stop_key(rsc);
	}

	break;
	case RSC_ROLE_MASTER:
	local_key = demote_key(rsc);
	break;
	default:
	break;
	}

	if(local_key) {
	custom_action_order(rsc, NULL, mon, rsc, local_key, NULL,
	pe_order_runnable_left, data_set);
	}

	mon = NULL;
	}

	do_crm_log(log_level, "%s action %s (%s vs. %s)",
	result , key, value?value:role2text(RSC_ROLE_SLAVE),
	role2text(rsc->next_role));

	crm_free(key);
	key = NULL;
	return;
	}

	mon = custom_action(rsc, key, name, node,
	is_optional, TRUE, data_set);
	key = mon->uuid;
	if(is_optional) {
	crm_debug_2("%s\t %s (optional)",
	crm_str(node_uname), mon->uuid);
	}

	if(start == NULL \|\| start->runnable == FALSE) {
	crm_debug("%s\t %s (cancelled : start un-runnable)",
	crm_str(node_uname), mon->uuid);
	mon->runnable = FALSE;

	} else if(node == NULL
	\|\| node->details->online == FALSE
	\|\| node->details->unclean) {
	crm_debug("%s\t %s (cancelled : no node available)",
	crm_str(node_uname), mon->uuid);
	mon->runnable = FALSE;

	} else if(mon->optional == FALSE) {
	crm_notice(" Start recurring %s (%llus) for %s on %s", mon->task, interval_ms/1000, rsc->id, crm_str(node_uname));
	}

	if(rsc->next_role == RSC_ROLE_MASTER) {
	char *running_master = crm_itoa(EXECRA_RUNNING_MASTER);
	add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_master);
	crm_free(running_master);
	}

	if(node == NULL \|\| is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(rsc, start_key(rsc), NULL,
	NULL, crm_strdup(key), mon,
	pe_order_implies_right\|pe_order_runnable_left, data_set);

	if(rsc->next_role == RSC_ROLE_MASTER) {
	custom_action_order(
	rsc, promote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional\|pe_order_runnable_left, data_set);

	} else if(rsc->role == RSC_ROLE_MASTER) {
	custom_action_order(
	rsc, demote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional\|pe_order_runnable_left, data_set);
	}
	}
	}

	void
	Recurring(resource_t rsc, action_t start, node_t *node,
	pe_working_set_t *data_set)
	{
	if(is_not_set(data_set->flags, pe_flag_maintenance_mode)) {
	xml_child_iter_filter(
	rsc->ops_xml, operation, "op",
	RecurringOp(rsc, start, node, operation, data_set);
	);
	}
	}

	void native_create_actions(resource_t rsc, pe_working_set_t data_set)
	{
	action_t *start = NULL;
	node_t *chosen = NULL;
	enum rsc_role_e role = RSC_ROLE_UNKNOWN;
	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;

	crm_debug_2("Creating actions for %s", rsc->id);

	chosen = rsc->allocated_to;
	if(chosen != NULL) {
	CRM_CHECK(rsc->next_role != RSC_ROLE_UNKNOWN, rsc->next_role = RSC_ROLE_STARTED);
	}

	get_rsc_attributes(rsc->parameters, rsc, chosen, data_set);

	crm_debug_2("%s: %s->%s", rsc->id,
	role2text(rsc->role), role2text(rsc->next_role));

	if(g_list_length(rsc->running_on) > 1) {
	if(rsc->recovery_type == recovery_stop_start) {
	pe_proc_warn("Attempting recovery of resource %s", rsc->id);
	if(rsc->role == RSC_ROLE_MASTER) {
	DemoteRsc(rsc, NULL, FALSE, data_set);
	}
	StopRsc(rsc, NULL, FALSE, data_set);
	rsc->role = RSC_ROLE_STOPPED;
	}

	} else if(rsc->running_on != NULL) {
	node_t *current = rsc->running_on->data;
	NoRoleChange(rsc, current, chosen, data_set);

	} else if(rsc->role == RSC_ROLE_STOPPED && rsc->next_role == RSC_ROLE_STOPPED) {
	char *key = start_key(rsc);
	GListPtr possible_matches = find_actions(rsc->actions, key, NULL);
	slist_iter(
	action, action_t, possible_matches, lpc,
	action->optional = TRUE;
	/* action->pseudo = TRUE; */
	);
	g_list_free(possible_matches);
	crm_debug_2("Stopping a stopped resource");
	crm_free(key);
	goto do_recurring;

	} else if(rsc->role != RSC_ROLE_STOPPED) {
	/* A cheap trick to account for the fact that Master/Slave groups may not be
	* completely running when we set their role to Slave
	*/
	crm_debug_2("Resetting %s.role = %s (was %s)",
	rsc->id, role2text(RSC_ROLE_STOPPED), role2text(rsc->role));
	rsc->role = RSC_ROLE_STOPPED;
	}

	role = rsc->role;

	while(role != rsc->next_role) {
	next_role = rsc_state_matrix[role][rsc->next_role];
	crm_debug_2("Executing: %s->%s (%s)",
	role2text(role), role2text(next_role), rsc->id);
	if(rsc_action_matrix[role][next_role](
	rsc, chosen, FALSE, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}

	do_recurring:
	if(rsc->next_role != RSC_ROLE_STOPPED \|\| is_set(rsc->flags, pe_rsc_managed) == FALSE) {
	start = start_action(rsc, chosen, TRUE);
	Recurring(rsc, start, chosen, data_set);
	}
	}

	void native_internal_constraints(resource_t rsc, pe_working_set_t data_set)
	{
	int type = pe_order_optional;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);

	if(rsc->variant == pe_native) {
	type \|= pe_order_implies_right;
	}

	if(rsc->parent == NULL \|\| rsc->parent->variant == pe_group) {
	type \|= pe_order_restart;
	}

	new_rsc_order(rsc, RSC_STOP, rsc, RSC_START, type, data_set);
	new_rsc_order(rsc, RSC_DEMOTE, rsc, RSC_STOP, pe_order_demote_stop, data_set);
	new_rsc_order(rsc, RSC_START, rsc, RSC_PROMOTE, pe_order_runnable_left, data_set);
	new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START, pe_order_optional, data_set);

	if(is_not_set(rsc->flags, pe_rsc_managed)) {
	crm_debug_3("Skipping fencing constraints for unmanaged resource: %s", rsc->id);
	return;
	}

	if(rsc->variant == pe_native && safe_str_neq(class, "stonith")) {
	custom_action_order(
	rsc, stop_key(rsc), NULL,
	NULL, crm_strdup(all_stopped->task), all_stopped,
	pe_order_implies_right\|pe_order_runnable_left, data_set);
	}

	}

	void native_rsc_colocation_lh(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	if(rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", constraint->id);
	return;

	} else if(constraint->rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;
	}

	crm_debug_2("Processing colocation constraint between %s and %s",
	rsc_lh->id, rsc_rh->id);

	rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint);
	}

	static gboolean
	filter_colocation_constraint(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	int level = LOG_DEBUG_4;
	if(constraint->score == 0){
	return FALSE;
	}

	if(constraint->score > 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN
	&& constraint->role_lh != rsc_lh->next_role) {
	do_crm_log_unlikely(level, "LH: Skipping constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return FALSE;
	}

	if(constraint->score > 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN
	&& constraint->role_rh != rsc_rh->next_role) {
	do_crm_log_unlikely(level, "RH: Skipping constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return FALSE;
	}

	if(constraint->score < 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN
	&& constraint->role_lh == rsc_lh->next_role) {
	do_crm_log_unlikely(level, "LH: Skipping -ve constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return FALSE;
	}

	if(constraint->score < 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN
	&& constraint->role_rh == rsc_rh->next_role) {
	do_crm_log_unlikely(level, "RH: Skipping -ve constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return FALSE;
	}

	return TRUE;
	}

	static void
	colocation_match(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	const char *tmp = NULL;
	const char *value = NULL;
	gboolean do_check = FALSE;
	const char *attribute = "#id";

	if(constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	if(rsc_rh->allocated_to) {
	value = g_hash_table_lookup(
	rsc_rh->allocated_to->details->attrs, attribute);
	do_check = TRUE;

	} else if(constraint->score < 0) {
	/* nothing to do:
	* anti-colocation with something thats not running
	*/
	return;
	}

	slist_iter(
	node, node_t, rsc_lh->allowed_nodes, lpc,
	tmp = g_hash_table_lookup(node->details->attrs, attribute);
	if(do_check && safe_str_eq(tmp, value)) {
	if(constraint->score < INFINITY) {
	crm_debug_2("%s: %s.%s += %d", constraint->id, rsc_lh->id,
	node->details->uname, constraint->score);
	node->weight = merge_weights(
	constraint->score, node->weight);
	}

	} else if(do_check == FALSE \|\| constraint->score >= INFINITY) {
	crm_debug_2("%s: %s.%s -= %d (%s)", constraint->id, rsc_lh->id,
	node->details->uname, constraint->score, do_check?"failed":"unallocated");
	node->weight = merge_weights(-constraint->score, node->weight);
	}

	);
	}

	void native_rsc_colocation_rh(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	crm_debug_2("%sColocating %s with %s (%s, weight=%d)",
	constraint->score >= 0?"":"Anti-",
	rsc_lh->id, rsc_rh->id, constraint->id, constraint->score);

	if(filter_colocation_constraint(rsc_lh, rsc_rh, constraint) == FALSE) {
	return;
	}

	if(is_set(rsc_rh->flags, pe_rsc_provisional)) {
	return;

	} else if(is_not_set(rsc_lh->flags, pe_rsc_provisional)) {
	/* error check */
	struct node_shared_s *details_lh;
	struct node_shared_s *details_rh;
	if((constraint->score > -INFINITY) && (constraint->score < INFINITY)) {
	return;
	}

	details_rh = rsc_rh->allocated_to?rsc_rh->allocated_to->details:NULL;
	details_lh = rsc_lh->allocated_to?rsc_lh->allocated_to->details:NULL;

	if(constraint->score == INFINITY && details_lh != details_rh) {
	crm_err("%s and %s are both allocated"
	" but to different nodes: %s vs. %s",
	rsc_lh->id, rsc_rh->id,
	details_lh?details_lh->uname:"n/a",
	details_rh?details_rh->uname:"n/a");

	} else if(constraint->score == -INFINITY && details_lh == details_rh) {
	crm_err("%s and %s are both allocated"
	" but to the SAME node: %s",
	rsc_lh->id, rsc_rh->id,
	details_rh?details_rh->uname:"n/a");
	}

	return;

	} else {
	colocation_match(rsc_lh, rsc_rh, constraint);
	}
	}

	static GListPtr find_actions_by_task(GListPtr actions, resource_t rsc, const char original_key)
	{
	GListPtr list = NULL;

	list = find_actions(actions, original_key, NULL);
	if(list == NULL) {
	/* we're potentially searching a child of the original resource */
	char *key = NULL;
	char *tmp = NULL;
	char *task = NULL;
	int interval = 0;
	if(parse_op_key(original_key, &tmp, &task, &interval)) {
	key = generate_op_key(rsc->id, task, interval);
	/* crm_err("looking up %s instead of %s", key, original_key); */
	/* slist_iter(action, action_t, actions, lpc, */
	/* crm_err(" - %s", action->uuid)); */
	list = find_actions(actions, key, NULL);

	} else {
	crm_err("search key: %s", original_key);
	}

	crm_free(key);
	crm_free(tmp);
	crm_free(task);
	}

	return list;
	}

	void native_rsc_order_lh(resource_t lh_rsc, order_constraint_t order, pe_working_set_t *data_set)
	{
	GListPtr lh_actions = NULL;
	action_t *lh_action = order->lh_action;
	resource_t *rh_rsc = order->rh_rsc;

	crm_debug_3("Processing LH of ordering constraint %d", order->id);
	CRM_ASSERT(lh_rsc != NULL);

	if(lh_action != NULL) {
	lh_actions = g_list_append(NULL, lh_action);

	} else if(lh_action == NULL) {
	lh_actions = find_actions_by_task(
	lh_rsc->actions, lh_rsc, order->lh_action_task);
	}

	if(lh_actions == NULL && lh_rsc != rh_rsc) {
	char *key = NULL;
	char *rsc_id = NULL;
	char *op_type = NULL;
	int interval = 0;

	crm_debug_4("No LH-Side (%s/%s) found for constraint %d with %s - creating",
	lh_rsc->id, order->lh_action_task,
	order->id, order->rh_action_task);

	parse_op_key(
	order->lh_action_task, &rsc_id, &op_type, &interval);

	key = generate_op_key(lh_rsc->id, op_type, interval);

	lh_action = custom_action(lh_rsc, key, op_type,
	NULL, TRUE, TRUE, data_set);

	if(lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED
	&& safe_str_eq(op_type, RSC_STOP)) {
	lh_action->pseudo = TRUE;
	lh_action->runnable = TRUE;
	}

	lh_actions = g_list_append(NULL, lh_action);

	crm_free(op_type);
	crm_free(rsc_id);
	}

	slist_iter(
	lh_action_iter, action_t, lh_actions, lpc,

	if(rh_rsc == NULL && order->rh_action) {
	rh_rsc = order->rh_action->rsc;
	}
	if(rh_rsc) {
	rh_rsc->cmds->rsc_order_rh(
	lh_action_iter, rh_rsc, order);

	} else if(order->rh_action) {
	order_actions(
	lh_action_iter, order->rh_action, order->type);

	}
	);

	pe_free_shallow_adv(lh_actions, FALSE);
	}

	void native_rsc_order_rh(
	action_t lh_action, resource_t rsc, order_constraint_t *order)
	{
	GListPtr rh_actions = NULL;
	action_t *rh_action = NULL;

	CRM_CHECK(rsc != NULL, return);
	CRM_CHECK(order != NULL, return);

	rh_action = order->rh_action;
	crm_debug_3("Processing RH of ordering constraint %d", order->id);

	if(rh_action != NULL) {
	rh_actions = g_list_append(NULL, rh_action);

	} else if(rsc != NULL) {
	rh_actions = find_actions_by_task(
	rsc->actions, rsc, order->rh_action_task);
	}

	if(rh_actions == NULL) {
	crm_debug_4("No RH-Side (%s/%s) found for constraint..."
	" ignoring", rsc->id,order->rh_action_task);
	if(lh_action) {
	crm_debug_4("LH-Side was: %s", lh_action->uuid);
	}
	return;
	}

	slist_iter(
	rh_action_iter, action_t, rh_actions, lpc,

	if(lh_action) {
	order_actions(lh_action, rh_action_iter, order->type);

	} else if(order->type & pe_order_implies_right) {
	rh_action_iter->runnable = FALSE;
	crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, order->type);
	} else {
	crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, order->type);
	}

	);

	pe_free_shallow_adv(rh_actions, FALSE);
	}

	void native_rsc_location(resource_t rsc, rsc_to_node_t constraint)
	{
	GListPtr or_list;

	crm_debug_2("Applying %s (%s) to %s", constraint->id,
	role2text(constraint->role_filter), rsc->id);

	/* take "lifetime" into account */
	if(constraint == NULL) {
	pe_err("Constraint is NULL");
	return;

	} else if(rsc == NULL) {
	pe_err("LHS of rsc_to_node (%s) is NULL", constraint->id);
	return;

	} else if(constraint->role_filter > 0
	&& constraint->role_filter != rsc->next_role) {
	crm_debug("Constraint (%s) is not active (role : %s)",
	constraint->id, role2text(constraint->role_filter));
	return;

	} else if(is_active(constraint) == FALSE) {
	crm_debug_2("Constraint (%s) is not active", constraint->id);
	return;
	}

	if(constraint->node_list_rh == NULL) {
	crm_debug_2("RHS of constraint %s is NULL", constraint->id);
	return;
	}
	or_list = node_list_or(
	rsc->allowed_nodes, constraint->node_list_rh, FALSE);

	pe_free_shallow(rsc->allowed_nodes);
	rsc->allowed_nodes = or_list;
	slist_iter(node, node_t, or_list, lpc,
	crm_debug_3("%s + %s : %d", rsc->id, node->details->uname, node->weight);
	);
	}

	void native_expand(resource_t rsc, pe_working_set_t data_set)
	{
	crm_debug_3("Processing actions from %s", rsc->id);

	slist_iter(
	action, action_t, rsc->actions, lpc,
	crm_debug_4("processing action %d for rsc=%s",
	action->id, rsc->id);
	graph_element_from_action(action, data_set);
	);

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->expand(child_rsc, data_set);
	);
	}


	void
	LogActions(resource_t rsc, pe_working_set_t data_set)
	{
	node_t *next = NULL;
	node_t *current = NULL;
	gboolean moving = FALSE;

	if(rsc->children) {
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	LogActions(child_rsc, data_set);
	);
	return;
	}

	next = rsc->allocated_to;
	if(rsc->running_on) {
	current = rsc->running_on->data;
	if(rsc->role == RSC_ROLE_STOPPED) {
	/*
	* This can occur when resources are being recovered
	* We fiddle with the current role in native_create_actions()
	*/
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	if(current == NULL && is_set(rsc->flags, pe_rsc_orphan)) {
	/* Don't log stopped orphans */
	return;
	}

	if(is_not_set(rsc->flags, pe_rsc_managed)
	\|\| (current == NULL && next == NULL)) {
	crm_notice("Leave resource %s\t(%s%s)",
	rsc->id, role2text(rsc->role), is_not_set(rsc->flags, pe_rsc_managed)?" unmanaged":"");
	return;
	}

	if(current != NULL && next != NULL
	&& safe_str_neq(current->details->id, next->details->id)) {
	moving = TRUE;
	}

	if(rsc->role == rsc->next_role) {
	action_t *start = NULL;
	char *key = start_key(rsc);
	GListPtr possible_matches = find_actions(rsc->actions, key, next);
	crm_free(key);

	if(possible_matches) {
	start = possible_matches->data;
	g_list_free(possible_matches);
	}

	key = generate_op_key(rsc->id, CRMD_ACTION_MIGRATED, 0);
	possible_matches = find_actions(rsc->actions, key, next);
	crm_free(key);

	CRM_CHECK(next != NULL,);
	if(next == NULL) {
	} else if(possible_matches) {
	crm_notice("Migrate resource %s\t(%s %s -> %s)",
	rsc->id, role2text(rsc->role), current->details->uname, next->details->uname);
	g_list_free(possible_matches);

	} else if(start == NULL \|\| start->optional) {
	crm_notice("Leave resource %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next->details->uname);

	} else if(moving && current) {
	crm_notice("Move resource %s\t(%s %s -> %s)",
	rsc->id, role2text(rsc->role), current->details->uname, next->details->uname);

	} else if(is_set(rsc->flags, pe_rsc_failed)) {
	crm_notice("Recover resource %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next->details->uname);

	} else if(start && start->runnable == FALSE) {
	crm_notice("Stop resource %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next->details->uname);

	} else {
	crm_notice("Restart resource %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next->details->uname);
	}

	return;
	}

	if(rsc->role > RSC_ROLE_SLAVE && rsc->role > rsc->next_role) {
	CRM_CHECK(current != NULL,);
	if(current != NULL) {
	crm_notice("Demote %s\t(%s -> %s %s)", rsc->id,
	role2text(rsc->role), role2text(rsc->next_role),
	current->details->uname);
	}
	}

	if(rsc->next_role == RSC_ROLE_STOPPED \|\| moving) {
	CRM_CHECK(current != NULL,);
	slist_iter(node, node_t, rsc->running_on, lpc,
	crm_notice("Stop resource %s\t(%s)", rsc->id, node->details->uname));
	}

	if(rsc->role == RSC_ROLE_STOPPED \|\| moving) {
	CRM_CHECK(next != NULL,);
	if(next != NULL) {
	crm_notice("Start %s\t(%s)", rsc->id, next->details->uname);
	}
	}

	if(rsc->next_role > RSC_ROLE_SLAVE && rsc->role < rsc->next_role) {
	CRM_CHECK(next != NULL,);
	crm_notice("Promote %s\t(%s -> %s %s)", rsc->id,
	role2text(rsc->role), role2text(rsc->next_role),
	next->details->uname);
	}
	}

	void
	NoRoleChange(resource_t rsc, node_t current, node_t *next,
	pe_working_set_t *data_set)
	{
	action_t *stop = NULL;
	action_t *start = NULL;
	GListPtr possible_matches = NULL;

	crm_debug_2("Executing: %s (role=%s)", rsc->id, role2text(rsc->next_role));

	if(current == NULL \|\| next == NULL) {
	return;
	}

	if(is_set(rsc->flags, pe_rsc_failed)
	\|\| safe_str_neq(current->details->id, next->details->id)) {

	if(rsc->next_role > RSC_ROLE_STARTED) {
	gboolean optional = TRUE;
	if(rsc->role == RSC_ROLE_MASTER) {
	optional = FALSE;
	}
	DemoteRsc(rsc, current, optional, data_set);
	}
	if(rsc->role == RSC_ROLE_MASTER) {
	DemoteRsc(rsc, current, FALSE, data_set);
	}
	StopRsc(rsc, current, FALSE, data_set);
	StartRsc(rsc, next, FALSE, data_set);
	if(rsc->next_role == RSC_ROLE_MASTER) {
	PromoteRsc(rsc, next, FALSE, data_set);
	}

	possible_matches = find_recurring_actions(rsc->actions, next);
	slist_iter(match, action_t, possible_matches, lpc,
	if(match->optional == FALSE) {
	crm_debug("Fixing recurring action: %s",
	match->uuid);
	match->optional = TRUE;
	}
	);
	g_list_free(possible_matches);

	} else if(is_set(rsc->flags, pe_rsc_start_pending)) {
	start = start_action(rsc, next, TRUE);
	if(start->runnable) {
	/* wait for StartRsc() to be called */
	rsc->role = RSC_ROLE_STOPPED;
	} else {
	/* wait for StopRsc() to be called */
	rsc->next_role = RSC_ROLE_STOPPED;
	}

	} else {

	stop = stop_action(rsc, current, TRUE);
	start = start_action(rsc, next, TRUE);
	stop->optional = start->optional;
	if(rsc->next_role > RSC_ROLE_STARTED) {
	DemoteRsc(rsc, current, start->optional, data_set);
	}
	StopRsc(rsc, current, start->optional, data_set);
	StartRsc(rsc, current, start->optional, data_set);
	if(rsc->next_role == RSC_ROLE_MASTER) {
	PromoteRsc(rsc, next, start->optional, data_set);
	}

	if(start->runnable == FALSE) {
	rsc->next_role = RSC_ROLE_STOPPED;
	}
	}
	}


	gboolean
	StopRsc(resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set)
	{
	action_t *stop = NULL;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	crm_debug_2("Executing: %s", rsc->id);

	if(rsc->next_role == RSC_ROLE_STOPPED
	&& rsc->variant == pe_native
	&& safe_str_eq(class, "stonith")) {
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
	custom_action_order(
	NULL, crm_strdup(all_stopped->task), all_stopped,
	rsc, stop_key(rsc), NULL,
	pe_order_implies_left\|pe_order_stonith_stop, data_set);
	}

	slist_iter(
	current, node_t, rsc->running_on, lpc,
	stop = stop_action(rsc, current, optional);

	if(is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, current, optional, data_set);
	}
	);

	return TRUE;
	}


	gboolean
	StartRsc(resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set)
	{
	action_t *start = NULL;

	crm_debug_2("Executing: %s", rsc->id);
	start = start_action(rsc, next, TRUE);
	if(start->runnable && optional == FALSE) {
	start->optional = FALSE;
	}
	return TRUE;
	}

	gboolean
	PromoteRsc(resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set)
	{
	char *key = NULL;
	gboolean runnable = TRUE;
	GListPtr action_list = NULL;
	crm_debug_2("Executing: %s", rsc->id);

	CRM_CHECK(rsc->next_role == RSC_ROLE_MASTER,
	crm_err("Next role: %s", role2text(rsc->next_role));
	return FALSE);

	CRM_CHECK(next != NULL, return FALSE);

	key = start_key(rsc);
	action_list = find_actions_exact(rsc->actions, key, next);
	crm_free(key);

	slist_iter(start, action_t, action_list, lpc,
	if(start->runnable == FALSE) {
	runnable = FALSE;
	}
	);

	g_list_free(action_list);

	if(runnable) {
	promote_action(rsc, next, optional);
	return TRUE;
	}

	crm_debug("%s\tPromote %s (canceled)", next->details->uname, rsc->id);

	key = promote_key(rsc);
	action_list = find_actions_exact(rsc->actions, key, next);
	crm_free(key);

	slist_iter(promote, action_t, action_list, lpc,
	promote->runnable = FALSE;
	);

	g_list_free(action_list);
	return TRUE;
	}

	gboolean
	DemoteRsc(resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set)
	{
	crm_debug_2("Executing: %s", rsc->id);

	/* CRM_CHECK(rsc->next_role == RSC_ROLE_SLAVE, return FALSE); */
	slist_iter(
	current, node_t, rsc->running_on, lpc,
	demote_action(rsc, current, optional);
	);
	return TRUE;
	}

	gboolean
	RoleError(resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set)
	{
	crm_debug("Executing: %s", rsc->id);
	CRM_CHECK(FALSE, return FALSE);
	return FALSE;
	}

	gboolean
	NullOp(resource_t rsc, node_t next, gboolean optional, pe_working_set_t *data_set)
	{
	crm_debug_2("Executing: %s", rsc->id);
	return FALSE;
	}

	gboolean
	DeleteRsc(resource_t rsc, node_t node, gboolean optional, pe_working_set_t *data_set)
	{
	action_t *delete = NULL;
	#if DELETE_THEN_REFRESH
	action_t *refresh = NULL;
	#endif
	if(is_set(rsc->flags, pe_rsc_failed)) {
	crm_debug_2("Resource %s not deleted from %s: failed",
	rsc->id, node->details->uname);
	return FALSE;

	} else if(node == NULL) {
	crm_debug_2("Resource %s not deleted: NULL node", rsc->id);
	return FALSE;

	} else if(node->details->unclean \|\| node->details->online == FALSE) {
	crm_debug_2("Resource %s not deleted from %s: unrunnable",
	rsc->id, node->details->uname);
	return FALSE;
	}

	crm_notice("Removing %s from %s",
	rsc->id, node->details->uname);

	delete = delete_action(rsc, node, optional);

	new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE,
	optional?pe_order_implies_right:pe_order_implies_left, data_set);

	#if DELETE_THEN_REFRESH
	refresh = custom_action(
	NULL, crm_strdup(CRM_OP_LRM_REFRESH), CRM_OP_LRM_REFRESH,
	node, FALSE, TRUE, data_set);

	add_hash_param(refresh->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);

	order_actions(delete, refresh, pe_order_optional);
	#endif

	return TRUE;
	}

	gboolean
	native_create_probe(resource_t rsc, node_t node, action_t *complete,
	gboolean force, pe_working_set_t *data_set)
	{
	char *key = NULL;
	char *target_rc = NULL;
	action_t *probe = NULL;
	node_t *running = NULL;

	CRM_CHECK(node != NULL, return FALSE);

	if(rsc->children) {
	gboolean any_created = FALSE;

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	any_created = child_rsc->cmds->create_probe(
	child_rsc, node, complete, force, data_set) \|\| any_created;
	);

	return any_created;
	}

	if(is_set(rsc->flags, pe_rsc_orphan)) {
	crm_debug_2("Skipping orphan: %s", rsc->id);
	return FALSE;
	}

	running = pe_find_node_id(rsc->known_on, node->details->id);
	if(force == FALSE && running != NULL) {
	/* we already know the status of the resource on this node */
	crm_debug_3("Skipping active: %s", rsc->id);
	return FALSE;
	}

	key = generate_op_key(rsc->id, RSC_STATUS, 0);
	probe = custom_action(rsc, key, RSC_STATUS, node,
	FALSE, TRUE, data_set);
	probe->optional = FALSE;

	running = pe_find_node_id(rsc->running_on, node->details->id);
	if(running == NULL) {
	target_rc = crm_itoa(EXECRA_NOT_RUNNING);

	} else if(rsc->role == RSC_ROLE_MASTER) {
	target_rc = crm_itoa(EXECRA_RUNNING_MASTER);
	}

	if(target_rc != NULL) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, target_rc);
	crm_free(target_rc);
	}

	crm_debug("Probing %s on %s (%s)", rsc->id, node->details->uname, role2text(rsc->role));
	order_actions(probe, complete, pe_order_implies_right);

	return TRUE;
	}

	static void
	native_start_constraints(
	resource_t rsc, action_t stonith_op, gboolean is_stonith,
	pe_working_set_t *data_set)
	{
	node_t *target = stonith_op?stonith_op->node:NULL;

	if(is_stonith) {
	char *key = start_key(rsc);
	action_t *ready = get_pseudo_op(STONITH_UP, data_set);

	crm_debug_2("Ordering %s action before stonith events", key);
	custom_action_order(
	rsc, key, NULL,
	NULL, crm_strdup(ready->task), ready,
	pe_order_optional, data_set);

	} else {
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
	slist_iter(action, action_t, rsc->actions, lpc2,
	if(action->needs == rsc_req_stonith) {
	order_actions(all_stopped, action, pe_order_implies_left);

	} else if(target != NULL
	&& safe_str_eq(action->task, RSC_START)
	&& NULL == pe_find_node_id(
	rsc->known_on, target->details->id)) {
	/* if known == NULL, then we dont know if
	* the resource is active on the node
	* we're about to shoot
	*
	* in this case, regardless of action->needs,
	* the only safe option is to wait until
	* the node is shot before doing anything
	* to with the resource
	*
	* its analogous to waiting for all the probes
	* for rscX to complete before starting rscX
	*
	* the most likely explaination is that the
	* DC died and took its status with it
	*/

	crm_info("Ordering %s after %s recovery",
	action->uuid, target->details->uname);
	order_actions(all_stopped, action,
	pe_order_implies_left\|pe_order_runnable_left);
	}

	);
	}
	}

	static void
	native_stop_constraints(
	resource_t rsc, action_t stonith_op, gboolean is_stonith,
	pe_working_set_t *data_set)
	{
	char *key = NULL;
	GListPtr action_list = NULL;

	key = stop_key(rsc);
	action_list = find_actions(rsc->actions, key, stonith_op->node);
	crm_free(key);

	/* add the stonith OP as a stop pre-req and the mark the stop
	* as a pseudo op - since its now redundant
	*/

	slist_iter(
	action, action_t, action_list, lpc2,

	resource_t *parent = NULL;
	if(action->node->details->online
	&& action->node->details->unclean == FALSE
	&& is_set(rsc->flags, pe_rsc_failed)) {
	continue;
	}

	if(is_set(rsc->flags, pe_rsc_failed)) {
	crm_warn("Stop of failed resource %s is"
	" implicit after %s is fenced",
	rsc->id, action->node->details->uname);
	} else {
	crm_info("%s is implicit after %s is fenced",
	action->uuid, action->node->details->uname);
	}

	/* the stop would never complete and is
	* now implied by the stonith operation
	*/
	action->pseudo = TRUE;
	action->runnable = TRUE;
	action->implied_by_stonith = TRUE;

	if(is_stonith == FALSE) {
	order_actions(stonith_op, action, pe_order_optional);
	}

	if(is_set(rsc->flags, pe_rsc_notify)) {
	/* Create a second notification that will be delivered
	* immediately after the node is fenced
	*
	* Basic problem:
	* - C is a clone active on the node to be shot and stopping on another
	* - R is a resource that depends on C
	*
	* + C.stop depends on R.stop
	* + C.stopped depends on STONITH
	* + C.notify depends on C.stopped
	* + C.healthy depends on C.notify
	* + R.stop depends on C.healthy
	*
	* The extra notification here changes
	* + C.healthy depends on C.notify
	* into:
	* + C.healthy depends on C.notify'
	* + C.notify' depends on STONITH'
	* thus breaking the loop
	*/
	notify_data_t *n_data = create_notification_boundaries(rsc, RSC_STOP, NULL, stonith_op, data_set);
	crm_info("Creating secondary notification for %s", action->uuid);

	collect_notification_data(rsc, TRUE, FALSE, n_data);
	g_hash_table_insert(n_data->keys, crm_strdup("notify_stop_resource"), crm_strdup(rsc->id));
	g_hash_table_insert(n_data->keys, crm_strdup("notify_stop_uname"), crm_strdup(action->node->details->uname));
	create_notifications(uber_parent(rsc), n_data, data_set);
	free_notification_data(n_data);
	}

	/* find the top-most resource */
	parent = rsc->parent;
	while(parent != NULL && parent->parent != NULL) {
	parent = parent->parent;
	}

	if(parent) {
	crm_debug_2("Re-creating actions for %s", parent->id);
	parent->cmds->create_actions(parent, data_set);

	/* make sure we dont mess anything up in create_actions */
	CRM_CHECK(action->pseudo, action->pseudo = TRUE);
	CRM_CHECK(action->runnable, action->runnable = TRUE);
	}
	/* From Bug #1601, successful fencing must be an input to a failed resources stop action.

	However given group(rA, rB) running on nodeX and B.stop has failed,
	A := stop healthy resource (rA.stop)
	B := stop failed resource (pseudo operation B.stop)
	C := stonith nodeX
	A requires B, B requires C, C requires A
	This loop would prevent the cluster from making progress.

	This block creates the "C requires A" dependancy and therefore must (at least
	for now) be disabled.

	Instead, run the block above and treat all resources on nodeX as B would be
	(marked as a pseudo op depending on the STONITH).

	TODO: Break the "A requires B" dependancy in update_action() and re-enable this block

	} else if(is_stonith == FALSE) {
	crm_info("Moving healthy resource %s"
	" off %s before fencing",
	rsc->id, node->details->uname);

	* stop healthy resources before the
	* stonith op
	*
	custom_action_order(
	rsc, stop_key(rsc), NULL,
	NULL,crm_strdup(CRM_OP_FENCE),stonith_op,
	pe_order_optional, data_set);
	*/
	);

	g_list_free(action_list);

	key = demote_key(rsc);
	action_list = find_actions(rsc->actions, key, stonith_op->node);
	crm_free(key);

	slist_iter(
	action, action_t, action_list, lpc2,
	if(action->node->details->online == FALSE \|\| is_set(rsc->flags, pe_rsc_failed)) {
	crm_info("Demote of failed resource %s is"
	" implict after %s is fenced",
	rsc->id, action->node->details->uname);
	/* the stop would never complete and is
	* now implied by the stonith operation
	*/
	action->pseudo = TRUE;
	action->runnable = TRUE;
	if(is_stonith == FALSE) {
	order_actions(stonith_op, action, pe_order_optional);
	}
	}
	);

	g_list_free(action_list);
	}

	void
	complex_stonith_ordering(
	resource_t rsc, action_t stonith_op, pe_working_set_t *data_set)
	{
	gboolean is_stonith = FALSE;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if(rsc->children) {
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->stonith_ordering(
	child_rsc, stonith_op, data_set);
	);
	return;
	}

	if(is_not_set(rsc->flags, pe_rsc_managed)) {
	crm_debug_3("Skipping fencing constraints for unmanaged resource: %s", rsc->id);
	return;
	}

	if(stonith_op != NULL && safe_str_eq(class, "stonith")) {
	is_stonith = TRUE;
	}

	/* Start constraints */
	native_start_constraints(rsc, stonith_op, is_stonith, data_set);

	/* Stop constraints */
	native_stop_constraints(rsc, stonith_op, is_stonith, data_set);
	}

	#define ALLOW_WEAK_MIGRATION 0

	enum stack_activity
	{
	stack_stable = 0,
	stack_starting = 1,
	stack_stopping = 2,
	stack_middle = 4,
	};

	static enum stack_activity
	find_clone_activity_on(resource_t rsc, resource_t target, node_t node, const char type)
	{
	int mode = stack_stable;
	action_t *active = NULL;

	if(target->children) {
	slist_iter(
	child, resource_t, target->children, lpc,
	mode \|= find_clone_activity_on(rsc, child, node, type);
	);
	return mode;
	}

	active = find_first_action(target->actions, NULL, CRMD_ACTION_START, NULL);
	if(active && active->optional == FALSE && active->pseudo == FALSE) {
	crm_debug("%s: found scheduled %s action (%s)", rsc->id, active->uuid, type);
	mode \|= stack_starting;
	}

	active = find_first_action(target->actions, NULL, CRMD_ACTION_STOP, node);
	if(active && active->optional == FALSE && active->pseudo == FALSE) {
	crm_debug("%s: found scheduled %s action (%s)", rsc->id, active->uuid, type);
	mode \|= stack_stopping;
	}

	return mode;
	}

	static enum stack_activity
	check_stack_element(resource_t rsc, resource_t other_rsc, const char *type)
	{
	if(other_rsc == NULL \|\| other_rsc == rsc) {
	return stack_stable;

	} else if(other_rsc->variant == pe_native) {
	crm_notice("Cannot migrate %s due to dependancy on %s (%s)",
	rsc->id, other_rsc->id, type);
	return stack_middle;

	} else if(other_rsc == rsc->parent) {
	int mode = 0;
	slist_iter(constraint, rsc_colocation_t, other_rsc->rsc_cons, lpc,
	if(constraint->score > 0) {
	mode \|= check_stack_element(rsc, constraint->rsc_rh, type);
	}
	);
	return mode;

	} else if(other_rsc->variant == pe_group) {
	crm_notice("Cannot migrate %s due to dependancy on group %s (%s)",
	rsc->id, other_rsc->id, type);
	return stack_middle;

	} /* else: >= clone */

	/*

	## Assumption
	A depends on clone(B)

	## Resource Activity During Move

	N1 N2 N3
	--- --- ---
	t0 A.stop
	t1 B.stop B.stop
	t2 B.start B.start
	t3 A.start

	## Resource Activity During Migration

	N1 N2 N3
	--- --- ---
	t0 B.start B.start
	t1 A.stop (1)
	t2 A.start (2)
	t3 B.stop B.stop

	Node 1: Rewritten to be a migrate-to operation
	Node 2: Rewritten to be a migrate-from operation

	# Constraints

	The following constraints already exist in the system.
	The 'ok' and 'fail' column refers to whether they still hold for migration.

	a) A.stop -> A.start - ok
	b) B.stop -> B.start - fail

	c) A.stop -> B.stop - ok
	d) B.start -> A.start - ok

	e) B.stop -> A.start - fail
	f) A.stop -> B.start - fail

	## Scenarios
	B unchanged - ok
	B stopping only - fail - possible after fixing 'e'
	B starting only - fail - possible after fixing 'f'
	B stoping and starting - fail - constraint 'b' is unfixable
	B restarting only on N2 - fail - as-per previous only rarer

	*/

	/* Only allow migration when the clone is either stable, only starting or only stopping */
	return find_clone_activity_on(rsc, other_rsc, NULL, type);
	}

	static gboolean
	at_stack_bottom(resource_t *rsc)
	{
	char *key = NULL;
	action_t *start = NULL;
	action_t *other = NULL;
	int mode = stack_stable;
	GListPtr action_list = NULL;

	key = start_key(rsc);
	action_list = find_actions(rsc->actions, key, NULL);
	crm_free(key);

	crm_debug_3("%s: processing", rsc->id);
	CRM_CHECK(action_list != NULL, return FALSE);

	start = action_list->data;
	g_list_free(action_list);

	slist_iter(
	constraint, rsc_colocation_t, rsc->rsc_cons, lpc,

	resource_t *target = constraint->rsc_rh;
	crm_debug_4("Checking %s: %s == %s (%d)", constraint->id, rsc->id, target->id, constraint->score);
	if(constraint->score > 0) {
	mode \|= check_stack_element(rsc, target, "coloc");
	if(mode & stack_middle) {
	return FALSE;

	} else if((mode & stack_stopping) && (mode & stack_starting)) {
	crm_notice("Cannot migrate %s due to colocation activity (last was %s)",
	rsc->id, target->id);
	return FALSE;
	}
	}
	);

	slist_iter(
	other_w, action_wrapper_t, start->actions_before, lpc,
	other = other_w->action;

	#if ALLOW_WEAK_MIGRATION
	if((other_w->type & pe_order_implies_right) == 0) {
	crm_debug_3("%s: depends on %s (optional ordering)",
	rsc->id, other->uuid);
	continue;
	}
	#endif

	crm_debug_2("%s: Checking %s ordering", rsc->id, other->uuid);

	if(other->optional == FALSE) {
	mode \|= check_stack_element(rsc, other->rsc, "order");
	if(mode & stack_middle) {
	return FALSE;

	} else if((mode & stack_stopping) && (mode & stack_starting)) {
	crm_notice("Cannot migrate %s due to ordering activity (last was %s)",
	rsc->id, other->rsc->id);
	return FALSE;
	}
	}

	);

	return TRUE;
	}

	void
	complex_migrate_reload(resource_t rsc, pe_working_set_t data_set)
	{
	char *key = NULL;
	int level = LOG_DEBUG;
	GListPtr action_list = NULL;

	action_t *stop = NULL;
	action_t *start = NULL;
	action_t *other = NULL;
	action_t *action = NULL;
	const char *value = NULL;

	if(rsc->children) {
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->migrate_reload(child_rsc, data_set);
	);
	other = NULL;
	return;
	} else if(rsc->variant > pe_native) {
	return;
	}

	do_crm_log_unlikely(level+1, "Processing %s", rsc->id);

	if(is_not_set(rsc->flags, pe_rsc_managed)
	\|\| is_set(rsc->flags, pe_rsc_failed)
	\|\| is_set(rsc->flags, pe_rsc_start_pending)
	\|\| rsc->next_role < RSC_ROLE_STARTED
	\|\| g_list_length(rsc->running_on) != 1) {
	do_crm_log_unlikely(
	level+1, "%s: general resource state: flags=0x%.16llx",
	rsc->id, rsc->flags);
	return;
	}

	value = g_hash_table_lookup(rsc->meta, XML_OP_ATTR_ALLOW_MIGRATE);
	if(crm_is_true(value)) {
	set_bit(rsc->flags, pe_rsc_can_migrate);
	}

	if(rsc->next_role > RSC_ROLE_SLAVE) {
	clear_bit(rsc->flags, pe_rsc_can_migrate);
	do_crm_log_unlikely(
	level+1, "%s: resource role: role=%s", rsc->id, role2text(rsc->next_role));
	}

	key = start_key(rsc);
	action_list = find_actions(rsc->actions, key, NULL);
	crm_free(key);

	if(action_list == NULL) {
	do_crm_log_unlikely(level, "%s: no start action", rsc->id);
	return;
	}

	start = action_list->data;
	g_list_free(action_list);

	if(is_not_set(rsc->flags, pe_rsc_can_migrate)
	&& start->allow_reload_conversion == FALSE) {
	do_crm_log_unlikely(level+1, "%s: no need to continue", rsc->id);
	return;
	}

	key = stop_key(rsc);
	action_list = find_actions(rsc->actions, key, NULL);
	crm_free(key);

	if(action_list == NULL) {
	do_crm_log_unlikely(level, "%s: no stop action", rsc->id);
	return;
	}

	stop = action_list->data;
	g_list_free(action_list);

	action = start;
	if(action->pseudo
	\|\| action->optional
	\|\| action->node == NULL
	\|\| action->runnable == FALSE) {
	do_crm_log_unlikely(level, "%s: %s", rsc->id, action->task);
	return;
	}

	action = stop;
	if(action->pseudo
	\|\| action->optional
	\|\| action->node == NULL
	\|\| action->runnable == FALSE) {
	do_crm_log_unlikely(level, "%s: %s", rsc->id, action->task);
	return;
	}

	if(is_set(rsc->flags, pe_rsc_can_migrate)) {
	if(start->node == NULL
	\|\| stop->node == NULL
	\|\| stop->node->details == start->node->details) {
	clear_bit(rsc->flags, pe_rsc_can_migrate);

	} else if(at_stack_bottom(rsc) == FALSE) {
	clear_bit(rsc->flags, pe_rsc_can_migrate);
	}
	}

	if(is_set(rsc->flags, pe_rsc_can_migrate)) {
	crm_info("Migrating %s from %s to %s", rsc->id,
	stop->node->details->uname,
	start->node->details->uname);

	crm_free(stop->uuid);
	crm_free(stop->task);
	stop->task = crm_strdup(RSC_MIGRATE);
	stop->uuid = generate_op_key(rsc->id, stop->task, 0);
	add_hash_param(stop->meta, "migrate_source",
	stop->node->details->uname);
	add_hash_param(stop->meta, "migrate_target",
	start->node->details->uname);

	/* Create the correct ordering ajustments based on find_clone_activity_on(); */

	slist_iter(
	constraint, rsc_colocation_t, rsc->rsc_cons, lpc,

	resource_t *target = constraint->rsc_rh;
	crm_info("Repairing %s: %s == %s (%d)", constraint->id, rsc->id, target->id, constraint->score);

	if(constraint->score > 0) {
	int mode = check_stack_element(rsc, target, "coloc");
	action_t *clone_stop = find_first_action(target->actions, NULL, RSC_STOP, NULL);
	action_t *clone_start = find_first_action(target->actions, NULL, RSC_STARTED, NULL);

	CRM_ASSERT(clone_stop != NULL);
	CRM_ASSERT(clone_start != NULL);
	CRM_ASSERT((mode & stack_middle) == 0);
	CRM_ASSERT(((mode & stack_stopping) && (mode & stack_starting)) == 0);

	if(mode & stack_stopping) {
	action_t *clone_stop = find_first_action(target->actions, NULL, RSC_STOP, NULL);
	action_t *clone_start = find_first_action(target->actions, NULL, RSC_STARTED, NULL);
	crm_debug("Creating %s.start -> %s.stop ordering", rsc->id, target->id);

	order_actions(start, clone_stop, pe_order_optional);

	slist_iter(
	other_w, action_wrapper_t, start->actions_before, lpc2,
	/* Needed if the clone's started pseudo-action ever gets printed in the graph */
	if(other_w->action == clone_start) {
	crm_debug("Breaking %s -> %s ordering", other_w->action->uuid, start->uuid);
	other_w->type = pe_order_none;
	}
	);

	} else if(mode & stack_starting) {
	crm_debug("Creating %s.started -> %s.stop ordering", target->id, rsc->id);

	order_actions(clone_start, stop, pe_order_optional);

	slist_iter(
	other_w, action_wrapper_t, clone_stop->actions_before, lpc2,
	/* Needed if the clone's stop pseudo-action ever gets printed in the graph */
	if(other_w->action == stop) {
	crm_debug("Breaking %s -> %s ordering", other_w->action->uuid, clone_stop->uuid);
	other_w->type = pe_order_none;
	}
	);
	}
	}
	);

	crm_free(start->uuid);
	crm_free(start->task);
	start->task = crm_strdup(RSC_MIGRATED);
	start->uuid = generate_op_key(rsc->id, start->task, 0);
	add_hash_param(start->meta, "migrate_source_uuid", stop->node->details->id);
	add_hash_param(start->meta, "migrate_source", stop->node->details->uname);
	add_hash_param(start->meta, "migrate_target", start->node->details->uname);

	/* Anything that needed stop to complete, now also needs start to have completed */
	slist_iter(
	other_w, action_wrapper_t, stop->actions_after, lpc,
	other = other_w->action;
	if(other->optional \|\| other->rsc != NULL) {
	continue;
	}
	crm_debug("Ordering %s before %s (stop)", start->uuid, other_w->action->uuid);
	order_actions(start, other, other_w->type);
	);

	/* Stop also needs anything that the start needed to have completed too */
	slist_iter(
	other_w, action_wrapper_t, start->actions_before, lpc,
	other = other_w->action;
	if(other->rsc == NULL) {
	/* nothing */
	} else if(other->optional \|\| other->rsc == rsc \|\| other->rsc == rsc->parent) {
	continue;
	}
	crm_debug("Ordering %s before %s (start)", other_w->action->uuid, stop->uuid);
	order_actions(other, stop, other_w->type);
	);

	} else if(start && stop
	&& start->allow_reload_conversion
	&& stop->node->details == start->node->details) {
	action_t *rewrite = NULL;

	start->pseudo = TRUE; /* easier than trying to delete it from the graph */

	action = NULL;
	key = promote_key(rsc);
	action_list = find_actions(rsc->actions, key, NULL);
	if(action_list) {
	action = action_list->data;
	}
	if(action && action->optional == FALSE) {
	action->pseudo = TRUE;
	}
	g_list_free(action_list);
	crm_free(key);

	action = NULL;
	key = demote_key(rsc);
	action_list = find_actions(rsc->actions, key, NULL);
	if(action_list) {
	action = action_list->data;
	}
	g_list_free(action_list);
	crm_free(key);

	if(action && action->optional == FALSE) {
	rewrite = action;
	stop->pseudo = TRUE;

	} else {
	rewrite = stop;
	}
	crm_info("Rewriting %s of %s on %s as a reload",
	rewrite->task, rsc->id, stop->node->details->uname);

	crm_free(rewrite->uuid);
	crm_free(rewrite->task);
	rewrite->task = crm_strdup("reload");
	rewrite->uuid = generate_op_key(rsc->id, rewrite->task, 0);

	} else {
	do_crm_log_unlikely(level+1, "%s nothing to do", rsc->id);
	}
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 6:05 AM (10 h, 53 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1018280
Default Alt Text: (112 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions